Remove more unnecessary load_weights methods (#47058)
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
This commit is contained in:
@@ -17,7 +17,6 @@ from vllm.model_executor.models.moss_audio import (
|
||||
MOSS_AUDIO_PLACEHOLDER,
|
||||
MOSS_AUDIO_TOKEN,
|
||||
MOSS_AUDIO_TOKEN_ID,
|
||||
GatedMLP,
|
||||
MossAudioConfig,
|
||||
MossAudioDummyInputsBuilder,
|
||||
MossAudioEncoder,
|
||||
@@ -594,53 +593,6 @@ def test_moss_qwen3_deepstack_keys_for_pp(monkeypatch):
|
||||
assert set(forward_tensors.tensors) == set(tensors.tensors)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("tp_size", [1, 2])
|
||||
def test_moss_audio_gated_mlp_tp_shapes_and_loading(monkeypatch, tp_size):
|
||||
from vllm.config import VllmConfig, set_current_vllm_config
|
||||
from vllm.config.device import DeviceConfig
|
||||
|
||||
_patch_tensor_parallel_for_linear_layers(monkeypatch, tp_size=tp_size)
|
||||
with set_current_vllm_config(VllmConfig(device_config=DeviceConfig(device="cpu"))):
|
||||
mlp = GatedMLP(input_size=4, hidden_size=8, output_size=6)
|
||||
|
||||
params = dict(mlp.named_parameters())
|
||||
assert params["gate_up_proj.weight"].shape == torch.Size([16 // tp_size, 4])
|
||||
assert params["down_proj.weight"].shape == torch.Size([6, 8 // tp_size])
|
||||
|
||||
gate_weight = torch.arange(32, dtype=torch.float32).reshape(8, 4)
|
||||
up_weight = torch.arange(100, 132, dtype=torch.float32).reshape(8, 4)
|
||||
down_weight = torch.arange(48, dtype=torch.float32).reshape(6, 8)
|
||||
loaded = mlp.load_weights(
|
||||
[
|
||||
("gate_proj.weight", gate_weight),
|
||||
("up_proj.weight", up_weight),
|
||||
("down_proj.weight", down_weight),
|
||||
]
|
||||
)
|
||||
|
||||
assert loaded == {"gate_up_proj.weight", "down_proj.weight"}
|
||||
shard = 8 // tp_size
|
||||
assert torch.equal(params["gate_up_proj.weight"][:shard], gate_weight[:shard])
|
||||
assert torch.equal(params["gate_up_proj.weight"][shard:], up_weight[:shard])
|
||||
assert torch.equal(params["down_proj.weight"], down_weight[:, : 8 // tp_size])
|
||||
|
||||
with set_current_vllm_config(VllmConfig(device_config=DeviceConfig(device="cpu"))):
|
||||
packed_mlp = GatedMLP(input_size=4, hidden_size=8, output_size=6)
|
||||
packed_params = dict(packed_mlp.named_parameters())
|
||||
loaded = packed_mlp.load_weights(
|
||||
[("gate_up_proj.weight", torch.cat([gate_weight, up_weight], dim=0))]
|
||||
)
|
||||
assert loaded == {"gate_up_proj.weight"}
|
||||
assert torch.equal(
|
||||
packed_params["gate_up_proj.weight"][:shard],
|
||||
gate_weight[:shard],
|
||||
)
|
||||
assert torch.equal(
|
||||
packed_params["gate_up_proj.weight"][shard:],
|
||||
up_weight[:shard],
|
||||
)
|
||||
|
||||
|
||||
def test_moss_audio_encoder_loads_realistic_attention_weight_names(monkeypatch):
|
||||
from vllm.config import VllmConfig, set_current_vllm_config
|
||||
from vllm.config.device import DeviceConfig
|
||||
|
||||
@@ -39,6 +39,7 @@ class FusedMoEWithLoRA(BaseLayerWithLoRA):
|
||||
self._ep_check()
|
||||
|
||||
routed_experts = self.base_layer.routed_experts
|
||||
routed_experts.lora_base_layer_prefix = "base_layer."
|
||||
assert not routed_experts.quant_method.is_monolithic, (
|
||||
"Monolithic kernels are not supported for Fused MoE LoRA."
|
||||
)
|
||||
|
||||
+20
-26
@@ -372,32 +372,26 @@ def process_packed_modules_mapping(
|
||||
model: nn.Module, force_2d_moe: bool = False
|
||||
) -> dict[str, list[str]]:
|
||||
if is_moe_model(model):
|
||||
if moe_packed_mapping := get_moe_expert_mapping(model):
|
||||
# This method generates and returns a dictionary mapping packed module
|
||||
# names to lists of their corresponding submodule names. It includes
|
||||
# both static mappings and dynamic mappings for expert layers, where
|
||||
# the expert indices are expanded based on the configured number
|
||||
# of routed experts.
|
||||
packed_modules_mapping = get_packed_modules_mapping(model)
|
||||
# The 2D mapping is needed when the model itself is 2D, or when
|
||||
# the engine forces the universal 2D wrapper via
|
||||
# enable_mixed_moe_lora_format (so 3D models can also load 2D
|
||||
# adapters through FusedMoEWithLoRA).
|
||||
if (not model.is_3d_moe_weight) or force_2d_moe:
|
||||
# Filter out malformed entries: non-gated MoE has empty
|
||||
# ckpt_up_proj_name which results in weight_name containing ".."
|
||||
# (e.g., "experts.0.." instead of "experts.0.layer_name.")
|
||||
packed_modules_mapping["experts"] = [
|
||||
weight_name.rstrip(".")
|
||||
for _, weight_name, _, _ in moe_packed_mapping
|
||||
if ".." not in weight_name
|
||||
]
|
||||
# This method generates and returns a dictionary mapping packed module
|
||||
# names to lists of their corresponding submodule names. It includes
|
||||
# both static mappings and dynamic mappings for expert layers, where
|
||||
# the expert indices are expanded based on the configured number
|
||||
# of routed experts.
|
||||
packed_modules_mapping = get_packed_modules_mapping(model)
|
||||
# The 2D mapping is needed when the model itself is 2D, or when
|
||||
# the engine forces the universal 2D wrapper via
|
||||
# enable_mixed_moe_lora_format (so 3D models can also load 2D
|
||||
# adapters through FusedMoEWithLoRA).
|
||||
if (not model.is_3d_moe_weight) or force_2d_moe:
|
||||
# Filter out malformed entries: non-gated MoE has empty
|
||||
# ckpt_up_proj_name which results in weight_name containing ".."
|
||||
# (e.g., "experts.0.." instead of "experts.0.layer_name.")
|
||||
packed_modules_mapping["experts"] = [
|
||||
weight_name.rstrip(".")
|
||||
for _, weight_name, _, _ in get_moe_expert_mapping(model)
|
||||
if ".." not in weight_name
|
||||
]
|
||||
|
||||
return packed_modules_mapping
|
||||
else:
|
||||
raise AttributeError(
|
||||
"To support LoRA for MoE model, "
|
||||
"'get_expert_mapping' must be implemented"
|
||||
)
|
||||
return packed_modules_mapping
|
||||
else:
|
||||
return get_packed_modules_mapping(model)
|
||||
|
||||
@@ -127,7 +127,7 @@ def FusedMoE(
|
||||
num_redundant_experts: int = 0,
|
||||
has_bias: bool = False,
|
||||
is_sequence_parallel: bool = False,
|
||||
expert_mapping: list[tuple[str, str, int, str]] | None = None,
|
||||
ckpt_names: tuple[str, str, str] = ("gate_proj", "down_proj", "up_proj"),
|
||||
n_shared_experts: int | None = None,
|
||||
router_logits_dtype: torch.dtype | None = None,
|
||||
gate: torch.nn.Module | None = None,
|
||||
@@ -356,7 +356,9 @@ def FusedMoE(
|
||||
moe_config,
|
||||
quant_config,
|
||||
expert_map_manager=expert_map_manager,
|
||||
expert_mapping=expert_mapping,
|
||||
ckpt_gate_proj_name=ckpt_names[0],
|
||||
ckpt_down_proj_name=ckpt_names[1],
|
||||
ckpt_up_proj_name=ckpt_names[2],
|
||||
# Extra params that are needed by quant_methods, pass along for now
|
||||
# Prefer getting these from other sources, e.g. moe_config or
|
||||
# router object
|
||||
|
||||
@@ -58,7 +58,9 @@ class RoutedExperts(PluggableLayer):
|
||||
moe_config: FusedMoEConfig,
|
||||
quant_config: QuantizationConfig | None,
|
||||
expert_map_manager: ExpertMapManager,
|
||||
expert_mapping: list[tuple[str, str, int, str]] | None = None,
|
||||
ckpt_gate_proj_name: str = "gate_proj",
|
||||
ckpt_down_proj_name: str = "down_proj",
|
||||
ckpt_up_proj_name: str = "up_proj",
|
||||
#
|
||||
# Extra params that are needed by quant_methods, pass along for now
|
||||
# Prefer getting these from other sources, e.g. moe_config or
|
||||
@@ -81,7 +83,9 @@ class RoutedExperts(PluggableLayer):
|
||||
self.layer_name = layer_name
|
||||
self.moe_config = moe_config
|
||||
self.quant_config = quant_config
|
||||
self.expert_mapping = expert_mapping
|
||||
self.ckpt_gate_proj_name = ckpt_gate_proj_name
|
||||
self.ckpt_down_proj_name = ckpt_down_proj_name
|
||||
self.ckpt_up_proj_name = ckpt_up_proj_name
|
||||
self.expert_map_manager = expert_map_manager
|
||||
self.hidden_size = moe_config.hidden_dim
|
||||
self.global_num_experts = moe_config.num_experts
|
||||
@@ -167,6 +171,8 @@ class RoutedExperts(PluggableLayer):
|
||||
|
||||
self.quant_method.create_weights(layer=self, **moe_quant_params)
|
||||
|
||||
self.lora_base_layer_prefix = ""
|
||||
|
||||
# TODO(bnell): Temporary hack. Get rid of this.
|
||||
def _replace_quant_method(self, quant_method: FusedMoEMethodBase):
|
||||
self.quant_method = quant_method
|
||||
@@ -863,26 +869,33 @@ class RoutedExperts(PluggableLayer):
|
||||
def load_weights(
|
||||
self, weights: Iterable[tuple[str, torch.Tensor]]
|
||||
) -> Iterable[str]:
|
||||
if (expert_mapping := self.expert_mapping) is None:
|
||||
raise ValueError(
|
||||
"`self.expert_mapping` must be provided to "
|
||||
"load weights using `self.load_weights`."
|
||||
)
|
||||
expert_mapping = self.get_expert_mapping(include_fused=True)
|
||||
unpadded_hidden = self.moe_config.hidden_dim_unpadded
|
||||
for expert_name, loaded_weight in weights:
|
||||
qual_name = f"{self.layer_name}.{expert_name}"
|
||||
# Fused expert weights can be identified by their 3D tensors
|
||||
is_fused = loaded_weight.dim() == 3
|
||||
matched = False
|
||||
for param_name, weight_name, expert_id, shard_id in expert_mapping:
|
||||
if weight_name not in qual_name:
|
||||
if matched and is_fused:
|
||||
break
|
||||
continue
|
||||
matched = True
|
||||
weight_name = qual_name.replace(weight_name, param_name)
|
||||
param_name = weight_name.removeprefix(f"{self.layer_name}.")
|
||||
param = getattr(self, param_name)
|
||||
# Fused expert weights can be identified by their 3D tensors
|
||||
if loaded_weight.dim() == 3:
|
||||
# Repurpose expert_id as shard_idx for deconcatenating w1 and w3
|
||||
if is_fused:
|
||||
if shard_id in {"w1", "w3"}:
|
||||
shard_idx = expert_id
|
||||
experts_shard = loaded_weight.chunk(2, dim=1)[shard_idx]
|
||||
if loaded_weight.shape[-1] != unpadded_hidden:
|
||||
# [..., hidden, intermediate] -> [..., intermediate, hidden]
|
||||
loaded_weight = loaded_weight.transpose(-1, -2)
|
||||
# Repurpose expert_id for deconcatenating w1 and w3
|
||||
experts_shard = loaded_weight.chunk(2, dim=1)[expert_id]
|
||||
else:
|
||||
if loaded_weight.shape[-2] != unpadded_hidden:
|
||||
# [..., intermediate, hidden] -> [..., hidden, intermediate]
|
||||
loaded_weight = loaded_weight.transpose(-1, -2)
|
||||
experts_shard = loaded_weight
|
||||
start = 0
|
||||
else:
|
||||
@@ -912,6 +925,27 @@ class RoutedExperts(PluggableLayer):
|
||||
)
|
||||
yield param_name
|
||||
|
||||
def get_expert_mapping(
|
||||
self,
|
||||
ckpt_gate_proj_name: str | None = None,
|
||||
ckpt_down_proj_name: str | None = None,
|
||||
ckpt_up_proj_name: str | None = None,
|
||||
include_fused: bool = False,
|
||||
) -> list[tuple[str, str, int, str]]:
|
||||
moe_config = self.moe_config
|
||||
num_fused_shared_experts = self.expert_map_manager.num_fused_shared_experts
|
||||
num_redundant_experts = moe_config.num_experts - moe_config.num_logical_experts
|
||||
return self.build_expert_params_mapping(
|
||||
ckpt_gate_proj_name or self.ckpt_gate_proj_name,
|
||||
ckpt_down_proj_name or self.ckpt_down_proj_name,
|
||||
ckpt_up_proj_name or self.ckpt_up_proj_name,
|
||||
num_experts=moe_config.num_logical_experts + num_fused_shared_experts,
|
||||
num_redundant_experts=num_redundant_experts,
|
||||
routed_experts_prefix="",
|
||||
lora_base_layer_prefix=self.lora_base_layer_prefix,
|
||||
include_fused=include_fused,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def make_expert_params_mapping(
|
||||
model: torch.nn.Module,
|
||||
@@ -921,6 +955,36 @@ class RoutedExperts(PluggableLayer):
|
||||
num_experts: int,
|
||||
num_redundant_experts: int = 0,
|
||||
routed_experts_prefix: str = "routed_experts",
|
||||
) -> list[tuple[str, str, int, str]]:
|
||||
"""Build the expert mapping, detecting the LoRA `base_layer.` prefix by
|
||||
scanning `model`'s parameters.
|
||||
|
||||
Legacy entry point for models that still hand-roll `load_weights`; the
|
||||
`RoutedExperts` weight loader uses `get_expert_mapping` /
|
||||
`build_expert_params_mapping` instead (which take the prefix directly).
|
||||
See `build_expert_params_mapping` for the returned tuple format.
|
||||
"""
|
||||
has_base_layer = any(".base_layer." in n for n, _ in model.named_parameters())
|
||||
return RoutedExperts.build_expert_params_mapping(
|
||||
ckpt_gate_proj_name,
|
||||
ckpt_down_proj_name,
|
||||
ckpt_up_proj_name,
|
||||
num_experts,
|
||||
num_redundant_experts,
|
||||
routed_experts_prefix,
|
||||
"base_layer." if has_base_layer else "",
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def build_expert_params_mapping(
|
||||
ckpt_gate_proj_name: str,
|
||||
ckpt_down_proj_name: str,
|
||||
ckpt_up_proj_name: str,
|
||||
num_experts: int,
|
||||
num_redundant_experts: int = 0,
|
||||
routed_experts_prefix: str = "routed_experts",
|
||||
lora_base_layer_prefix: str = "",
|
||||
include_fused: bool = False,
|
||||
) -> list[tuple[str, str, int, str]]:
|
||||
"""
|
||||
Create expert parameter mapping for weight loading with redundant experts.
|
||||
@@ -929,12 +993,13 @@ class RoutedExperts(PluggableLayer):
|
||||
when loading weights with EPLB redundant experts.
|
||||
|
||||
Args:
|
||||
model: The model containing the MoE layer
|
||||
ckpt_gate_proj_name: Name of gate projection in checkpoint
|
||||
ckpt_down_proj_name: Name of down projection in checkpoint
|
||||
ckpt_up_proj_name: Name of up projection in checkpoint
|
||||
num_experts: Number of logical (non-redundant) experts
|
||||
num_redundant_experts: Number of redundant experts
|
||||
lora_base_layer_prefix: Prefix to add if this layer is a LoRA base layer
|
||||
include_fused: Prepend the fused pre-fused-checkpoint entries
|
||||
|
||||
Returns:
|
||||
List of tuples (param_name, weight_name, expert_id, shard_id)
|
||||
@@ -956,22 +1021,39 @@ class RoutedExperts(PluggableLayer):
|
||||
)
|
||||
)
|
||||
|
||||
base_layer = (
|
||||
"base_layer."
|
||||
if any(".base_layer." in name for name, _ in model.named_parameters())
|
||||
else ""
|
||||
)
|
||||
|
||||
if routed_experts_prefix != "":
|
||||
routed_experts_prefix = f"{routed_experts_prefix}."
|
||||
|
||||
return [
|
||||
w13 = f"experts.{routed_experts_prefix}{lora_base_layer_prefix}w13_"
|
||||
w2 = f"experts.{routed_experts_prefix}{lora_base_layer_prefix}w2_"
|
||||
|
||||
fused_mapping = []
|
||||
if include_fused:
|
||||
gate_up = None
|
||||
if ckpt_gate_proj_name == "gate_proj" and ckpt_up_proj_name == "up_proj":
|
||||
gate_up = "gate_up_proj"
|
||||
elif ckpt_gate_proj_name == "w1" and ckpt_up_proj_name == "w3":
|
||||
gate_up = "w13"
|
||||
else:
|
||||
logger.warning(
|
||||
"Unexpected gate/up projection names: %s, %s. "
|
||||
"Fused gate/up mapping will be skipped.",
|
||||
ckpt_gate_proj_name,
|
||||
ckpt_up_proj_name,
|
||||
)
|
||||
if gate_up is not None:
|
||||
fused_mapping = [
|
||||
# (param_name, weight_name, expert_id, shard_id)
|
||||
(f"{w13}weight", f"experts.{gate_up}", 0, "w1"),
|
||||
(f"{w13}weight", f"experts.{gate_up}", 1, "w3"),
|
||||
(f"{w2}weight", f"experts.{ckpt_down_proj_name}", 0, "w2"),
|
||||
]
|
||||
|
||||
per_expert_mapping = [
|
||||
# (param_name, weight_name, expert_id, shard_id)
|
||||
(
|
||||
f"experts.{routed_experts_prefix}{base_layer}w13_"
|
||||
if weight_name in [ckpt_gate_proj_name, ckpt_up_proj_name]
|
||||
else f"experts.{routed_experts_prefix}{base_layer}w2_",
|
||||
f"experts.{physical_to_logical_map[expert_id]}.{weight_name}.{base_layer}",
|
||||
w13 if weight_name in [ckpt_gate_proj_name, ckpt_up_proj_name] else w2,
|
||||
f"experts.{physical_to_logical_map[expert_id]}.{weight_name}.{lora_base_layer_prefix}",
|
||||
expert_id,
|
||||
shard_id,
|
||||
)
|
||||
@@ -983,6 +1065,8 @@ class RoutedExperts(PluggableLayer):
|
||||
]
|
||||
]
|
||||
|
||||
return fused_mapping + per_expert_mapping
|
||||
|
||||
def get_expert_weights(self) -> Iterable[torch.Tensor]:
|
||||
def _maybe_make_contiguous(
|
||||
name: str, p: torch.nn.Parameter
|
||||
|
||||
@@ -292,6 +292,11 @@ class MoERunner(MoERunnerInterface):
|
||||
# For smuggling this layer into the fused moe custom op
|
||||
register_layer_for_moe_forward_op(get_current_vllm_config(), self)
|
||||
|
||||
def load_weights(
|
||||
self, weights: Iterable[tuple[str, torch.Tensor]]
|
||||
) -> Iterable[str]:
|
||||
return self.routed_experts.load_weights(weights)
|
||||
|
||||
def _select_forward(self) -> Callable:
|
||||
if current_platform.is_tpu() or current_platform.is_cpu():
|
||||
# TODO: Once the OOM issue for the TPU backend is resolved, we
|
||||
|
||||
@@ -567,12 +567,6 @@ class BitsAndBytesModelLoader(BaseModelLoader):
|
||||
|
||||
if is_moe_model(model):
|
||||
self.expert_params_mapping = get_moe_expert_mapping(model)
|
||||
if not self.expert_params_mapping:
|
||||
raise AttributeError(
|
||||
f"MoE Model {type(model).__name__} does not support "
|
||||
"BitsAndBytes quantization yet. Ensure this model has "
|
||||
"'get_expert_mapping' method."
|
||||
)
|
||||
# For some models like Molmo, we need to use hf_to_vllm_mapper
|
||||
# to ensure correct loading of weights.
|
||||
if hf_to_vllm_mapper := getattr(model, "hf_to_vllm_mapper", None):
|
||||
|
||||
@@ -2,8 +2,7 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Inference-only AfMoE model compatible with HuggingFace weights."""
|
||||
|
||||
import typing
|
||||
from collections.abc import Callable, Iterable
|
||||
from collections.abc import Iterable
|
||||
from itertools import islice
|
||||
|
||||
import torch
|
||||
@@ -21,7 +20,6 @@ from vllm.model_executor.layers.attention import Attention
|
||||
from vllm.model_executor.layers.fused_moe import (
|
||||
FusedMoE,
|
||||
MoERunner,
|
||||
fused_moe_make_expert_params_mapping,
|
||||
)
|
||||
from vllm.model_executor.layers.layernorm import RMSNorm
|
||||
from vllm.model_executor.layers.linear import (
|
||||
@@ -36,10 +34,6 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
|
||||
ParallelLMHead,
|
||||
VocabParallelEmbedding,
|
||||
)
|
||||
from vllm.model_executor.model_loader.weight_utils import (
|
||||
default_weight_loader,
|
||||
maybe_remap_kv_scale_name,
|
||||
)
|
||||
from vllm.model_executor.models.interfaces import (
|
||||
EagleModelMixin,
|
||||
MixtureOfExperts,
|
||||
@@ -53,7 +47,6 @@ from vllm.model_executor.models.utils import (
|
||||
PPMissingLayer,
|
||||
WeightsMapper,
|
||||
extract_layer_index,
|
||||
is_pp_missing_parameter,
|
||||
make_empty_intermediate_tensors_factory,
|
||||
make_layers,
|
||||
maybe_prefix,
|
||||
@@ -481,120 +474,6 @@ class AfmoeModel(nn.Module, EagleModelMixin):
|
||||
}
|
||||
)
|
||||
|
||||
def get_expert_mapping(self) -> list[tuple[str, str, int, str]]:
|
||||
# Params for weights, fp8 weight scales, fp8 activation scales
|
||||
# (param_name, weight_name, expert_id, shard_id)
|
||||
return fused_moe_make_expert_params_mapping(
|
||||
self,
|
||||
ckpt_gate_proj_name="gate_proj",
|
||||
ckpt_down_proj_name="down_proj",
|
||||
ckpt_up_proj_name="up_proj",
|
||||
num_experts=self.config.num_experts,
|
||||
)
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
stacked_params_mapping = [
|
||||
# (param_name, shard_name, shard_id)
|
||||
("qkv_proj", "q_proj", "q"),
|
||||
("qkv_proj", "k_proj", "k"),
|
||||
("qkv_proj", "v_proj", "v"),
|
||||
("gate_up_proj", "gate_proj", 0),
|
||||
("gate_up_proj", "up_proj", 1),
|
||||
]
|
||||
|
||||
params_dict = dict(self.named_parameters())
|
||||
loaded_params: set[str] = set()
|
||||
expert_params_mapping = self.get_expert_mapping()
|
||||
|
||||
for name, loaded_weight in weights:
|
||||
for param_name, weight_name, shard_id in stacked_params_mapping:
|
||||
# Skip non-stacked layers and experts (experts handled below).
|
||||
if (weight_name not in name) or ("self_attn.gate_proj" in name):
|
||||
continue
|
||||
# We have mlp.experts[0].gate_proj in the checkpoint.
|
||||
# Since we handle the experts below in expert_params_mapping,
|
||||
# we need to skip here BEFORE we update the name, otherwise
|
||||
# name will be updated to mlp.experts[0].gate_up_proj, which
|
||||
# will then be updated below in expert_params_mapping
|
||||
# for mlp.experts[0].gate_gate_up_proj, which breaks load.
|
||||
if ("mlp.experts." in name) and name not in params_dict:
|
||||
continue
|
||||
|
||||
name = name.replace(weight_name, param_name)
|
||||
# Skip loading extra bias for GPTQ models.
|
||||
if name.endswith(".bias") and name not in params_dict:
|
||||
continue
|
||||
if is_pp_missing_parameter(name, self):
|
||||
continue
|
||||
|
||||
param = params_dict[name]
|
||||
weight_loader = param.weight_loader
|
||||
weight_loader(param, loaded_weight, shard_id)
|
||||
break
|
||||
else:
|
||||
is_expert_weight = False
|
||||
for mapping in expert_params_mapping:
|
||||
param_name, weight_name, expert_id, shard_id = mapping
|
||||
if weight_name not in name:
|
||||
continue
|
||||
|
||||
# Anyway, this is an expert weight and should not be
|
||||
# attempted to load as other weights later
|
||||
is_expert_weight = True
|
||||
|
||||
# Do not modify `name` since the loop may continue here
|
||||
# Instead, create a new variable
|
||||
name_mapped = name.replace(weight_name, param_name)
|
||||
|
||||
if is_pp_missing_parameter(name_mapped, self):
|
||||
continue
|
||||
|
||||
param = params_dict[name_mapped]
|
||||
# We should ask the weight loader to return success or not
|
||||
# here since otherwise we may skip experts with other
|
||||
# available replicas.
|
||||
weight_loader = typing.cast(
|
||||
Callable[..., bool], param.weight_loader
|
||||
)
|
||||
success = weight_loader(
|
||||
param,
|
||||
loaded_weight,
|
||||
name_mapped,
|
||||
shard_id=shard_id,
|
||||
expert_id=expert_id,
|
||||
return_success=True,
|
||||
)
|
||||
if success:
|
||||
name = name_mapped
|
||||
break
|
||||
else:
|
||||
if is_expert_weight:
|
||||
# We've checked that this is an expert weight
|
||||
# However it's not mapped locally to this rank
|
||||
# So we simply skip it
|
||||
continue
|
||||
|
||||
# Skip loading extra bias for GPTQ models.
|
||||
if name.endswith(".bias") and name not in params_dict:
|
||||
continue
|
||||
|
||||
# Remapping the name of FP8 kv-scale.
|
||||
name = maybe_remap_kv_scale_name(name, params_dict)
|
||||
if name is None:
|
||||
continue
|
||||
|
||||
if is_pp_missing_parameter(name, self):
|
||||
continue
|
||||
|
||||
param = params_dict[name]
|
||||
weight_loader = getattr(
|
||||
param, "weight_loader", default_weight_loader
|
||||
)
|
||||
weight_loader(param, loaded_weight)
|
||||
loaded_params.add(name)
|
||||
|
||||
return loaded_params
|
||||
|
||||
|
||||
class AfmoeForCausalLM(
|
||||
nn.Module, SupportsPP, SupportsEagle3, SupportsLoRA, MixtureOfExperts
|
||||
@@ -615,6 +494,17 @@ class AfmoeForCausalLM(
|
||||
orig_to_new_suffix={
|
||||
".router.gate.weight": ".gate.weight",
|
||||
},
|
||||
orig_to_new_stacked={
|
||||
# weight_name: (param_name, shard_id)
|
||||
".q_proj": (".qkv_proj", "q"),
|
||||
".k_proj": (".qkv_proj", "k"),
|
||||
".v_proj": (".qkv_proj", "v"),
|
||||
# `.self_attn.gate_proj` is a gated-attention projection (not fused).
|
||||
".mlp.gate_proj": (".mlp.gate_up_proj", 0),
|
||||
".mlp.up_proj": (".mlp.gate_up_proj", 1),
|
||||
".shared_experts.gate_proj": (".shared_experts.gate_up_proj", 0),
|
||||
".shared_experts.up_proj": (".shared_experts.gate_up_proj", 1),
|
||||
},
|
||||
)
|
||||
|
||||
fall_back_to_pt_during_load = False
|
||||
@@ -705,6 +595,3 @@ class AfmoeForCausalLM(
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
loader = AutoWeightsLoader(self)
|
||||
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
|
||||
|
||||
def get_expert_mapping(self) -> list[tuple[str, str, int, str]]:
|
||||
return self.model.get_expert_mapping()
|
||||
|
||||
@@ -20,7 +20,7 @@ from vllm.model_executor.layers.linear import (
|
||||
RowParallelLinear,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization import QuantizationConfig
|
||||
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
|
||||
from vllm.model_executor.models.utils import AutoWeightsLoader, WeightsMapper
|
||||
from vllm.transformers_utils.configs.ovis import AIMv2Config
|
||||
|
||||
|
||||
@@ -194,6 +194,13 @@ class AIMv2Transformer(nn.Module):
|
||||
|
||||
|
||||
class AIMv2Model(torch.nn.Module):
|
||||
hf_to_vllm_mapper = WeightsMapper(
|
||||
orig_to_new_stacked={
|
||||
".fc1": (".fc13", 0),
|
||||
".fc3": (".fc13", 1),
|
||||
}
|
||||
)
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
config: AIMv2Config,
|
||||
@@ -218,34 +225,13 @@ class AIMv2Model(torch.nn.Module):
|
||||
return x
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
stacked_params_mapping = [
|
||||
# (param_name, shard_name, shard_id)
|
||||
(".fc13", ".fc1", 0),
|
||||
(".fc13", ".fc3", 1),
|
||||
]
|
||||
params_dict = dict(self.named_parameters())
|
||||
loaded_params: set[str] = set()
|
||||
|
||||
for name, loaded_weight in weights:
|
||||
# post_layernorm is optional in SiglipVisionModel
|
||||
if (
|
||||
name.startswith("trunk.post_trunk_norm")
|
||||
and self.trunk.post_trunk_norm is None
|
||||
):
|
||||
continue
|
||||
|
||||
for param_name, weight_name, shard_id in stacked_params_mapping:
|
||||
if weight_name not in name:
|
||||
continue
|
||||
name = name.replace(weight_name, param_name)
|
||||
|
||||
param = params_dict[name]
|
||||
weight_loader = param.weight_loader
|
||||
weight_loader(param, loaded_weight, shard_id)
|
||||
break
|
||||
else:
|
||||
param = params_dict[name]
|
||||
weight_loader = getattr(param, "weight_loader", default_weight_loader)
|
||||
weight_loader(param, loaded_weight)
|
||||
loaded_params.add(name)
|
||||
return loaded_params
|
||||
loader = AutoWeightsLoader(
|
||||
self,
|
||||
# post_trunk_norm is optional (absent for clip-skip backbones).
|
||||
skip_prefixes=(
|
||||
["trunk.post_trunk_norm."]
|
||||
if self.trunk.post_trunk_norm is None
|
||||
else None
|
||||
),
|
||||
)
|
||||
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
|
||||
|
||||
@@ -53,10 +53,6 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
|
||||
ParallelLMHead,
|
||||
VocabParallelEmbedding,
|
||||
)
|
||||
from vllm.model_executor.model_loader.weight_utils import (
|
||||
default_weight_loader,
|
||||
maybe_remap_kv_scale_name,
|
||||
)
|
||||
from vllm.sequence import IntermediateTensors
|
||||
from vllm.v1.attention.backend import AttentionType
|
||||
|
||||
@@ -70,8 +66,8 @@ from .interfaces import (
|
||||
from .utils import (
|
||||
AutoWeightsLoader,
|
||||
PPMissingLayer,
|
||||
WeightsMapper,
|
||||
extract_layer_index,
|
||||
is_pp_missing_parameter,
|
||||
make_empty_intermediate_tensors_factory,
|
||||
make_layers,
|
||||
maybe_prefix,
|
||||
@@ -404,66 +400,17 @@ class ApertusModel(nn.Module, EagleModelMixin):
|
||||
return hidden_states, aux_hidden_states
|
||||
return hidden_states
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
stacked_params_mapping = [
|
||||
# (param_name, shard_name, shard_id)
|
||||
(".qkv_proj", ".q_proj", "q"),
|
||||
(".qkv_proj", ".k_proj", "k"),
|
||||
(".qkv_proj", ".v_proj", "v"),
|
||||
]
|
||||
params_dict = dict(self.named_parameters())
|
||||
|
||||
# we need to load the buffers for beta and eps (XIELU)
|
||||
for name, buffer in self.named_buffers():
|
||||
if name.endswith(".beta") or name.endswith(".eps"):
|
||||
params_dict[name] = buffer
|
||||
|
||||
loaded_params: set[str] = set()
|
||||
for name, loaded_weight in weights:
|
||||
if "rotary_emb.inv_freq" in name:
|
||||
continue
|
||||
if "rotary_emb.cos_cached" in name or "rotary_emb.sin_cached" in name:
|
||||
# Models trained using ColossalAI may include these tensors in
|
||||
# the checkpoint. Skip them.
|
||||
continue
|
||||
if "scale" in name or "zero_point" in name:
|
||||
# Remapping the name of FP8 kv-scale.
|
||||
name = maybe_remap_kv_scale_name(name, params_dict)
|
||||
if name is None:
|
||||
continue
|
||||
for param_name, weight_name, shard_id in stacked_params_mapping:
|
||||
if weight_name not in name:
|
||||
continue
|
||||
name = name.replace(weight_name, param_name)
|
||||
# Skip loading extra bias for GPTQ models.
|
||||
if name.endswith(".bias") and name not in params_dict:
|
||||
continue
|
||||
|
||||
if is_pp_missing_parameter(name, self):
|
||||
continue
|
||||
|
||||
param = params_dict[name]
|
||||
weight_loader = param.weight_loader
|
||||
weight_loader(param, loaded_weight, shard_id)
|
||||
break
|
||||
else:
|
||||
# Skip loading extra bias for GPTQ models.
|
||||
if name.endswith(".bias") and name not in params_dict:
|
||||
continue
|
||||
|
||||
if is_pp_missing_parameter(name, self):
|
||||
continue
|
||||
|
||||
param = params_dict[name]
|
||||
weight_loader = getattr(param, "weight_loader", default_weight_loader)
|
||||
weight_loader(param, loaded_weight)
|
||||
loaded_params.add(name)
|
||||
return loaded_params
|
||||
|
||||
|
||||
class ApertusForCausalLM(
|
||||
nn.Module, SupportsLoRA, SupportsPP, SupportsEagle, SupportsEagle3
|
||||
):
|
||||
hf_to_vllm_mapper = WeightsMapper(
|
||||
orig_to_new_stacked={
|
||||
".q_proj": (".qkv_proj", "q"),
|
||||
".k_proj": (".qkv_proj", "k"),
|
||||
".v_proj": (".qkv_proj", "v"),
|
||||
}
|
||||
)
|
||||
packed_modules_mapping = {"qkv_proj": ["q_proj", "k_proj", "v_proj"]}
|
||||
|
||||
# LoRA specific attributes
|
||||
@@ -548,4 +495,4 @@ class ApertusForCausalLM(
|
||||
self,
|
||||
skip_prefixes=(["lm_head."] if self.config.tie_word_embeddings else None),
|
||||
)
|
||||
return loader.load_weights(weights)
|
||||
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
|
||||
|
||||
@@ -369,6 +369,14 @@ class BertModel(nn.Module, SupportsQuant):
|
||||
|
||||
packed_modules_mapping = {"qkv_proj": ["query", "key", "value"]}
|
||||
|
||||
hf_to_vllm_mapper = WeightsMapper(
|
||||
orig_to_new_stacked={
|
||||
".self.query": (".self.qkv_proj", "q"),
|
||||
".self.key": (".self.qkv_proj", "k"),
|
||||
".self.value": (".self.qkv_proj", "v"),
|
||||
}
|
||||
)
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
@@ -400,43 +408,9 @@ class BertModel(nn.Module, SupportsQuant):
|
||||
|
||||
return self.encoder(hidden_states)
|
||||
|
||||
def _load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]):
|
||||
stacked_params_mapping = [
|
||||
# (param_name, shard_name, shard_id)
|
||||
("qkv_proj", "query", "q"),
|
||||
("qkv_proj", "key", "k"),
|
||||
("qkv_proj", "value", "v"),
|
||||
]
|
||||
|
||||
loaded_stacked_params = []
|
||||
other_weights = []
|
||||
params_dict = dict(self.named_parameters())
|
||||
for name, loaded_weight in weights:
|
||||
for param_name, weight_name, shard_id in stacked_params_mapping:
|
||||
if weight_name not in name:
|
||||
continue
|
||||
|
||||
name = name.replace(weight_name, param_name)
|
||||
if name not in params_dict:
|
||||
continue
|
||||
param = params_dict[name]
|
||||
weight_loader = param.weight_loader
|
||||
weight_loader(param, loaded_weight, shard_id)
|
||||
loaded_stacked_params.append(name)
|
||||
break
|
||||
else:
|
||||
if name in params_dict:
|
||||
other_weights.append((name, loaded_weight))
|
||||
|
||||
return other_weights, loaded_stacked_params
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
other_weights, loaded_stacked_params = self._load_weights(weights)
|
||||
|
||||
loader = AutoWeightsLoader(self, skip_prefixes=["pooler."])
|
||||
loaded_params = loader.load_weights(other_weights)
|
||||
loaded_params.update(loaded_stacked_params)
|
||||
return loaded_params
|
||||
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
|
||||
|
||||
|
||||
class BertPoolingModel(BertModel):
|
||||
@@ -458,12 +432,8 @@ class BertPoolingModel(BertModel):
|
||||
self.pooler = BertPooler(vllm_config.model_config)
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
other_weights, loaded_stacked_params = self._load_weights(weights)
|
||||
|
||||
loader = AutoWeightsLoader(self)
|
||||
loaded_params = loader.load_weights(other_weights)
|
||||
loaded_params.update(loaded_stacked_params)
|
||||
return loaded_params
|
||||
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
|
||||
|
||||
|
||||
@default_pooling_type(seq_pooling_type="CLS")
|
||||
|
||||
@@ -19,9 +19,9 @@ from vllm.model_executor.layers.linear import (
|
||||
RowParallelLinear,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization import QuantizationConfig
|
||||
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
|
||||
|
||||
from .interfaces import SupportsQuant
|
||||
from .utils import AutoWeightsLoader, WeightsMapper
|
||||
|
||||
|
||||
def get_blip_patch_grid_length(*, image_size: int, patch_size: int) -> int:
|
||||
@@ -268,6 +268,14 @@ class BlipVisionModel(nn.Module, SupportsQuant):
|
||||
main_input_name = "pixel_values"
|
||||
packed_modules_mapping = {"qkv_proj": ["q_proj", "k_proj", "v_proj"]}
|
||||
|
||||
hf_to_vllm_mapper = WeightsMapper(
|
||||
orig_to_new_stacked={
|
||||
".q_proj": (".qkv", "q"),
|
||||
".k_proj": (".qkv", "k"),
|
||||
".v_proj": (".qkv", "v"),
|
||||
}
|
||||
)
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
config: BlipVisionConfig,
|
||||
@@ -316,38 +324,18 @@ class BlipVisionModel(nn.Module, SupportsQuant):
|
||||
return self.post_layernorm(hidden_states)
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
stacked_params_mapping = [
|
||||
# (param_name, shard_name, shard_id)
|
||||
("qkv_proj", "q_proj", "q"),
|
||||
("qkv_proj", "k_proj", "k"),
|
||||
("qkv_proj", "v_proj", "v"),
|
||||
]
|
||||
params_dict = dict(self.named_parameters())
|
||||
loaded_params: set[str] = set()
|
||||
layer_count = len(self.encoder.layers)
|
||||
skip_prefixes: list[str] = []
|
||||
if self.post_layernorm is None:
|
||||
skip_prefixes.append("post_layernorm.")
|
||||
loader = AutoWeightsLoader(self, skip_prefixes=skip_prefixes)
|
||||
|
||||
for name, loaded_weight in weights:
|
||||
# post_layernorm is not needed in BlipVisionModel
|
||||
if name.startswith("post_layernorm") and self.post_layernorm is None:
|
||||
continue
|
||||
|
||||
# omit layers when num_hidden_layers_override is set
|
||||
if name.startswith("encoder.layers"):
|
||||
layer_idx = int(name.split(".")[2])
|
||||
if layer_idx >= layer_count:
|
||||
# omit layers when num_hidden_layers_override is set
|
||||
def _filter(ws):
|
||||
for name, weight in ws:
|
||||
if name.startswith("encoder.layers.") and int(
|
||||
name.split(".")[2]
|
||||
) >= len(self.encoder.layers):
|
||||
continue
|
||||
yield name, weight
|
||||
|
||||
for param_name, weight_name, shard_id in stacked_params_mapping:
|
||||
if weight_name not in name:
|
||||
continue
|
||||
name = name.replace(weight_name, param_name)
|
||||
param = params_dict[name]
|
||||
weight_loader = param.weight_loader
|
||||
weight_loader(param, loaded_weight, shard_id)
|
||||
break
|
||||
else:
|
||||
param = params_dict[name]
|
||||
weight_loader = getattr(param, "weight_loader", default_weight_loader)
|
||||
weight_loader(param, loaded_weight)
|
||||
loaded_params.add(name)
|
||||
return loaded_params
|
||||
return loader.load_weights(_filter(weights), mapper=self.hf_to_vllm_mapper)
|
||||
|
||||
@@ -37,10 +37,7 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
|
||||
ParallelLMHead,
|
||||
VocabParallelEmbedding,
|
||||
)
|
||||
from vllm.model_executor.model_loader.weight_utils import (
|
||||
default_weight_loader,
|
||||
row_parallel_weight_loader,
|
||||
)
|
||||
from vllm.model_executor.model_loader.weight_utils import row_parallel_weight_loader
|
||||
from vllm.model_executor.utils import set_weight_attrs
|
||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
from vllm.multimodal.inputs import (
|
||||
@@ -66,7 +63,8 @@ from .interfaces import (
|
||||
SupportsQuant,
|
||||
)
|
||||
from .utils import (
|
||||
is_pp_missing_parameter,
|
||||
AutoWeightsLoader,
|
||||
WeightsMapper,
|
||||
make_empty_intermediate_tensors_factory,
|
||||
make_layers,
|
||||
maybe_prefix,
|
||||
@@ -922,6 +920,16 @@ class ChameleonForConditionalGeneration(
|
||||
"gate_up_proj": ["gate_proj", "up_proj"],
|
||||
}
|
||||
|
||||
hf_to_vllm_mapper = WeightsMapper(
|
||||
orig_to_new_stacked={
|
||||
".q_proj": (".qkv_proj", "q"),
|
||||
".k_proj": (".qkv_proj", "k"),
|
||||
".v_proj": (".qkv_proj", "v"),
|
||||
".gate_proj": (".gate_up_proj", 0),
|
||||
".up_proj": (".gate_up_proj", 1),
|
||||
},
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def get_placeholder_str(cls, modality: str, i: int) -> str | None:
|
||||
if modality.startswith("image"):
|
||||
@@ -1023,81 +1031,6 @@ class ChameleonForConditionalGeneration(
|
||||
return logits
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
stacked_params_mapping = [
|
||||
# (param_name, shard_name, shard_id)
|
||||
(".qkv_proj", ".q_proj", "q"),
|
||||
(".qkv_proj", ".k_proj", "k"),
|
||||
(".qkv_proj", ".v_proj", "v"),
|
||||
(".gate_up_proj", ".gate_proj", 0),
|
||||
(".gate_up_proj", ".up_proj", 1),
|
||||
]
|
||||
params_dict = dict(self.named_parameters())
|
||||
loaded_params: set[str] = set()
|
||||
for name, loaded_weight in weights:
|
||||
if "rotary_emb.inv_freq" in name:
|
||||
continue
|
||||
|
||||
if "rotary_emb.cos_cached" in name or "rotary_emb.sin_cached" in name:
|
||||
# Models trained using ColossalAI may include these tensors in
|
||||
# the checkpoint. Skip them.
|
||||
continue
|
||||
|
||||
# With tie_word_embeddings, we can skip lm_head.weight
|
||||
# The weight might appear unnecessarily in the files if the model is
|
||||
# processed with quantization, LoRA, fine-tuning, etc.
|
||||
if self.config.tie_word_embeddings and "lm_head.weight" in name:
|
||||
continue
|
||||
|
||||
use_default_weight_loading = False
|
||||
if "vqmodel" in name:
|
||||
if self.model.vqmodel is not None:
|
||||
# We only do sharding for language model and
|
||||
# not vqvae for now.
|
||||
use_default_weight_loading = True
|
||||
else:
|
||||
for param_name, weight_name, shard_id in stacked_params_mapping:
|
||||
if weight_name not in name:
|
||||
continue
|
||||
name = name.replace(weight_name, param_name)
|
||||
# Skip loading extra bias for GPTQ models.
|
||||
if name.endswith(".bias") and name not in params_dict:
|
||||
continue
|
||||
if is_pp_missing_parameter(name, self):
|
||||
continue
|
||||
param = params_dict[name]
|
||||
weight_loader = param.weight_loader
|
||||
weight_loader(param, loaded_weight, shard_id)
|
||||
break
|
||||
else:
|
||||
# Skip loading extra bias for GPTQ models.
|
||||
if name.endswith(".bias") and name not in params_dict:
|
||||
continue
|
||||
# Remapping the name of FP8 kv-scale.
|
||||
if name.endswith("kv_scale"):
|
||||
remapped_kv_scale_name = name.replace(
|
||||
".kv_scale", ".attn.kv_scale"
|
||||
)
|
||||
if remapped_kv_scale_name not in params_dict:
|
||||
logger.warning_once(
|
||||
"Found kv scale in the checkpoint (e.g. %s), but not found the expected name in the model (e.g. %s). kv-scale is not loaded.", # noqa: E501
|
||||
name,
|
||||
remapped_kv_scale_name,
|
||||
)
|
||||
continue
|
||||
else:
|
||||
name = remapped_kv_scale_name
|
||||
if is_pp_missing_parameter(name, self):
|
||||
continue
|
||||
param = params_dict[name]
|
||||
weight_loader = getattr(
|
||||
param, "weight_loader", default_weight_loader
|
||||
)
|
||||
weight_loader(param, loaded_weight)
|
||||
if use_default_weight_loading and name in params_dict:
|
||||
if is_pp_missing_parameter(name, self):
|
||||
continue
|
||||
param = params_dict[name]
|
||||
weight_loader = getattr(param, "weight_loader", default_weight_loader)
|
||||
weight_loader(param, loaded_weight)
|
||||
loaded_params.add(name)
|
||||
return loaded_params
|
||||
skip_prefixes = ["lm_head."] if self.config.tie_word_embeddings else None
|
||||
loader = AutoWeightsLoader(self, skip_prefixes=skip_prefixes)
|
||||
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
|
||||
|
||||
@@ -29,7 +29,6 @@ from vllm.model_executor.layers.linear import (
|
||||
from vllm.model_executor.layers.pooler import DispatchPooler
|
||||
from vllm.model_executor.layers.quantization import QuantizationConfig
|
||||
from vllm.model_executor.layers.vocab_parallel_embedding import VocabParallelEmbedding
|
||||
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
|
||||
from vllm.model_executor.models.interfaces import SupportsQuant
|
||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
from vllm.multimodal.inputs import (
|
||||
@@ -56,7 +55,7 @@ from vllm.utils.tensor_schema import TensorSchema, TensorShape
|
||||
|
||||
from .interfaces import MultiModalEmbeddings, SupportsMultiModal
|
||||
from .interfaces_base import default_pooling_type
|
||||
from .utils import AutoWeightsLoader, maybe_prefix
|
||||
from .utils import AutoWeightsLoader, WeightsMapper, maybe_prefix
|
||||
from .vision import (
|
||||
VisionEncoderInfo,
|
||||
VisionFeatureSelectStrategy,
|
||||
@@ -555,6 +554,14 @@ class CLIPEncoder(nn.Module):
|
||||
|
||||
|
||||
class CLIPTextTransformer(nn.Module):
|
||||
hf_to_vllm_mapper = WeightsMapper(
|
||||
orig_to_new_stacked={
|
||||
".q_proj": (".qkv_proj", "q"),
|
||||
".k_proj": (".qkv_proj", "k"),
|
||||
".v_proj": (".qkv_proj", "v"),
|
||||
}
|
||||
)
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
config: CLIPTextConfig,
|
||||
@@ -605,34 +612,19 @@ class CLIPTextTransformer(nn.Module):
|
||||
return last_hidden_state
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
stacked_params_mapping = [
|
||||
# (param_name, shard_name, shard_id)
|
||||
("qkv_proj", "q_proj", "q"),
|
||||
("qkv_proj", "k_proj", "k"),
|
||||
("qkv_proj", "v_proj", "v"),
|
||||
]
|
||||
params_dict = dict(self.named_parameters())
|
||||
loaded_params: set[str] = set()
|
||||
|
||||
for name, loaded_weight in weights:
|
||||
for param_name, weight_name, shard_id in stacked_params_mapping:
|
||||
if weight_name not in name:
|
||||
continue
|
||||
name = name.replace(weight_name, param_name)
|
||||
|
||||
param = params_dict[name]
|
||||
weight_loader = param.weight_loader
|
||||
weight_loader(param, loaded_weight, shard_id)
|
||||
break
|
||||
else:
|
||||
param = params_dict[name]
|
||||
weight_loader = getattr(param, "weight_loader", default_weight_loader)
|
||||
weight_loader(param, loaded_weight)
|
||||
loaded_params.add(name)
|
||||
return loaded_params
|
||||
loader = AutoWeightsLoader(self)
|
||||
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
|
||||
|
||||
|
||||
class CLIPVisionTransformer(nn.Module):
|
||||
hf_to_vllm_mapper = WeightsMapper(
|
||||
orig_to_new_stacked={
|
||||
".q_proj": (".qkv_proj", "q"),
|
||||
".k_proj": (".qkv_proj", "k"),
|
||||
".v_proj": (".qkv_proj", "v"),
|
||||
}
|
||||
)
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
config: CLIPVisionConfig,
|
||||
@@ -714,42 +706,21 @@ class CLIPVisionTransformer(nn.Module):
|
||||
return encoder_outputs
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
stacked_params_mapping = [
|
||||
# (param_name, shard_name, shard_id)
|
||||
("qkv_proj", "q_proj", "q"),
|
||||
("qkv_proj", "k_proj", "k"),
|
||||
("qkv_proj", "v_proj", "v"),
|
||||
]
|
||||
params_dict = dict(self.named_parameters())
|
||||
loaded_params: set[str] = set()
|
||||
layer_count = len(self.encoder.layers)
|
||||
skip_prefixes: list[str] = []
|
||||
if self.post_layernorm is None:
|
||||
skip_prefixes.append("post_layernorm.")
|
||||
loader = AutoWeightsLoader(self, skip_prefixes=skip_prefixes)
|
||||
|
||||
for name, loaded_weight in weights:
|
||||
# post_layernorm is not needed in CLIPVisionModel
|
||||
if name.startswith("post_layernorm") and self.post_layernorm is None:
|
||||
continue
|
||||
|
||||
# omit layers when num_hidden_layers_override is set
|
||||
if name.startswith("encoder.layers"):
|
||||
layer_idx = int(name.split(".")[2])
|
||||
if layer_idx >= layer_count:
|
||||
# Drop layers beyond num_hidden_layers_override.
|
||||
def _filter(ws):
|
||||
for name, w in ws:
|
||||
if name.startswith("encoder.layers.") and int(
|
||||
name.split(".")[2]
|
||||
) >= len(self.encoder.layers):
|
||||
continue
|
||||
yield name, w
|
||||
|
||||
for param_name, weight_name, shard_id in stacked_params_mapping:
|
||||
if weight_name not in name:
|
||||
continue
|
||||
name = name.replace(weight_name, param_name)
|
||||
|
||||
param = params_dict[name]
|
||||
weight_loader = param.weight_loader
|
||||
weight_loader(param, loaded_weight, shard_id)
|
||||
break
|
||||
else:
|
||||
param = params_dict[name]
|
||||
weight_loader = getattr(param, "weight_loader", default_weight_loader)
|
||||
weight_loader(param, loaded_weight)
|
||||
loaded_params.add(name)
|
||||
return loaded_params
|
||||
return loader.load_weights(_filter(weights), mapper=self.hf_to_vllm_mapper)
|
||||
|
||||
|
||||
class CLIPVisionModel(nn.Module):
|
||||
|
||||
@@ -18,7 +18,6 @@ from vllm.model_executor.layers.activation import SiluAndMul
|
||||
from vllm.model_executor.layers.attention import Attention
|
||||
from vllm.model_executor.layers.fused_moe import (
|
||||
FusedMoE,
|
||||
fused_moe_make_expert_params_mapping,
|
||||
)
|
||||
from vllm.model_executor.layers.linear import (
|
||||
MergedColumnParallelLinear,
|
||||
@@ -31,8 +30,6 @@ from vllm.model_executor.layers.quantization import QuantizationConfig
|
||||
from vllm.model_executor.layers.rotary_embedding import get_rope
|
||||
from vllm.model_executor.layers.vocab_parallel_embedding import VocabParallelEmbedding
|
||||
from vllm.model_executor.model_loader.weight_utils import (
|
||||
default_weight_loader,
|
||||
maybe_remap_kv_scale_name,
|
||||
row_parallel_weight_loader,
|
||||
)
|
||||
from vllm.model_executor.utils import set_weight_attrs
|
||||
@@ -43,8 +40,8 @@ from .commandr import LayerNorm
|
||||
from .interfaces import SupportsPP, SupportsQuant
|
||||
from .utils import (
|
||||
AutoWeightsLoader,
|
||||
WeightsMapper,
|
||||
extract_layer_index,
|
||||
is_pp_missing_parameter,
|
||||
make_empty_intermediate_tensors_factory,
|
||||
make_layers,
|
||||
maybe_prefix,
|
||||
@@ -462,88 +459,23 @@ class Cohere2MoeModel(nn.Module):
|
||||
hidden_states, _ = self.norm(hidden_states, residual)
|
||||
return hidden_states
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
stacked_params_mapping = [
|
||||
("qkv_proj", "q_proj", "q"),
|
||||
("qkv_proj", "k_proj", "k"),
|
||||
("qkv_proj", "v_proj", "v"),
|
||||
("gate_up_proj", "gate_proj", 0),
|
||||
("gate_up_proj", "up_proj", 1),
|
||||
]
|
||||
|
||||
expert_params_mapping = fused_moe_make_expert_params_mapping(
|
||||
self,
|
||||
ckpt_gate_proj_name="gate_proj",
|
||||
ckpt_down_proj_name="down_proj",
|
||||
ckpt_up_proj_name="up_proj",
|
||||
num_experts=self.config.num_experts,
|
||||
)
|
||||
|
||||
params_dict = dict(self.named_parameters())
|
||||
loaded_params: set[str] = set()
|
||||
for name, loaded_weight in weights:
|
||||
if "rotary_emb.inv_freq" in name:
|
||||
continue
|
||||
|
||||
for param_name, shard_name, shard_id in stacked_params_mapping:
|
||||
if shard_name not in name:
|
||||
continue
|
||||
if "mlp.experts" in name:
|
||||
continue
|
||||
name = name.replace(shard_name, param_name)
|
||||
if name.endswith(".bias") and name not in params_dict:
|
||||
continue
|
||||
if is_pp_missing_parameter(name, self):
|
||||
continue
|
||||
param = params_dict[name]
|
||||
weight_loader = param.weight_loader
|
||||
weight_loader(param, loaded_weight, shard_id)
|
||||
break
|
||||
else:
|
||||
for mapping in expert_params_mapping:
|
||||
param_name, weight_name, expert_id, shard_id = mapping
|
||||
if weight_name not in name:
|
||||
continue
|
||||
name = name.replace(weight_name, param_name)
|
||||
if is_pp_missing_parameter(name, self):
|
||||
continue
|
||||
if (
|
||||
name.endswith(".bias") or name.endswith("_bias")
|
||||
) and name not in params_dict:
|
||||
continue
|
||||
param = params_dict[name]
|
||||
weight_loader = param.weight_loader
|
||||
weight_loader(
|
||||
param,
|
||||
loaded_weight,
|
||||
name,
|
||||
shard_id=shard_id,
|
||||
expert_id=expert_id,
|
||||
)
|
||||
break
|
||||
else:
|
||||
if (
|
||||
name.endswith(".bias") or name.endswith("_bias")
|
||||
) and name not in params_dict:
|
||||
continue
|
||||
if is_pp_missing_parameter(name, self):
|
||||
continue
|
||||
name = maybe_remap_kv_scale_name(name, params_dict)
|
||||
if name is None:
|
||||
continue
|
||||
param = params_dict[name]
|
||||
weight_loader = getattr(
|
||||
param, "weight_loader", default_weight_loader
|
||||
)
|
||||
weight_loader(param, loaded_weight)
|
||||
loaded_params.add(name)
|
||||
|
||||
return loaded_params
|
||||
|
||||
|
||||
class Cohere2MoeForCausalLM(nn.Module, SupportsPP, SupportsQuant):
|
||||
is_text_generation_model = True
|
||||
|
||||
hf_to_vllm_mapper = WeightsMapper(
|
||||
orig_to_new_stacked={
|
||||
# weight_name: (param_name, shard_id)
|
||||
".q_proj": (".qkv_proj", "q"),
|
||||
".k_proj": (".qkv_proj", "k"),
|
||||
".v_proj": (".qkv_proj", "v"),
|
||||
# .experts.gate_up_proj must be handled by MoERunner.load_weights for EP
|
||||
".mlp.gate_proj": (".mlp.gate_up_proj", 0),
|
||||
".mlp.up_proj": (".mlp.gate_up_proj", 1),
|
||||
".shared_experts.gate_proj": (".shared_experts.gate_up_proj", 0),
|
||||
".shared_experts.up_proj": (".shared_experts.gate_up_proj", 1),
|
||||
}
|
||||
)
|
||||
packed_modules_mapping = {
|
||||
"qkv_proj": [
|
||||
"q_proj",
|
||||
@@ -599,4 +531,4 @@ class Cohere2MoeForCausalLM(nn.Module, SupportsPP, SupportsQuant):
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
loader = AutoWeightsLoader(self, skip_prefixes=["lm_head."])
|
||||
return loader.load_weights(weights)
|
||||
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
|
||||
|
||||
@@ -20,10 +20,6 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
|
||||
ParallelLMHead,
|
||||
VocabParallelEmbedding,
|
||||
)
|
||||
from vllm.model_executor.model_loader.weight_utils import (
|
||||
default_weight_loader,
|
||||
maybe_remap_kv_scale_name,
|
||||
)
|
||||
from vllm.model_executor.models.deepseek_v2 import (
|
||||
DeepseekV2ForCausalLM,
|
||||
DeepseekV2MLAAttention,
|
||||
@@ -34,6 +30,7 @@ from vllm.multimodal.inputs import NestedTensors
|
||||
from .interfaces import LocalArgmaxMixin
|
||||
from .utils import (
|
||||
AutoWeightsLoader,
|
||||
WeightsMapper,
|
||||
get_draft_quant_config,
|
||||
maybe_prefix,
|
||||
process_eagle_weight,
|
||||
@@ -270,44 +267,20 @@ class DeepseekV2Eagle3Model(nn.Module):
|
||||
|
||||
return hidden_states, aux_output
|
||||
|
||||
# midlayer rename + gate_up / MLA fused_qkv_a merges
|
||||
hf_to_vllm_mapper = WeightsMapper(
|
||||
orig_to_new_substr={"midlayer.": "layers.0."},
|
||||
orig_to_new_stacked={
|
||||
".gate_proj": (".gate_up_proj", 0),
|
||||
".up_proj": (".gate_up_proj", 1),
|
||||
".q_a_proj": (".fused_qkv_a_proj", 0),
|
||||
".kv_a_proj_with_mqa": (".fused_qkv_a_proj", 1),
|
||||
},
|
||||
)
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
stacked_params_mapping = [
|
||||
# (param_name, shard_name, shard_id)
|
||||
(".gate_up_proj", ".gate_proj", 0),
|
||||
(".gate_up_proj", ".up_proj", 1),
|
||||
(".fused_qkv_a_proj", ".q_a_proj", 0),
|
||||
(".fused_qkv_a_proj", ".kv_a_proj_with_mqa", 1),
|
||||
]
|
||||
params_dict = dict(self.named_parameters())
|
||||
loaded_params: set[str] = set()
|
||||
|
||||
for name, loaded_weight in weights:
|
||||
if "midlayer." in name:
|
||||
name = name.replace("midlayer.", "layers.0.")
|
||||
|
||||
# Remapping the name FP8 kv-scale
|
||||
if "scale" in name:
|
||||
name = maybe_remap_kv_scale_name(name, params_dict)
|
||||
if name is None:
|
||||
continue
|
||||
|
||||
for param_name, weight_name, shard_id in stacked_params_mapping:
|
||||
if weight_name not in name:
|
||||
continue
|
||||
name = name.replace(weight_name, param_name)
|
||||
param = params_dict[name]
|
||||
weight_loader = param.weight_loader
|
||||
weight_loader(param, loaded_weight, shard_id)
|
||||
break
|
||||
else:
|
||||
if name not in params_dict:
|
||||
continue
|
||||
param = params_dict[name]
|
||||
weight_loader = getattr(param, "weight_loader", default_weight_loader)
|
||||
weight_loader(param, loaded_weight)
|
||||
loaded_params.add(name)
|
||||
|
||||
return loaded_params
|
||||
loader = AutoWeightsLoader(self)
|
||||
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
|
||||
|
||||
|
||||
class Eagle3DeepseekV2ForCausalLM(LocalArgmaxMixin, DeepseekV2ForCausalLM):
|
||||
|
||||
@@ -32,7 +32,6 @@ from vllm.model_executor.layers.quantization import QuantizationConfig
|
||||
from vllm.model_executor.layers.rotary_embedding.common import (
|
||||
ApplyRotaryEmb,
|
||||
)
|
||||
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
|
||||
from vllm.model_executor.models.interfaces import (
|
||||
MultiModalEmbeddings,
|
||||
SupportsLoRA,
|
||||
@@ -355,36 +354,6 @@ class DotsSwiGLUFFN(nn.Module):
|
||||
x, _ = self.fc2(x)
|
||||
return x
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
stacked_params_mapping = [
|
||||
("fc13", "fc1", 0),
|
||||
("fc13", "fc3", 1),
|
||||
]
|
||||
params_dict = dict(self.named_parameters())
|
||||
loaded_params: set[str] = set()
|
||||
for name, loaded_weight in weights:
|
||||
for param_name, weight_name, shard_id in stacked_params_mapping:
|
||||
if weight_name not in name:
|
||||
continue
|
||||
name = name.replace(weight_name, param_name)
|
||||
# Skip loading extra bias for GPTQ models.
|
||||
if name.endswith(".bias") and name not in params_dict:
|
||||
continue
|
||||
param = params_dict[name]
|
||||
weight_loader = param.weight_loader
|
||||
weight_loader(param, loaded_weight, shard_id)
|
||||
break
|
||||
else:
|
||||
# Skip loading extra bias for GPTQ models.
|
||||
if name.endswith(".bias") and name not in params_dict:
|
||||
continue
|
||||
|
||||
param = params_dict[name]
|
||||
weight_loader = getattr(param, "weight_loader", default_weight_loader)
|
||||
weight_loader(param, loaded_weight)
|
||||
loaded_params.add(name)
|
||||
return loaded_params
|
||||
|
||||
|
||||
class DotsPatchEmbed(nn.Module):
|
||||
def __init__(self, config):
|
||||
@@ -622,6 +591,10 @@ class DotsOCRForCausalLM(nn.Module, SupportsMultiModal, SupportsPP, SupportsLoRA
|
||||
".attn.qkv_proj.": ".attn.qkv.",
|
||||
".attn.out_proj.": ".attn.proj.",
|
||||
},
|
||||
orig_to_new_stacked={
|
||||
".fc1.": (".fc13.", 0),
|
||||
".fc3.": (".fc13.", 1),
|
||||
},
|
||||
orig_to_new_prefix={
|
||||
"lm_head.": "language_model.lm_head.",
|
||||
"model.": "language_model.model.",
|
||||
|
||||
@@ -27,7 +27,6 @@ from collections.abc import Iterable
|
||||
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
from transformers import PretrainedConfig
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.model_executor.layers.layernorm import RMSNorm
|
||||
@@ -36,11 +35,10 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
|
||||
ParallelLMHead,
|
||||
VocabParallelEmbedding,
|
||||
)
|
||||
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
|
||||
from vllm.sequence import IntermediateTensors
|
||||
|
||||
from .llama import LlamaDecoderLayer
|
||||
from .utils import is_pp_missing_parameter, maybe_prefix
|
||||
from .utils import AutoWeightsLoader, WeightsMapper, maybe_prefix
|
||||
|
||||
|
||||
class ErnieMultiTokenPredictorLayer(nn.Module):
|
||||
@@ -147,6 +145,28 @@ class ErnieMTP(nn.Module):
|
||||
super().__init__()
|
||||
|
||||
self.config = vllm_config.model_config.hf_config
|
||||
# MTP weights are stored under a flat `mtp_*.0.` block in the
|
||||
# checkpoint; rewrite them into `model.layers.{spec_layer}.*`.
|
||||
spec_layer = self.config.num_hidden_layers
|
||||
self.hf_to_vllm_mapper = WeightsMapper(
|
||||
orig_to_new_substr={
|
||||
"model.mtp_emb_norm.0.": f"model.layers.{spec_layer}.mtp_emb_norm.",
|
||||
"model.mtp_hidden_norm.0.": (
|
||||
f"model.layers.{spec_layer}.mtp_hidden_norm."
|
||||
),
|
||||
"model.mtp_linear_proj.0.": (
|
||||
f"model.layers.{spec_layer}.mtp_linear_proj."
|
||||
),
|
||||
"model.mtp_block.0.": f"model.layers.{spec_layer}.mtp_block.",
|
||||
},
|
||||
orig_to_new_stacked={
|
||||
".q_proj": (".qkv_proj", "q"),
|
||||
".k_proj": (".qkv_proj", "k"),
|
||||
".v_proj": (".qkv_proj", "v"),
|
||||
".gate_proj": (".gate_up_proj", 0),
|
||||
".up_proj": (".gate_up_proj", 1),
|
||||
},
|
||||
)
|
||||
self.model = ErnieMultiTokenPredictor(
|
||||
vllm_config=vllm_config, prefix=maybe_prefix(prefix, "model")
|
||||
)
|
||||
@@ -185,94 +205,15 @@ class ErnieMTP(nn.Module):
|
||||
return self.model.compute_logits(hidden_states, self.lm_head, spec_step_idx)
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
stacked_params_mapping = [
|
||||
("qkv_proj", "q_proj", "q"),
|
||||
("qkv_proj", "k_proj", "k"),
|
||||
("qkv_proj", "v_proj", "v"),
|
||||
("gate_up_proj", "gate_proj", 0),
|
||||
("gate_up_proj", "up_proj", 1),
|
||||
]
|
||||
# Checkpoint bundles the full base model; only MTP, embed_tokens and
|
||||
# lm_head weights belong to this module.
|
||||
def _filter(
|
||||
weights: Iterable[tuple[str, torch.Tensor]],
|
||||
) -> Iterable[tuple[str, torch.Tensor]]:
|
||||
for name, weight in weights:
|
||||
if any(k in name for k in ("mtp", "embed_tokens", "lm_head")):
|
||||
yield name, weight
|
||||
|
||||
params_dict = dict(self.named_parameters())
|
||||
loaded_params: set[str] = set()
|
||||
for name, loaded_weight in weights:
|
||||
if self.config.tie_word_embeddings and name.endswith("lm_head.weight"):
|
||||
continue
|
||||
if "rotary_emb.inv_freq" in name:
|
||||
continue
|
||||
if "mtp" in name:
|
||||
name = self._rewrite_spec_layer_name(self.config, name)
|
||||
|
||||
for param_name, weight_name, shard_id in stacked_params_mapping:
|
||||
# Skip non-stacked layers and experts (experts handled below).
|
||||
if weight_name not in name:
|
||||
continue
|
||||
if "mtp" not in name:
|
||||
continue
|
||||
# We have mlp.experts[0].gate_proj in the checkpoint.
|
||||
# Since we handle the experts below in expert_params_mapping,
|
||||
# we need to skip here BEFORE we update the name, otherwise
|
||||
# name will be updated to mlp.experts[0].gate_up_proj, which
|
||||
# will then be updated below in expert_params_mapping
|
||||
# for mlp.experts[0].gate_gate_up_proj, which breaks load.
|
||||
if ("mlp.experts." in name) and name not in params_dict:
|
||||
continue
|
||||
name = name.replace(weight_name, param_name)
|
||||
# Skip loading extra bias for GPTQ models.
|
||||
if (
|
||||
name.endswith(".bias") or name.endswith("_bias")
|
||||
) and name not in params_dict:
|
||||
continue
|
||||
# Skip layers on other devices.
|
||||
if is_pp_missing_parameter(name, self):
|
||||
continue
|
||||
|
||||
param = params_dict[name]
|
||||
weight_loader = param.weight_loader
|
||||
weight_loader(param, loaded_weight, shard_id)
|
||||
break
|
||||
else:
|
||||
# Skip loading extra bias for GPTQ models.
|
||||
if (
|
||||
name.endswith(".bias") or name.endswith("_bias")
|
||||
) and name not in params_dict:
|
||||
continue
|
||||
# Skip layers on other devices.
|
||||
if is_pp_missing_parameter(name, self):
|
||||
continue
|
||||
|
||||
# According to DeepSeek-V3 Technical Report, MTP modules
|
||||
# shares embedding layer. We only load the first weights.
|
||||
if "mtp_" not in name and (
|
||||
"embed_tokens" not in name and "lm_head" not in name
|
||||
):
|
||||
continue
|
||||
|
||||
param = params_dict[name]
|
||||
weight_loader = getattr(param, "weight_loader", default_weight_loader)
|
||||
weight_loader(param, loaded_weight)
|
||||
loaded_params.add(name)
|
||||
return loaded_params
|
||||
|
||||
def _rewrite_spec_layer_name(self, config: PretrainedConfig, name: str) -> str:
|
||||
"""
|
||||
Rewrite the weight name to match the format of the original model.
|
||||
"""
|
||||
spec_layer_weight_names = [
|
||||
"embed_tokens",
|
||||
"mtp_emb_norm",
|
||||
"mtp_hidden_norm",
|
||||
"mtp_linear_proj",
|
||||
]
|
||||
layer_idx = config.num_hidden_layers
|
||||
for weight_name in spec_layer_weight_names:
|
||||
if weight_name in name:
|
||||
name = name.replace(
|
||||
f"model.{weight_name}.0.",
|
||||
f"model.layers.{layer_idx}.{weight_name}.",
|
||||
)
|
||||
return name
|
||||
name = name.replace(
|
||||
"model.mtp_block.0.", f"model.layers.{layer_idx}.mtp_block."
|
||||
)
|
||||
return name
|
||||
skip_prefixes = ["lm_head"] if self.config.tie_word_embeddings else []
|
||||
loader = AutoWeightsLoader(self, skip_prefixes=skip_prefixes)
|
||||
return loader.load_weights(_filter(weights), mapper=self.hf_to_vllm_mapper)
|
||||
|
||||
@@ -15,8 +15,7 @@
|
||||
# limitations under the License.
|
||||
"""Inference-only K-EXAONE-236B-A22B model compatible with HuggingFace weights."""
|
||||
|
||||
import typing
|
||||
from collections.abc import Callable, Iterable
|
||||
from collections.abc import Iterable
|
||||
from itertools import islice
|
||||
|
||||
import torch
|
||||
@@ -32,7 +31,6 @@ from vllm.distributed import (
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe import (
|
||||
FusedMoE,
|
||||
fused_moe_make_expert_params_mapping,
|
||||
)
|
||||
from vllm.model_executor.layers.layernorm import RMSNorm
|
||||
from vllm.model_executor.layers.linear import ReplicatedLinear
|
||||
@@ -43,10 +41,6 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
|
||||
ParallelLMHead,
|
||||
VocabParallelEmbedding,
|
||||
)
|
||||
from vllm.model_executor.model_loader.weight_utils import (
|
||||
default_weight_loader,
|
||||
maybe_remap_kv_scale_name,
|
||||
)
|
||||
from vllm.sequence import IntermediateTensors
|
||||
|
||||
from .exaone4 import Exaone4Attention as ExaoneMoeAttention
|
||||
@@ -55,8 +49,8 @@ from .interfaces import SupportsLoRA, SupportsPP
|
||||
from .utils import (
|
||||
AutoWeightsLoader,
|
||||
PPMissingLayer,
|
||||
WeightsMapper,
|
||||
extract_layer_index,
|
||||
is_pp_missing_parameter,
|
||||
make_empty_intermediate_tensors_factory,
|
||||
make_layers,
|
||||
maybe_prefix,
|
||||
@@ -325,142 +319,21 @@ class ExaoneMoeModel(nn.Module):
|
||||
hidden_states, _ = self.norm(hidden_states, residual)
|
||||
return hidden_states
|
||||
|
||||
def get_expert_mapping(self) -> list[tuple[str, str, int, str]]:
|
||||
# Params for weights, fp8 weight scales, fp8 activation scales
|
||||
# (param_name, weight_name, expert_id, shard_id)
|
||||
return fused_moe_make_expert_params_mapping(
|
||||
self,
|
||||
ckpt_gate_proj_name="gate_proj",
|
||||
ckpt_down_proj_name="down_proj",
|
||||
ckpt_up_proj_name="up_proj",
|
||||
num_experts=self.config.num_experts,
|
||||
num_redundant_experts=self.num_redundant_experts,
|
||||
)
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
stacked_params_mapping = [
|
||||
# (param_name, shard_name, shard_id)
|
||||
(".qkv_proj", ".q_proj", "q"),
|
||||
(".qkv_proj", ".k_proj", "k"),
|
||||
(".qkv_proj", ".v_proj", "v"),
|
||||
(".gate_up_proj", ".gate_proj", 0),
|
||||
(".gate_up_proj", ".up_proj", 1),
|
||||
]
|
||||
|
||||
# Skip loading extra parameters for GPTQ/modelopt models.
|
||||
ignore_suffixes = (
|
||||
".bias",
|
||||
"_bias",
|
||||
".k_scale",
|
||||
"_k_scale",
|
||||
".v_scale",
|
||||
"_v_scale",
|
||||
".weight_scale",
|
||||
"_weight_scale",
|
||||
".input_scale",
|
||||
"_input_scale",
|
||||
)
|
||||
|
||||
params_dict = dict(self.named_parameters())
|
||||
loaded_params: set[str] = set()
|
||||
expert_params_mapping = self.get_expert_mapping()
|
||||
for name, loaded_weight in weights:
|
||||
if name.startswith("mtp."):
|
||||
continue
|
||||
if "rotary_emb.inv_freq" in name:
|
||||
continue
|
||||
if "rotary_emb.cos_cached" in name or "rotary_emb.sin_cached" in name:
|
||||
# Models trained using ColossalAI may include these tensors in
|
||||
# the checkpoint. Skip them.
|
||||
continue
|
||||
for param_name, weight_name, shard_id in stacked_params_mapping:
|
||||
if weight_name not in name:
|
||||
continue
|
||||
if "mlp.experts" in name:
|
||||
continue
|
||||
name = name.replace(weight_name, param_name)
|
||||
# Skip loading extra bias for GPTQ models.
|
||||
if name.endswith(".bias") and name not in params_dict:
|
||||
continue
|
||||
if is_pp_missing_parameter(name, self):
|
||||
continue
|
||||
if name not in params_dict:
|
||||
continue
|
||||
|
||||
param = params_dict[name]
|
||||
weight_loader = param.weight_loader
|
||||
weight_loader(param, loaded_weight, shard_id)
|
||||
break
|
||||
else:
|
||||
is_expert_weight = False
|
||||
for mapping in expert_params_mapping:
|
||||
param_name, weight_name, expert_id, shard_id = mapping
|
||||
if weight_name not in name:
|
||||
continue
|
||||
|
||||
# Anyway, this is an expert weight and should not be
|
||||
# attempted to load as other weights later
|
||||
is_expert_weight = True
|
||||
|
||||
# Do not modify `name` since the loop may continue here
|
||||
# Instead, create a new variable
|
||||
name_mapped = name.replace(weight_name, param_name)
|
||||
|
||||
if is_pp_missing_parameter(name_mapped, self):
|
||||
continue
|
||||
|
||||
# Skip loading extra parameters for GPTQ/modelopt models.
|
||||
if (
|
||||
name_mapped.endswith(ignore_suffixes)
|
||||
and name_mapped not in params_dict
|
||||
):
|
||||
continue
|
||||
|
||||
param = params_dict[name_mapped]
|
||||
# We should ask the weight loader to return success or not
|
||||
# here since otherwise we may skip experts with other
|
||||
# available replicas.
|
||||
weight_loader = typing.cast(
|
||||
Callable[..., bool], param.weight_loader
|
||||
)
|
||||
success = weight_loader(
|
||||
param,
|
||||
loaded_weight,
|
||||
name_mapped,
|
||||
shard_id=shard_id,
|
||||
expert_id=expert_id,
|
||||
return_success=True,
|
||||
)
|
||||
if success:
|
||||
name = name_mapped
|
||||
break
|
||||
else:
|
||||
if is_expert_weight:
|
||||
continue
|
||||
# Skip loading extra bias for GPTQ models.
|
||||
if name.endswith(".bias") and name not in params_dict:
|
||||
continue
|
||||
# Skip loading extra parameters for GPTQ/modelopt models.
|
||||
if name.endswith(ignore_suffixes) and name not in params_dict:
|
||||
continue
|
||||
# Remapping the name of FP8 kv-scale.
|
||||
name = maybe_remap_kv_scale_name(name, params_dict)
|
||||
if name is None:
|
||||
continue
|
||||
|
||||
if is_pp_missing_parameter(name, self):
|
||||
continue
|
||||
|
||||
param = params_dict[name]
|
||||
weight_loader = getattr(
|
||||
param, "weight_loader", default_weight_loader
|
||||
)
|
||||
weight_loader(param, loaded_weight)
|
||||
loaded_params.add(name)
|
||||
return loaded_params
|
||||
|
||||
|
||||
class ExaoneMoeForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
|
||||
hf_to_vllm_mapper = WeightsMapper(
|
||||
orig_to_new_stacked={
|
||||
# weight_name: (param_name, shard_id)
|
||||
".q_proj": (".qkv_proj", "q"),
|
||||
".k_proj": (".qkv_proj", "k"),
|
||||
".v_proj": (".qkv_proj", "v"),
|
||||
# .experts.gate_up_proj must be handled by MoERunner.load_weights for EP
|
||||
".mlp.gate_proj": (".mlp.gate_up_proj", 0),
|
||||
".mlp.up_proj": (".mlp.gate_up_proj", 1),
|
||||
".shared_experts.gate_proj": (".shared_experts.gate_up_proj", 0),
|
||||
".shared_experts.up_proj": (".shared_experts.gate_up_proj", 1),
|
||||
}
|
||||
)
|
||||
packed_modules_mapping = {
|
||||
"qkv_proj": [
|
||||
"q_proj",
|
||||
@@ -554,5 +427,18 @@ class ExaoneMoeForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
|
||||
skip_prefixes=(
|
||||
["lm_head.", "mtp."] if self.config.tie_word_embeddings else ["mtp."]
|
||||
),
|
||||
# Skip loading extra parameters for GPTQ/modelopt models.
|
||||
ignore_unexpected_suffixes=[
|
||||
".bias",
|
||||
"_bias",
|
||||
".k_scale",
|
||||
"_k_scale",
|
||||
".v_scale",
|
||||
"_v_scale",
|
||||
".weight_scale",
|
||||
"_weight_scale",
|
||||
".input_scale",
|
||||
"_input_scale",
|
||||
],
|
||||
)
|
||||
return loader.load_weights(weights)
|
||||
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
|
||||
|
||||
@@ -18,15 +18,10 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
|
||||
ParallelLMHead,
|
||||
VocabParallelEmbedding,
|
||||
)
|
||||
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
|
||||
from vllm.model_executor.models.exaone_moe import ExaoneMoeDecoderLayer
|
||||
from vllm.sequence import IntermediateTensors
|
||||
|
||||
from .utils import (
|
||||
AutoWeightsLoader,
|
||||
is_pp_missing_parameter,
|
||||
maybe_prefix,
|
||||
)
|
||||
from .utils import AutoWeightsLoader, WeightsMapper, maybe_prefix
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
@@ -35,6 +30,17 @@ KVCache = tuple[torch.Tensor, torch.Tensor]
|
||||
|
||||
@support_torch_compile
|
||||
class ExaoneMoeMultiTokenPredictor(nn.Module):
|
||||
hf_to_vllm_mapper = WeightsMapper(
|
||||
orig_to_new_stacked={
|
||||
".q_proj": (".qkv_proj", "q"),
|
||||
".k_proj": (".qkv_proj", "k"),
|
||||
".v_proj": (".qkv_proj", "v"),
|
||||
# Scope to dense mlp; experts are handled separately.
|
||||
".mlp.gate_proj": (".mlp.gate_up_proj", 0),
|
||||
".mlp.up_proj": (".mlp.gate_up_proj", 1),
|
||||
}
|
||||
)
|
||||
|
||||
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
|
||||
super().__init__()
|
||||
|
||||
@@ -128,55 +134,8 @@ class ExaoneMoeMultiTokenPredictor(nn.Module):
|
||||
return hidden_states
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
stacked_params_mapping = [
|
||||
# (param_name, shard_name, shard_id)
|
||||
("qkv_proj", "q_proj", "q"),
|
||||
("qkv_proj", "k_proj", "k"),
|
||||
("qkv_proj", "v_proj", "v"),
|
||||
("gate_up_proj", "gate_proj", 0),
|
||||
("gate_up_proj", "up_proj", 1),
|
||||
]
|
||||
|
||||
# Params for weights, fp8 weight scales, fp8 activation scales
|
||||
# (param_name, weight_name, expert_id, shard_id)
|
||||
params_dict = dict(self.named_parameters())
|
||||
loaded_params: set[str] = set()
|
||||
for name, loaded_weight in weights:
|
||||
if "rotary_emb.inv_freq" in name:
|
||||
continue
|
||||
|
||||
for param_name, weight_name, shard_id in stacked_params_mapping:
|
||||
if weight_name not in name:
|
||||
continue
|
||||
|
||||
if "mlp.experts" in name:
|
||||
continue
|
||||
|
||||
name = name.replace(weight_name, param_name)
|
||||
# Skip loading extra bias for GPTQ models.
|
||||
if name.endswith(".bias") and name not in params_dict:
|
||||
continue
|
||||
# Skip layers on other devices.
|
||||
if is_pp_missing_parameter(name, self):
|
||||
continue
|
||||
if name not in params_dict:
|
||||
continue
|
||||
param = params_dict[name]
|
||||
weight_loader = param.weight_loader
|
||||
weight_loader(param, loaded_weight, shard_id)
|
||||
break
|
||||
else:
|
||||
# Skip loading extra bias for GPTQ models.
|
||||
if name.endswith(".bias") and name not in params_dict:
|
||||
continue
|
||||
if is_pp_missing_parameter(name, self):
|
||||
continue
|
||||
|
||||
param = params_dict[name]
|
||||
weight_loader = getattr(param, "weight_loader", default_weight_loader)
|
||||
weight_loader(param, loaded_weight)
|
||||
loaded_params.add(name)
|
||||
return loaded_params
|
||||
loader = AutoWeightsLoader(self)
|
||||
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
|
||||
|
||||
|
||||
@support_torch_compile
|
||||
|
||||
@@ -35,10 +35,6 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
|
||||
ParallelLMHead,
|
||||
VocabParallelEmbedding,
|
||||
)
|
||||
from vllm.model_executor.model_loader.weight_utils import (
|
||||
default_weight_loader,
|
||||
maybe_remap_kv_scale_name,
|
||||
)
|
||||
from vllm.sequence import IntermediateTensors
|
||||
from vllm.transformers_utils.config import set_default_rope_theta
|
||||
|
||||
@@ -52,7 +48,7 @@ from .interfaces import (
|
||||
from .utils import (
|
||||
AutoWeightsLoader,
|
||||
PPMissingLayer,
|
||||
is_pp_missing_parameter,
|
||||
WeightsMapper,
|
||||
make_empty_intermediate_tensors_factory,
|
||||
make_layers,
|
||||
maybe_prefix,
|
||||
@@ -496,63 +492,6 @@ class FalconH1Model(nn.Module):
|
||||
hidden_states = self.final_layernorm(hidden_states)
|
||||
return hidden_states
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
stacked_params_mapping = [
|
||||
# (param_name, shard_name, shard_id)
|
||||
("qkv_proj", "q_proj", "q"),
|
||||
("qkv_proj", "k_proj", "k"),
|
||||
("qkv_proj", "v_proj", "v"),
|
||||
("gate_up_proj", "gate_proj", 0),
|
||||
("gate_up_proj", "up_proj", 1),
|
||||
]
|
||||
|
||||
params_dict = dict(self.named_parameters())
|
||||
loaded_params: set[str] = set()
|
||||
for name, loaded_weight in weights:
|
||||
if "rotary_emb.inv_freq" in name:
|
||||
continue
|
||||
|
||||
if "A_log" in name:
|
||||
name = name.replace("A_log", "A")
|
||||
|
||||
if "mamba" in name:
|
||||
name = name.replace("mamba", "mamba.mamba")
|
||||
|
||||
if "scale" in name:
|
||||
# Remapping the name of kv-scale.
|
||||
name = maybe_remap_kv_scale_name(name, params_dict)
|
||||
if name is None:
|
||||
continue
|
||||
|
||||
for param_name, weight_name, shard_id in stacked_params_mapping:
|
||||
if weight_name not in name:
|
||||
continue
|
||||
|
||||
name = name.replace(weight_name, param_name)
|
||||
# Skip loading extra bias for GPTQ models.
|
||||
if name.endswith(".bias") and name not in params_dict:
|
||||
continue
|
||||
# Skip layers on other devices.
|
||||
if is_pp_missing_parameter(name, self):
|
||||
continue
|
||||
param = params_dict[name]
|
||||
weight_loader = param.weight_loader
|
||||
weight_loader(param, loaded_weight, shard_id)
|
||||
break
|
||||
else:
|
||||
# Skip loading extra bias for GPTQ models.
|
||||
if name.endswith(".bias") and name not in params_dict:
|
||||
continue
|
||||
if is_pp_missing_parameter(name, self):
|
||||
continue
|
||||
|
||||
param = params_dict[name]
|
||||
weight_loader = getattr(param, "weight_loader", default_weight_loader)
|
||||
weight_loader(param, loaded_weight)
|
||||
loaded_params.add(name)
|
||||
|
||||
return loaded_params
|
||||
|
||||
|
||||
class FalconH1ForCausalLM(
|
||||
nn.Module,
|
||||
@@ -567,6 +506,20 @@ class FalconH1ForCausalLM(
|
||||
"gate_up_proj": ["gate_proj", "up_proj"],
|
||||
}
|
||||
|
||||
hf_to_vllm_mapper = WeightsMapper(
|
||||
orig_to_new_substr={
|
||||
"A_log": "A",
|
||||
"mamba": "mamba.mamba",
|
||||
},
|
||||
orig_to_new_stacked={
|
||||
".q_proj": (".qkv_proj", "q"),
|
||||
".k_proj": (".qkv_proj", "k"),
|
||||
".v_proj": (".qkv_proj", "v"),
|
||||
".gate_proj": (".gate_up_proj", 0),
|
||||
".up_proj": (".gate_up_proj", 1),
|
||||
},
|
||||
)
|
||||
|
||||
embedding_modules = {
|
||||
"embed_tokens": "input_embeddings",
|
||||
"lm_head": "output_embeddings",
|
||||
@@ -694,4 +647,4 @@ class FalconH1ForCausalLM(
|
||||
self,
|
||||
skip_prefixes=(["lm_head."] if self.tie_word_embeddings else None),
|
||||
)
|
||||
return loader.load_weights(weights)
|
||||
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
|
||||
|
||||
@@ -31,7 +31,6 @@ from vllm.model_executor.layers.linear import (
|
||||
)
|
||||
from vllm.model_executor.layers.logits_processor import LogitsProcessor
|
||||
from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
|
||||
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
|
||||
from vllm.model_executor.models.whisper_utils import (
|
||||
ISO639_1_SUPPORTED_LANGS,
|
||||
)
|
||||
@@ -618,6 +617,14 @@ class FunASRAudioInputs(TensorSchema):
|
||||
|
||||
|
||||
class FunASREncoder(nn.Module):
|
||||
hf_to_vllm_mapper = WeightsMapper(
|
||||
orig_to_new_stacked={
|
||||
".self_attn.q_proj": (".self_attn.qkv", "q"),
|
||||
".self_attn.k_proj": (".self_attn.qkv", "k"),
|
||||
".self_attn.v_proj": (".self_attn.qkv", "v"),
|
||||
}
|
||||
)
|
||||
|
||||
def __init__(
|
||||
self, *, vllm_config: VllmConfig, prefix: str = "", init_in_fp32: bool = False
|
||||
):
|
||||
@@ -637,35 +644,8 @@ class FunASREncoder(nn.Module):
|
||||
)
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
"""Load weights with mapping from HuggingFace format."""
|
||||
stacked_params_mapping = [
|
||||
# (param_name, shard_name, shard_id)
|
||||
("self_attn.qkv.", "self_attn.q_proj.", "q"),
|
||||
("self_attn.qkv.", "self_attn.k_proj.", "k"),
|
||||
("self_attn.qkv.", "self_attn.v_proj.", "v"),
|
||||
]
|
||||
params_dict = dict(self.named_parameters(remove_duplicate=False))
|
||||
loaded_params: set[str] = set()
|
||||
|
||||
for name, loaded_weight in weights:
|
||||
for param_name, weight_name, shard_id in stacked_params_mapping:
|
||||
if weight_name not in name:
|
||||
continue
|
||||
name = name.replace(weight_name, param_name)
|
||||
|
||||
param = params_dict[name]
|
||||
weight_loader = param.weight_loader
|
||||
weight_loader(param, loaded_weight, shard_id)
|
||||
break
|
||||
else:
|
||||
param = params_dict.get(name)
|
||||
if param is not None:
|
||||
weight_loader = getattr(
|
||||
param, "weight_loader", default_weight_loader
|
||||
)
|
||||
weight_loader(param, loaded_weight)
|
||||
loaded_params.add(name)
|
||||
return loaded_params
|
||||
loader = AutoWeightsLoader(self)
|
||||
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
|
||||
|
||||
|
||||
class FunASRModel(nn.Module):
|
||||
|
||||
@@ -44,7 +44,6 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
|
||||
ParallelLMHead,
|
||||
VocabParallelEmbedding,
|
||||
)
|
||||
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
|
||||
from vllm.sequence import IntermediateTensors
|
||||
|
||||
from .gemma4 import Gemma4MLP, _get_text_config
|
||||
@@ -412,44 +411,6 @@ class Gemma4MultiTokenPredictor(nn.Module):
|
||||
def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
|
||||
return self.embed_tokens(input_ids) * self.normalizer
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
stacked_params_mapping = [
|
||||
("gate_up_proj", "gate_proj", 0),
|
||||
("gate_up_proj", "up_proj", 1),
|
||||
]
|
||||
|
||||
params_dict = dict(self.named_parameters())
|
||||
params_dict.update(dict(self.named_buffers()))
|
||||
loaded_params: set[str] = set()
|
||||
|
||||
for name, loaded_weight in weights:
|
||||
if "rotary_emb.inv_freq" in name:
|
||||
continue
|
||||
|
||||
for param_name, weight_name, shard_id in stacked_params_mapping:
|
||||
if weight_name not in name:
|
||||
continue
|
||||
name = name.replace(weight_name, param_name)
|
||||
if name.endswith(".bias") and name not in params_dict:
|
||||
continue
|
||||
if name not in params_dict:
|
||||
continue
|
||||
param = params_dict[name]
|
||||
weight_loader = param.weight_loader
|
||||
weight_loader(param, loaded_weight, shard_id)
|
||||
break
|
||||
else:
|
||||
if name.endswith(".bias") and name not in params_dict:
|
||||
continue
|
||||
if name not in params_dict:
|
||||
continue
|
||||
param = params_dict[name]
|
||||
weight_loader = getattr(param, "weight_loader", default_weight_loader)
|
||||
weight_loader(param, loaded_weight)
|
||||
loaded_params.add(name)
|
||||
|
||||
return loaded_params
|
||||
|
||||
def forward(
|
||||
self,
|
||||
input_ids: torch.Tensor,
|
||||
@@ -502,6 +463,10 @@ class Gemma4MTP(nn.Module):
|
||||
"pre_projection.": "model.pre_projection.",
|
||||
"post_projection.": "model.post_projection.",
|
||||
},
|
||||
orig_to_new_stacked={
|
||||
".gate_proj": (".gate_up_proj", 0),
|
||||
".up_proj": (".gate_up_proj", 1),
|
||||
},
|
||||
)
|
||||
|
||||
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
|
||||
|
||||
@@ -76,7 +76,6 @@ from vllm.model_executor.layers.rotary_embedding import get_rope
|
||||
from vllm.model_executor.layers.rotary_embedding.common import (
|
||||
ApplyRotaryEmb,
|
||||
)
|
||||
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
|
||||
from vllm.model_executor.models.module_mapping import MultiModelKeys
|
||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
from vllm.multimodal.inputs import (
|
||||
@@ -618,6 +617,16 @@ class Glm4vVisionEmbeddings(nn.Module):
|
||||
|
||||
|
||||
class Glm4vVisionTransformer(nn.Module):
|
||||
hf_to_vllm_mapper = WeightsMapper(
|
||||
orig_to_new_stacked={
|
||||
".attn.q.": (".attn.qkv.", "q"),
|
||||
".attn.k.": (".attn.qkv.", "k"),
|
||||
".attn.v.": (".attn.qkv.", "v"),
|
||||
".gate_proj": (".gate_up_proj", 0),
|
||||
".up_proj": (".gate_up_proj", 1),
|
||||
}
|
||||
)
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
text_config: Glm4vTextConfig,
|
||||
@@ -958,33 +967,8 @@ class Glm4vVisionTransformer(nn.Module):
|
||||
return x
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
stacked_params_mapping = [
|
||||
# (param_name, shard_name, shard_id)
|
||||
("attn.qkv.", "attn.q.", "q"),
|
||||
("attn.qkv.", "attn.k.", "k"),
|
||||
("attn.qkv.", "attn.v.", "v"),
|
||||
("gate_up_proj", "gate_proj", 0),
|
||||
("gate_up_proj", "up_proj", 1),
|
||||
]
|
||||
params_dict = dict(self.named_parameters(remove_duplicate=False))
|
||||
loaded_params: set[str] = set()
|
||||
|
||||
for name, loaded_weight in weights:
|
||||
for param_name, weight_name, shard_id in stacked_params_mapping:
|
||||
if weight_name not in name:
|
||||
continue
|
||||
name = name.replace(weight_name, param_name)
|
||||
|
||||
param = params_dict[name]
|
||||
weight_loader = param.weight_loader
|
||||
weight_loader(param, loaded_weight, shard_id)
|
||||
break
|
||||
else:
|
||||
param = params_dict[name]
|
||||
weight_loader = getattr(param, "weight_loader", default_weight_loader)
|
||||
weight_loader(param, loaded_weight)
|
||||
loaded_params.add(name)
|
||||
return loaded_params
|
||||
loader = AutoWeightsLoader(self)
|
||||
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
|
||||
|
||||
|
||||
class Glm4vProcessingInfo(BaseProcessingInfo):
|
||||
|
||||
@@ -48,7 +48,6 @@ from vllm.model_executor.layers.linear import (
|
||||
RowParallelLinear,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization import QuantizationConfig
|
||||
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
|
||||
from vllm.model_executor.models.module_mapping import MultiModelKeys
|
||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
from vllm.multimodal.inputs import (
|
||||
@@ -431,6 +430,14 @@ class HunYuanVisionPatchMerger(nn.Module):
|
||||
|
||||
|
||||
class HunYuanVisionTransformer(nn.Module):
|
||||
hf_to_vllm_mapper = WeightsMapper(
|
||||
orig_to_new_stacked={
|
||||
".q_proj": (".qkv", "q"),
|
||||
".k_proj": (".qkv", "k"),
|
||||
".v_proj": (".qkv", "v"),
|
||||
}
|
||||
)
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
vision_config: HunYuanVLVisionConfig,
|
||||
@@ -529,31 +536,8 @@ class HunYuanVisionTransformer(nn.Module):
|
||||
return image_embeds_list
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
stacked_params_mapping = [
|
||||
# (param_name, shard_name, shard_id)
|
||||
(".qkv", ".q_proj", "q"),
|
||||
(".qkv", ".k_proj", "k"),
|
||||
(".qkv", ".v_proj", "v"),
|
||||
]
|
||||
params_dict = dict(self.named_parameters(remove_duplicate=False))
|
||||
loaded_params: set[str] = set()
|
||||
|
||||
for name, loaded_weight in weights:
|
||||
for param_name, weight_name, shard_id in stacked_params_mapping:
|
||||
if weight_name not in name:
|
||||
continue
|
||||
name = name.replace(weight_name, param_name)
|
||||
|
||||
param = params_dict[name]
|
||||
weight_loader = param.weight_loader
|
||||
weight_loader(param, loaded_weight, shard_id)
|
||||
break
|
||||
else:
|
||||
param = params_dict[name]
|
||||
weight_loader = getattr(param, "weight_loader", default_weight_loader)
|
||||
weight_loader(param, loaded_weight)
|
||||
loaded_params.add(name)
|
||||
return loaded_params
|
||||
loader = AutoWeightsLoader(self)
|
||||
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
|
||||
|
||||
|
||||
def _hunyuan_vl_field_config(hf_inputs: Mapping[str, torch.Tensor]):
|
||||
|
||||
@@ -38,8 +38,8 @@ from vllm.model_executor.layers.linear import (
|
||||
RowParallelLinear,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization import QuantizationConfig
|
||||
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
|
||||
|
||||
from .utils import AutoWeightsLoader, WeightsMapper
|
||||
from .vision import is_vit_use_data_parallel, run_dp_sharded_vision_model
|
||||
|
||||
|
||||
@@ -352,6 +352,14 @@ class Idefics2Encoder(nn.Module):
|
||||
|
||||
|
||||
class Idefics2VisionTransformer(nn.Module):
|
||||
hf_to_vllm_mapper = WeightsMapper(
|
||||
orig_to_new_stacked={
|
||||
".q_proj": (".qkv_proj", "q"),
|
||||
".k_proj": (".qkv_proj", "k"),
|
||||
".v_proj": (".qkv_proj", "v"),
|
||||
}
|
||||
)
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
config: Idefics2VisionConfig,
|
||||
@@ -453,42 +461,22 @@ class Idefics2VisionTransformer(nn.Module):
|
||||
return last_hidden_state
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
stacked_params_mapping = [
|
||||
# (param_name, shard_name, shard_id)
|
||||
("qkv_proj", "q_proj", "q"),
|
||||
("qkv_proj", "k_proj", "k"),
|
||||
("qkv_proj", "v_proj", "v"),
|
||||
]
|
||||
params_dict = dict(self.named_parameters())
|
||||
loaded_params: set[str] = set()
|
||||
# head is a pooling header absent from this model.
|
||||
skip_prefixes = ["head."]
|
||||
if not self.require_post_norm:
|
||||
skip_prefixes.append("post_layernorm.")
|
||||
loader = AutoWeightsLoader(self, skip_prefixes=skip_prefixes)
|
||||
|
||||
layer_count = len(self.encoder.layers)
|
||||
|
||||
for name, loaded_weight in weights:
|
||||
# skip pooling header
|
||||
if name.startswith("head."):
|
||||
continue
|
||||
|
||||
# post_layernorm is optional
|
||||
if name.startswith("post_layernorm.") and not self.require_post_norm:
|
||||
continue
|
||||
|
||||
# omit layers when num_hidden_layers_override is set
|
||||
if name.startswith("encoder.layers."):
|
||||
layer_idx = int(name.split(".")[2])
|
||||
if layer_idx >= layer_count:
|
||||
def _filter(ws: Iterable[tuple[str, torch.Tensor]]):
|
||||
# Drop layers beyond num_hidden_layers_override.
|
||||
for name, w in ws:
|
||||
if (
|
||||
name.startswith("encoder.layers.")
|
||||
and int(name.split(".")[2]) >= layer_count
|
||||
):
|
||||
continue
|
||||
yield name, w
|
||||
|
||||
for param_name, weight_name, shard_id in stacked_params_mapping:
|
||||
if weight_name not in name or self.use_data_parallel:
|
||||
continue
|
||||
name = name.replace(weight_name, param_name)
|
||||
param = params_dict[name]
|
||||
weight_loader = param.weight_loader
|
||||
weight_loader(param, loaded_weight, shard_id)
|
||||
break
|
||||
else:
|
||||
param = params_dict[name]
|
||||
weight_loader = getattr(param, "weight_loader", default_weight_loader)
|
||||
weight_loader(param, loaded_weight)
|
||||
loaded_params.add(name)
|
||||
return loaded_params
|
||||
return loader.load_weights(_filter(weights), mapper=self.hf_to_vllm_mapper)
|
||||
|
||||
@@ -25,9 +25,6 @@ from vllm.model_executor.layers.linear import (
|
||||
RowParallelLinear,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization import QuantizationConfig
|
||||
from vllm.model_executor.model_loader.weight_utils import (
|
||||
default_weight_loader,
|
||||
)
|
||||
from vllm.model_executor.models.interfaces import (
|
||||
MultiModalEmbeddings,
|
||||
SupportsLoRA,
|
||||
@@ -652,6 +649,14 @@ class Siglip2Encoder(nn.Module):
|
||||
|
||||
|
||||
class Siglip2VisionTransformer(nn.Module):
|
||||
hf_to_vllm_mapper = WeightsMapper(
|
||||
orig_to_new_stacked={
|
||||
".q_proj": (".qkv_proj", "q"),
|
||||
".k_proj": (".qkv_proj", "k"),
|
||||
".v_proj": (".qkv_proj", "v"),
|
||||
}
|
||||
)
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
config: PixelShuffleSiglip2VisionConfig,
|
||||
@@ -715,31 +720,8 @@ class Siglip2VisionTransformer(nn.Module):
|
||||
return hidden_states
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
stacked_params_mapping = [
|
||||
# (param_name, shard_name, shard_id)
|
||||
("qkv_proj", "q_proj", "q"),
|
||||
("qkv_proj", "k_proj", "k"),
|
||||
("qkv_proj", "v_proj", "v"),
|
||||
]
|
||||
params_dict = dict(self.named_parameters())
|
||||
loaded_params: set[str] = set()
|
||||
|
||||
for name, loaded_weight in weights:
|
||||
for param_name, weight_name, shard_id in stacked_params_mapping:
|
||||
if weight_name not in name:
|
||||
continue
|
||||
name = name.replace(weight_name, param_name)
|
||||
|
||||
param = params_dict[name]
|
||||
weight_loader = param.weight_loader
|
||||
weight_loader(param, loaded_weight, shard_id)
|
||||
break
|
||||
else:
|
||||
param = params_dict[name]
|
||||
weight_loader = getattr(param, "weight_loader", default_weight_loader)
|
||||
weight_loader(param, loaded_weight)
|
||||
loaded_params.add(name)
|
||||
return loaded_params
|
||||
loader = AutoWeightsLoader(self)
|
||||
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
|
||||
|
||||
|
||||
def _resolve_vision_token_id(model_config: ModelConfig, vision_token: str) -> int:
|
||||
|
||||
@@ -34,10 +34,6 @@ from vllm.model_executor.layers.quantization import QuantizationConfig
|
||||
from vllm.model_executor.layers.rotary_embedding.common import (
|
||||
ApplyRotaryEmb,
|
||||
)
|
||||
from vllm.model_executor.model_loader.weight_utils import (
|
||||
default_weight_loader,
|
||||
maybe_remap_kv_scale_name,
|
||||
)
|
||||
from vllm.model_executor.models.module_mapping import MultiModelKeys
|
||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
from vllm.multimodal.inputs import (
|
||||
@@ -76,7 +72,6 @@ from .utils import (
|
||||
AutoWeightsLoader,
|
||||
WeightsMapper,
|
||||
init_vllm_registered_model,
|
||||
is_pp_missing_parameter,
|
||||
maybe_prefix,
|
||||
)
|
||||
from .vision import is_vit_use_data_parallel
|
||||
@@ -718,6 +713,14 @@ class KeyeSiglipVisionModel(nn.Module):
|
||||
config_class = PretrainedConfig
|
||||
main_input_name = "pixel_values"
|
||||
|
||||
hf_to_vllm_mapper = WeightsMapper(
|
||||
orig_to_new_stacked={
|
||||
".q_proj": (".qkv_proj", "q"),
|
||||
".k_proj": (".qkv_proj", "k"),
|
||||
".v_proj": (".qkv_proj", "v"),
|
||||
}
|
||||
)
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
config: PretrainedConfig,
|
||||
@@ -776,53 +779,8 @@ class KeyeSiglipVisionModel(nn.Module):
|
||||
)
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
stacked_params_mapping = [
|
||||
("qkv_proj", "q_proj", "q"),
|
||||
("qkv_proj", "k_proj", "k"),
|
||||
("qkv_proj", "v_proj", "v"),
|
||||
]
|
||||
params_dict = dict(self.named_parameters(remove_duplicate=False))
|
||||
loaded_params: set[str] = set()
|
||||
for name, loaded_weight in weights:
|
||||
if "rotary_emb.inv_freq" in name:
|
||||
continue
|
||||
if "head.attention" in name or "head.layernorm" in name:
|
||||
continue
|
||||
if "head.mlp" in name or "head.probe" in name:
|
||||
continue
|
||||
for (
|
||||
param_name,
|
||||
weight_name,
|
||||
shard_id,
|
||||
) in stacked_params_mapping:
|
||||
if weight_name not in name:
|
||||
continue
|
||||
name = name.replace(weight_name, param_name)
|
||||
if name.endswith(".bias") and name not in params_dict:
|
||||
continue
|
||||
if is_pp_missing_parameter(name, self):
|
||||
continue
|
||||
param = params_dict[name]
|
||||
weight_loader = param.weight_loader
|
||||
weight_loader(param, loaded_weight, shard_id)
|
||||
break
|
||||
else:
|
||||
if name.endswith(".bias") and name not in params_dict:
|
||||
continue
|
||||
name = maybe_remap_kv_scale_name(name, params_dict)
|
||||
if name is None:
|
||||
continue
|
||||
if is_pp_missing_parameter(name, self):
|
||||
continue
|
||||
param = params_dict[name]
|
||||
weight_loader = getattr(
|
||||
param,
|
||||
"weight_loader",
|
||||
default_weight_loader,
|
||||
)
|
||||
weight_loader(param, loaded_weight)
|
||||
loaded_params.add(name)
|
||||
return loaded_params
|
||||
loader = AutoWeightsLoader(self, skip_prefixes=["vision_model.head."])
|
||||
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
|
||||
|
||||
|
||||
class Projector(nn.Module):
|
||||
|
||||
@@ -17,7 +17,6 @@ from vllm.config.multimodal import BaseDummyOptions
|
||||
from vllm.config.speech_to_text import SpeechToTextParams
|
||||
from vllm.inputs import PromptType, TokensPrompt
|
||||
from vllm.model_executor.model_loader import DefaultModelLoader
|
||||
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
|
||||
from vllm.model_executor.models.interfaces import (
|
||||
SupportsMultiModal,
|
||||
SupportsPP,
|
||||
@@ -81,6 +80,14 @@ class KimiAudioWhisperEncoder(WhisperEncoder):
|
||||
"qkv_proj": ["q_proj", "k_proj", "v_proj"],
|
||||
}
|
||||
|
||||
hf_to_vllm_mapper = WeightsMapper(
|
||||
orig_to_new_stacked={
|
||||
".self_attn.q_proj": (".self_attn.qkv_proj", "q"),
|
||||
".self_attn.k_proj": (".self_attn.qkv_proj", "k"),
|
||||
".self_attn.v_proj": (".self_attn.qkv_proj", "v"),
|
||||
},
|
||||
)
|
||||
|
||||
def __init__(
|
||||
self, *, vllm_config: VllmConfig, prefix: str = "", init_in_fp32: bool = False
|
||||
):
|
||||
@@ -102,37 +109,8 @@ class KimiAudioWhisperEncoder(WhisperEncoder):
|
||||
)
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
stacked_params_mapping = [
|
||||
# (param_name, shard_name, shard_id)
|
||||
("qkv_proj", "q_proj", "q"),
|
||||
("qkv_proj", "k_proj", "k"),
|
||||
("qkv_proj", "v_proj", "v"),
|
||||
]
|
||||
params_dict = dict(self.named_parameters())
|
||||
loaded_params: set[str] = set()
|
||||
for name, loaded_weight in weights:
|
||||
for param_name, weight_name, shard_id in stacked_params_mapping:
|
||||
if weight_name not in name:
|
||||
continue
|
||||
name = name.replace(weight_name, param_name)
|
||||
# Skip loading extra bias for GPTQ models.
|
||||
if name.endswith(".bias") and name not in params_dict:
|
||||
continue
|
||||
|
||||
param = params_dict[name]
|
||||
weight_loader = param.weight_loader
|
||||
weight_loader(param, loaded_weight, shard_id)
|
||||
break
|
||||
else:
|
||||
# Skip loading extra bias for GPTQ models.
|
||||
if name.endswith(".bias") and name not in params_dict:
|
||||
continue
|
||||
|
||||
param = params_dict[name]
|
||||
weight_loader = getattr(param, "weight_loader", default_weight_loader)
|
||||
weight_loader(param, loaded_weight)
|
||||
loaded_params.add(name)
|
||||
return loaded_params
|
||||
loader = AutoWeightsLoader(self)
|
||||
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
|
||||
|
||||
|
||||
# -----------------------------------------------------------------------------
|
||||
|
||||
@@ -32,7 +32,6 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
|
||||
ParallelLMHead,
|
||||
VocabParallelEmbedding,
|
||||
)
|
||||
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
|
||||
from vllm.sequence import IntermediateTensors
|
||||
|
||||
from .interfaces import HasInnerState, IsHybrid, SupportsLoRA, SupportsPP, SupportsQuant
|
||||
@@ -41,7 +40,6 @@ from .utils import (
|
||||
PPMissingLayer,
|
||||
WeightsMapper,
|
||||
extract_layer_index,
|
||||
is_pp_missing_parameter,
|
||||
make_empty_intermediate_tensors_factory,
|
||||
make_layers,
|
||||
maybe_prefix,
|
||||
@@ -297,6 +295,20 @@ class Lfm2ShortConvDecoderLayer(nn.Module):
|
||||
|
||||
@support_torch_compile
|
||||
class Lfm2Model(nn.Module):
|
||||
# HF uses .conv. but vLLM uses .short_conv. to avoid LoRA regex collision
|
||||
# with the inner .conv.conv child (ShortConv has a child self.conv, so
|
||||
# naming the container .conv too makes _match_target_modules match both).
|
||||
hf_to_vllm_mapper = WeightsMapper(
|
||||
orig_to_new_substr={".conv.": ".short_conv."},
|
||||
orig_to_new_stacked={
|
||||
".q_proj": (".qkv_proj", "q"),
|
||||
".k_proj": (".qkv_proj", "k"),
|
||||
".v_proj": (".qkv_proj", "v"),
|
||||
".w1": (".w13", 0),
|
||||
".w3": (".w13", 1),
|
||||
},
|
||||
)
|
||||
|
||||
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
|
||||
super().__init__()
|
||||
|
||||
@@ -375,40 +387,8 @@ class Lfm2Model(nn.Module):
|
||||
return hidden_states
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
stacked_params_mapping = [
|
||||
(".qkv_proj", ".q_proj", "q"),
|
||||
(".qkv_proj", ".k_proj", "k"),
|
||||
(".qkv_proj", ".v_proj", "v"),
|
||||
(".w13", ".w1", 0),
|
||||
(".w13", ".w3", 1),
|
||||
]
|
||||
params_dict = dict(self.named_parameters())
|
||||
loaded_params: set[str] = set()
|
||||
for name, loaded_weight in weights:
|
||||
if ".conv." in name:
|
||||
name = name.replace(".conv.", ".short_conv.", 1)
|
||||
|
||||
for param_name, weight_name, shard_id in stacked_params_mapping:
|
||||
# Use segment-boundary matching (trailing dot) to prevent
|
||||
# e.g. ".w1" from matching inside ".w13" in pre-fused keys.
|
||||
if weight_name + "." not in name:
|
||||
continue
|
||||
name = name.replace(weight_name + ".", param_name + ".")
|
||||
|
||||
if is_pp_missing_parameter(name, self):
|
||||
continue
|
||||
param = params_dict[name]
|
||||
weight_loader = param.weight_loader
|
||||
weight_loader(param, loaded_weight, shard_id)
|
||||
break
|
||||
else:
|
||||
if is_pp_missing_parameter(name, self):
|
||||
continue
|
||||
param = params_dict[name]
|
||||
weight_loader = getattr(param, "weight_loader", default_weight_loader)
|
||||
weight_loader(param, loaded_weight)
|
||||
loaded_params.add(name)
|
||||
return loaded_params
|
||||
loader = AutoWeightsLoader(self)
|
||||
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
|
||||
|
||||
|
||||
class Lfm2ForCausalLM(
|
||||
@@ -427,12 +407,8 @@ class Lfm2ForCausalLM(
|
||||
"in_proj": ["in_proj"],
|
||||
}
|
||||
|
||||
# HF uses .conv. but vLLM uses .short_conv. to avoid LoRA regex collision
|
||||
# with the inner .conv.conv child (ShortConv has a child self.conv, so
|
||||
# naming the container .conv too makes _match_target_modules match both)
|
||||
hf_to_vllm_mapper = WeightsMapper(
|
||||
orig_to_new_substr={".conv.": ".short_conv."},
|
||||
)
|
||||
# Reuse the backbone mapper so LoRA/quantization see the same name mapping.
|
||||
hf_to_vllm_mapper = Lfm2Model.hf_to_vllm_mapper
|
||||
|
||||
# LoRA specific attributes
|
||||
embedding_modules = {
|
||||
|
||||
@@ -23,9 +23,8 @@ from vllm.model_executor.layers.linear import (
|
||||
RowParallelLinear,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization import QuantizationConfig
|
||||
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
|
||||
|
||||
from .utils import maybe_prefix
|
||||
from .utils import AutoWeightsLoader, WeightsMapper, maybe_prefix
|
||||
from .vision import (
|
||||
is_vit_use_data_parallel,
|
||||
resolve_visual_encoder_outputs,
|
||||
@@ -458,6 +457,14 @@ class Siglip2VisionTransformer(nn.Module):
|
||||
|
||||
|
||||
class Siglip2Model(torch.nn.Module):
|
||||
hf_to_vllm_mapper = WeightsMapper(
|
||||
orig_to_new_stacked={
|
||||
".q_proj": (".qkv_proj", "q"),
|
||||
".k_proj": (".qkv_proj", "k"),
|
||||
".v_proj": (".qkv_proj", "v"),
|
||||
}
|
||||
)
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
config: Siglip2VisionConfig,
|
||||
@@ -501,42 +508,21 @@ class Siglip2Model(torch.nn.Module):
|
||||
)
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
stacked_params_mapping = [
|
||||
# (param_name, shard_name, shard_id)
|
||||
("qkv_proj", "q_proj", "q"),
|
||||
("qkv_proj", "k_proj", "k"),
|
||||
("qkv_proj", "v_proj", "v"),
|
||||
]
|
||||
params_dict = dict(self.named_parameters())
|
||||
loaded_params: set[str] = set()
|
||||
skip_prefixes = []
|
||||
if self.vision_model.post_layernorm is None:
|
||||
skip_prefixes.append("vision_model.post_layernorm.")
|
||||
loader = AutoWeightsLoader(self, skip_prefixes=skip_prefixes)
|
||||
|
||||
# Drop layers omitted by num_hidden_layers_override.
|
||||
layer_count = len(self.vision_model.encoder.layers)
|
||||
|
||||
for name, loaded_weight in weights:
|
||||
# post_layernorm is optional in Siglip2Model
|
||||
if (
|
||||
name.startswith("vision_model.post_layernorm")
|
||||
and self.vision_model.post_layernorm is None
|
||||
):
|
||||
continue
|
||||
|
||||
# omit layers when num_hidden_layers_override is set
|
||||
if name.startswith("vision_model.encoder.layers"):
|
||||
layer_idx = int(name.split(".")[3])
|
||||
if layer_idx >= layer_count:
|
||||
def _filter(ws):
|
||||
for n, w in ws:
|
||||
if (
|
||||
n.startswith("vision_model.encoder.layers.")
|
||||
and int(n.split(".")[3]) >= layer_count
|
||||
):
|
||||
continue
|
||||
yield n, w
|
||||
|
||||
for param_name, weight_name, shard_id in stacked_params_mapping:
|
||||
if weight_name not in name:
|
||||
continue
|
||||
name = name.replace(weight_name, param_name)
|
||||
|
||||
param = params_dict[name]
|
||||
weight_loader = param.weight_loader
|
||||
weight_loader(param, loaded_weight, shard_id)
|
||||
break
|
||||
else:
|
||||
param = params_dict[name]
|
||||
weight_loader = getattr(param, "weight_loader", default_weight_loader)
|
||||
weight_loader(param, loaded_weight)
|
||||
loaded_params.add(name)
|
||||
return loaded_params
|
||||
return loader.load_weights(_filter(weights), mapper=self.hf_to_vllm_mapper)
|
||||
|
||||
@@ -32,18 +32,33 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
|
||||
ParallelLMHead,
|
||||
VocabParallelEmbedding,
|
||||
)
|
||||
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
|
||||
from vllm.model_executor.models.llama4 import Llama4DecoderLayer, Llama4ForCausalLM
|
||||
from vllm.model_executor.models.utils import extract_layer_index
|
||||
|
||||
from .interfaces import SupportsMultiModal
|
||||
from .utils import AutoWeightsLoader, maybe_prefix, process_eagle_weight
|
||||
from .utils import (
|
||||
AutoWeightsLoader,
|
||||
WeightsMapper,
|
||||
maybe_prefix,
|
||||
process_eagle_weight,
|
||||
)
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
@support_torch_compile
|
||||
class LlamaModel(nn.Module):
|
||||
hf_to_vllm_mapper = WeightsMapper(
|
||||
orig_to_new_prefix={"model.": ""},
|
||||
orig_to_new_stacked={
|
||||
".q_proj": (".qkv_proj", "q"),
|
||||
".k_proj": (".qkv_proj", "k"),
|
||||
".v_proj": (".qkv_proj", "v"),
|
||||
".gate_proj": (".gate_up_proj", 0),
|
||||
".up_proj": (".gate_up_proj", 1),
|
||||
},
|
||||
)
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
@@ -108,34 +123,8 @@ class LlamaModel(nn.Module):
|
||||
return hidden_states, hidden_states
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
stacked_params_mapping = [
|
||||
# (param_name, shard_name, shard_id)
|
||||
(".qkv_proj", ".q_proj", "q"),
|
||||
(".qkv_proj", ".k_proj", "k"),
|
||||
(".qkv_proj", ".v_proj", "v"),
|
||||
(".gate_up_proj", ".gate_proj", 0),
|
||||
(".gate_up_proj", ".up_proj", 1),
|
||||
]
|
||||
params_dict = dict(self.named_parameters())
|
||||
loaded_params: set[str] = set()
|
||||
for name, loaded_weight in weights:
|
||||
name = name.removeprefix("model.")
|
||||
for param_name, weight_name, shard_id in stacked_params_mapping:
|
||||
if weight_name not in name:
|
||||
continue
|
||||
name = name.replace(weight_name, param_name)
|
||||
param = params_dict[name]
|
||||
weight_loader = param.weight_loader
|
||||
weight_loader(param, loaded_weight, shard_id)
|
||||
break
|
||||
else:
|
||||
param = params_dict[name]
|
||||
weight_loader = getattr(param, "weight_loader", default_weight_loader)
|
||||
weight_loader(param, loaded_weight)
|
||||
loaded_params.add(name)
|
||||
for name in params_dict:
|
||||
assert name in loaded_params, f"{name} is not loaded!"
|
||||
return loaded_params
|
||||
loader = AutoWeightsLoader(self)
|
||||
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
|
||||
|
||||
def validate_and_update_config(
|
||||
self, start_layer_id: int, quant_config: QuantizationConfig | None = None
|
||||
|
||||
@@ -14,14 +14,11 @@ from vllm.model_executor.layers.linear import ReplicatedLinear
|
||||
from vllm.model_executor.layers.logits_processor import LogitsProcessor
|
||||
from vllm.model_executor.layers.quantization.base_config import QuantizationConfig
|
||||
from vllm.model_executor.layers.vocab_parallel_embedding import VocabParallelEmbedding
|
||||
from vllm.model_executor.model_loader.weight_utils import (
|
||||
default_weight_loader,
|
||||
maybe_remap_kv_scale_name,
|
||||
)
|
||||
from vllm.model_executor.models.llama import LlamaDecoderLayer, LlamaForCausalLM
|
||||
|
||||
from .utils import (
|
||||
AutoWeightsLoader,
|
||||
WeightsMapper,
|
||||
get_draft_quant_config,
|
||||
maybe_prefix,
|
||||
process_eagle_weight,
|
||||
@@ -53,6 +50,17 @@ class LlamaDecoderLayer(LlamaDecoderLayer):
|
||||
|
||||
@support_torch_compile
|
||||
class LlamaModel(nn.Module):
|
||||
hf_to_vllm_mapper = WeightsMapper(
|
||||
orig_to_new_stacked={
|
||||
# weight_name: (param_name, shard_id)
|
||||
".q_proj": (".qkv_proj", "q"),
|
||||
".k_proj": (".qkv_proj", "k"),
|
||||
".v_proj": (".qkv_proj", "v"),
|
||||
".gate_proj": (".gate_up_proj", 0),
|
||||
".up_proj": (".gate_up_proj", 1),
|
||||
}
|
||||
)
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
@@ -116,36 +124,8 @@ class LlamaModel(nn.Module):
|
||||
return hidden_states, hidden_states
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
stacked_params_mapping = [
|
||||
# (param_name, shard_name, shard_id)
|
||||
(".qkv_proj", ".q_proj", "q"),
|
||||
(".qkv_proj", ".k_proj", "k"),
|
||||
(".qkv_proj", ".v_proj", "v"),
|
||||
(".gate_up_proj", ".gate_proj", 0),
|
||||
(".gate_up_proj", ".up_proj", 1),
|
||||
]
|
||||
params_dict = dict(self.named_parameters())
|
||||
loaded_params: set[str] = set()
|
||||
for name, loaded_weight in weights:
|
||||
# Remapping the name FP8 kv-scale or zero point.
|
||||
if "scale" in name or "zero_point" in name:
|
||||
name = maybe_remap_kv_scale_name(name, params_dict)
|
||||
if name is None:
|
||||
continue
|
||||
for param_name, weight_name, shard_id in stacked_params_mapping:
|
||||
if weight_name not in name:
|
||||
continue
|
||||
name = name.replace(weight_name, param_name)
|
||||
param = params_dict[name]
|
||||
weight_loader = param.weight_loader
|
||||
weight_loader(param, loaded_weight, shard_id)
|
||||
break
|
||||
else:
|
||||
param = params_dict[name]
|
||||
weight_loader = getattr(param, "weight_loader", default_weight_loader)
|
||||
weight_loader(param, loaded_weight)
|
||||
loaded_params.add(name)
|
||||
return loaded_params
|
||||
loader = AutoWeightsLoader(self)
|
||||
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
|
||||
|
||||
|
||||
class EagleLlamaForCausalLM(LlamaForCausalLM):
|
||||
|
||||
@@ -18,15 +18,12 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
|
||||
ParallelLMHead,
|
||||
VocabParallelEmbedding,
|
||||
)
|
||||
from vllm.model_executor.model_loader.weight_utils import (
|
||||
default_weight_loader,
|
||||
maybe_remap_kv_scale_name,
|
||||
)
|
||||
from vllm.model_executor.models.llama import LlamaDecoderLayer, LlamaForCausalLM
|
||||
from vllm.multimodal.inputs import NestedTensors
|
||||
|
||||
from .utils import (
|
||||
AutoWeightsLoader,
|
||||
WeightsMapper,
|
||||
get_draft_quant_config,
|
||||
maybe_prefix,
|
||||
process_eagle_weight,
|
||||
@@ -253,39 +250,20 @@ class LlamaModel(nn.Module):
|
||||
|
||||
return hidden_states, aux_output
|
||||
|
||||
hf_to_vllm_mapper = WeightsMapper(
|
||||
orig_to_new_substr={"midlayer.": "layers.0."},
|
||||
orig_to_new_stacked={
|
||||
".q_proj": (".qkv_proj", "q"),
|
||||
".k_proj": (".qkv_proj", "k"),
|
||||
".v_proj": (".qkv_proj", "v"),
|
||||
".gate_proj": (".gate_up_proj", 0),
|
||||
".up_proj": (".gate_up_proj", 1),
|
||||
},
|
||||
)
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
stacked_params_mapping = [
|
||||
# (param_name, shard_name, shard_id)
|
||||
(".qkv_proj", ".q_proj", "q"),
|
||||
(".qkv_proj", ".k_proj", "k"),
|
||||
(".qkv_proj", ".v_proj", "v"),
|
||||
(".gate_up_proj", ".gate_proj", 0),
|
||||
(".gate_up_proj", ".up_proj", 1),
|
||||
]
|
||||
params_dict = dict(self.named_parameters())
|
||||
loaded_params: set[str] = set()
|
||||
for name, loaded_weight in weights:
|
||||
if "midlayer." in name:
|
||||
name = name.replace("midlayer.", "layers.0.")
|
||||
# Remapping the name FP8 kv-scale or zero point.
|
||||
if "scale" in name or "zero_point" in name:
|
||||
name = maybe_remap_kv_scale_name(name, params_dict)
|
||||
if name is None:
|
||||
continue
|
||||
for param_name, weight_name, shard_id in stacked_params_mapping:
|
||||
if weight_name not in name:
|
||||
continue
|
||||
name = name.replace(weight_name, param_name)
|
||||
param = params_dict[name]
|
||||
weight_loader = param.weight_loader
|
||||
weight_loader(param, loaded_weight, shard_id)
|
||||
break
|
||||
else:
|
||||
param = params_dict[name]
|
||||
weight_loader = getattr(param, "weight_loader", default_weight_loader)
|
||||
weight_loader(param, loaded_weight)
|
||||
loaded_params.add(name)
|
||||
return loaded_params
|
||||
loader = AutoWeightsLoader(self)
|
||||
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
|
||||
|
||||
|
||||
class Eagle3LlamaForCausalLM(LlamaForCausalLM):
|
||||
|
||||
@@ -34,11 +34,10 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
|
||||
ParallelLMHead,
|
||||
VocabParallelEmbedding,
|
||||
)
|
||||
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
|
||||
from vllm.model_executor.models.qwen2 import Qwen2DecoderLayer
|
||||
from vllm.sequence import IntermediateTensors
|
||||
|
||||
from .utils import maybe_prefix
|
||||
from .utils import AutoWeightsLoader, WeightsMapper, maybe_prefix
|
||||
|
||||
|
||||
class MiMoMultiTokenPredictorLayer(nn.Module):
|
||||
@@ -163,6 +162,28 @@ class MiMoMTP(nn.Module):
|
||||
self.config.hidden_size,
|
||||
prefix=maybe_prefix(prefix, "lm_head"),
|
||||
)
|
||||
# Checkpoint stores MTP layers 0-indexed and without the `mtp_block`
|
||||
# wrapper around the transformer block; remap onto the offset index.
|
||||
start = self.config.num_hidden_layers
|
||||
self.hf_to_vllm_mapper = WeightsMapper(
|
||||
orig_to_new_substr={
|
||||
".self_attn.": ".mtp_block.self_attn.",
|
||||
".mlp.": ".mtp_block.mlp.",
|
||||
".input_layernorm.": ".mtp_block.input_layernorm.",
|
||||
".post_attention_layernorm.": ".mtp_block.post_attention_layernorm.",
|
||||
},
|
||||
orig_to_new_stacked={
|
||||
".q_proj": (".qkv_proj", "q"),
|
||||
".k_proj": (".qkv_proj", "k"),
|
||||
".v_proj": (".qkv_proj", "v"),
|
||||
".gate_proj": (".gate_up_proj", 0),
|
||||
".up_proj": (".gate_up_proj", 1),
|
||||
},
|
||||
orig_to_new_prefix={
|
||||
f"model.mtp_layers.{i}.": f"model.mtp_layers.{i + start}."
|
||||
for i in range(self.config.num_nextn_predict_layers)
|
||||
},
|
||||
)
|
||||
|
||||
def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
|
||||
return self.model.embed_input_ids(input_ids)
|
||||
@@ -190,105 +211,12 @@ class MiMoMTP(nn.Module):
|
||||
return self.model.compute_logits(hidden_states, self.lm_head, spec_step_idx)
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
stacked_params_mapping = [
|
||||
("qkv_proj", "q_proj", "q"),
|
||||
("qkv_proj", "k_proj", "k"),
|
||||
("qkv_proj", "v_proj", "v"),
|
||||
("gate_up_proj", "gate_proj", 0),
|
||||
("gate_up_proj", "up_proj", 1),
|
||||
]
|
||||
# The checkpoint carries the full model; keep only the MTP layers and
|
||||
# the shared embedding/head.
|
||||
def mtp_weights():
|
||||
for name, weight in weights:
|
||||
if "mtp_layers" in name or "embed_tokens" in name or "lm_head" in name:
|
||||
yield name, weight
|
||||
|
||||
params_dict = dict(self.named_parameters())
|
||||
loaded_params: set[str] = set()
|
||||
for name, loaded_weight in weights:
|
||||
if "rotary_emb.inv_freq" in name:
|
||||
continue
|
||||
name = self.map_model_name_to_mtp_param_name(name)
|
||||
|
||||
for param_name, weight_name, shard_id in stacked_params_mapping:
|
||||
# Skip non-stacked layers and experts (experts handled below).
|
||||
if weight_name not in name:
|
||||
continue
|
||||
if "mtp_layers" not in name:
|
||||
break
|
||||
# We have mlp.experts[0].gate_proj in the checkpoint.
|
||||
# Since we handle the experts below in expert_params_mapping,
|
||||
# we need to skip here BEFORE we update the name, otherwise
|
||||
# name will be updated to mlp.experts[0].gate_up_proj, which
|
||||
# will then be updated below in expert_params_mapping
|
||||
# for mlp.experts[0].gate_gate_up_proj, which breaks load.
|
||||
if ("mlp.experts." in name) and name not in params_dict:
|
||||
continue
|
||||
name = name.replace(weight_name, param_name)
|
||||
# Skip loading extra bias for GPTQ models.
|
||||
if name.endswith(".bias") and name not in params_dict:
|
||||
continue
|
||||
|
||||
param = params_dict[name]
|
||||
weight_loader = param.weight_loader
|
||||
weight_loader(param, loaded_weight, shard_id)
|
||||
break
|
||||
else:
|
||||
# Skip loading extra bias for GPTQ models.
|
||||
if name.endswith(".bias") and name not in params_dict:
|
||||
continue
|
||||
if "mtp_layers" not in name and (
|
||||
"embed_tokens" not in name and "lm_head" not in name
|
||||
):
|
||||
continue
|
||||
param = params_dict[name]
|
||||
weight_loader = getattr(param, "weight_loader", default_weight_loader)
|
||||
weight_loader(param, loaded_weight)
|
||||
loaded_params.add(name)
|
||||
return loaded_params
|
||||
|
||||
def map_model_name_to_mtp_param_name(self, name: str) -> str:
|
||||
import regex as re
|
||||
|
||||
# append mtp_start_layer_idx
|
||||
pattern = r"(model\.mtp_layers\.)(\d+)(\.)"
|
||||
match = re.match(pattern, name)
|
||||
if match:
|
||||
original_num = int(match.group(2))
|
||||
new_num = original_num + self.config.num_hidden_layers
|
||||
name = name.replace(match.group(), f"{match.group(1)}{new_num}.")
|
||||
# check for early turn
|
||||
name_without_prefix = [
|
||||
"token_layernorm",
|
||||
"hidden_layernorm",
|
||||
"input_proj",
|
||||
"final_layernorm",
|
||||
]
|
||||
for sub_name in name_without_prefix:
|
||||
if sub_name in name:
|
||||
return name
|
||||
# add mtp_block
|
||||
pattern = r"(model\.mtp_layers\.\d+\.)"
|
||||
match = re.match(pattern, name)
|
||||
if match:
|
||||
name = name.replace(match.group(), match.group() + "mtp_block.")
|
||||
return name
|
||||
|
||||
def _rewrite_spec_layer_name(self, spec_layer: int, name: str) -> str:
|
||||
"""
|
||||
Rewrite the weight name to match the format of the original model.
|
||||
Add .mtp_block for modules in transformer layer block for spec layer
|
||||
"""
|
||||
spec_layer_weight_names = [
|
||||
"embed_tokens",
|
||||
"enorm",
|
||||
"hnorm",
|
||||
"eh_proj",
|
||||
"shared_head",
|
||||
]
|
||||
spec_layer_weight = False
|
||||
for weight_name in spec_layer_weight_names:
|
||||
if weight_name in name:
|
||||
spec_layer_weight = True
|
||||
break
|
||||
if not spec_layer_weight:
|
||||
# treat rest weights as weights for transformer layer block
|
||||
name = name.replace(
|
||||
f"model.layers.{spec_layer}.", f"model.layers.{spec_layer}.mtp_block."
|
||||
)
|
||||
return name
|
||||
loader = AutoWeightsLoader(self)
|
||||
return loader.load_weights(mtp_weights(), mapper=self.hf_to_vllm_mapper)
|
||||
|
||||
@@ -29,7 +29,6 @@ from vllm.model_executor.layers.linear import (
|
||||
from vllm.model_executor.layers.quantization import QuantizationConfig
|
||||
from vllm.model_executor.layers.rotary_embedding import get_rope
|
||||
from vllm.model_executor.layers.rotary_embedding.common import ApplyRotaryEmb
|
||||
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
|
||||
from vllm.model_executor.models.vision import is_vit_use_data_parallel
|
||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
from vllm.multimodal.inputs import MultiModalFieldConfig, MultiModalKwargsItems
|
||||
@@ -378,6 +377,13 @@ class MiMoVisionBlock(nn.Module):
|
||||
|
||||
|
||||
class MiMoVisionTransformer(nn.Module):
|
||||
hf_to_vllm_mapper = WeightsMapper(
|
||||
orig_to_new_stacked={
|
||||
"mlp.gate_proj": ("mlp.gate_up_proj", 0),
|
||||
"mlp.up_proj": ("mlp.gate_up_proj", 1),
|
||||
}
|
||||
)
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
vision_cfg: PretrainedConfig,
|
||||
@@ -627,28 +633,8 @@ class MiMoVisionTransformer(nn.Module):
|
||||
return x
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
stacked_params_mapping = [
|
||||
("mlp.gate_up_proj", "mlp.gate_proj", 0),
|
||||
("mlp.gate_up_proj", "mlp.up_proj", 1),
|
||||
]
|
||||
params_dict = dict(self.named_parameters(remove_duplicate=False))
|
||||
loaded_params: set[str] = set()
|
||||
|
||||
for name, loaded_weight in weights:
|
||||
for param_name, weight_name, shard_id in stacked_params_mapping:
|
||||
if weight_name not in name:
|
||||
continue
|
||||
name = name.replace(weight_name, param_name)
|
||||
param = params_dict[name]
|
||||
weight_loader = param.weight_loader
|
||||
weight_loader(param, loaded_weight, shard_id)
|
||||
break
|
||||
else:
|
||||
param = params_dict[name]
|
||||
weight_loader = getattr(param, "weight_loader", default_weight_loader)
|
||||
weight_loader(param, loaded_weight)
|
||||
loaded_params.add(name)
|
||||
return loaded_params
|
||||
loader = AutoWeightsLoader(self)
|
||||
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
|
||||
|
||||
|
||||
class MiMoV2OmniProcessingInfo(BaseProcessingInfo):
|
||||
|
||||
@@ -25,7 +25,6 @@ from vllm.model_executor.layers.mamba.mamba_utils import (
|
||||
MambaStateShapeCalculator,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization import QuantizationConfig
|
||||
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
|
||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
from vllm.multimodal.inputs import (
|
||||
MultiModalFeatureSpec,
|
||||
@@ -619,6 +618,14 @@ class MiniCPMV4_6ProcessingInfo(MiniCPMVProcessingInfo):
|
||||
|
||||
|
||||
class MiniCPMV4_6ViTWindowAttentionSelfAttn(nn.Module):
|
||||
hf_to_vllm_mapper = WeightsMapper(
|
||||
orig_to_new_stacked={
|
||||
"q_proj": ("qkv_proj", "q"),
|
||||
"k_proj": ("qkv_proj", "k"),
|
||||
"v_proj": ("qkv_proj", "v"),
|
||||
}
|
||||
)
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
config,
|
||||
@@ -667,31 +674,8 @@ class MiniCPMV4_6ViTWindowAttentionSelfAttn(nn.Module):
|
||||
return out
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
stacked_params_mapping = [
|
||||
("qkv_proj", "q_proj", "q"),
|
||||
("qkv_proj", "k_proj", "k"),
|
||||
("qkv_proj", "v_proj", "v"),
|
||||
]
|
||||
params_dict = dict(self.named_parameters())
|
||||
loaded_params: set[str] = set()
|
||||
for name, loaded_weight in weights:
|
||||
for param_name, weight_name, shard_id in stacked_params_mapping:
|
||||
if weight_name not in name:
|
||||
continue
|
||||
mapped_name = name.replace(weight_name, param_name, 1)
|
||||
if mapped_name not in params_dict:
|
||||
continue
|
||||
param = params_dict[mapped_name]
|
||||
param.weight_loader(param, loaded_weight, shard_id)
|
||||
break
|
||||
else:
|
||||
if name not in params_dict:
|
||||
continue
|
||||
param = params_dict[name]
|
||||
weight_loader = getattr(param, "weight_loader", default_weight_loader)
|
||||
weight_loader(param, loaded_weight)
|
||||
loaded_params.add(name)
|
||||
return loaded_params
|
||||
loader = AutoWeightsLoader(self)
|
||||
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
|
||||
|
||||
|
||||
class MiniCPMV4_6ViTWindowAttentionMerger(nn.Module):
|
||||
|
||||
@@ -41,7 +41,6 @@ from vllm.distributed import (
|
||||
from vllm.model_executor.layers.attention import Attention
|
||||
from vllm.model_executor.layers.fused_moe import (
|
||||
FusedMoE,
|
||||
fused_moe_make_expert_params_mapping,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.router.gate_linear import GateLinear
|
||||
from vllm.model_executor.layers.layernorm import RMSNorm
|
||||
@@ -57,17 +56,13 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
|
||||
ParallelLMHead,
|
||||
VocabParallelEmbedding,
|
||||
)
|
||||
from vllm.model_executor.model_loader.weight_utils import (
|
||||
default_weight_loader,
|
||||
maybe_remap_kv_scale_name,
|
||||
)
|
||||
from vllm.sequence import IntermediateTensors
|
||||
|
||||
from .interfaces import EagleModelMixin, SupportsEagle3, SupportsLoRA, SupportsPP
|
||||
from .utils import (
|
||||
AutoWeightsLoader,
|
||||
PPMissingLayer,
|
||||
is_pp_missing_parameter,
|
||||
WeightsMapper,
|
||||
make_empty_intermediate_tensors_factory,
|
||||
make_layers,
|
||||
maybe_prefix,
|
||||
@@ -111,6 +106,7 @@ class MiniMaxM2MoE(nn.Module):
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.experts",
|
||||
router_logits_dtype=torch.float32,
|
||||
ckpt_names=("w1", "w2", "w3"),
|
||||
)
|
||||
|
||||
self.gate = GateLinear(
|
||||
@@ -327,6 +323,15 @@ class MiniMaxM2DecoderLayer(nn.Module):
|
||||
class MiniMaxM2Model(nn.Module, EagleModelMixin):
|
||||
fall_back_to_pt_during_load = False
|
||||
|
||||
hf_to_vllm_mapper = WeightsMapper(
|
||||
orig_to_new_stacked={
|
||||
# weight_name: (param_name, shard_id)
|
||||
".q_proj": (".qkv_proj", "q"),
|
||||
".k_proj": (".qkv_proj", "k"),
|
||||
".v_proj": (".qkv_proj", "v"),
|
||||
}
|
||||
)
|
||||
|
||||
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
|
||||
super().__init__()
|
||||
|
||||
@@ -409,115 +414,23 @@ class MiniMaxM2Model(nn.Module, EagleModelMixin):
|
||||
|
||||
return hidden_states
|
||||
|
||||
def get_expert_mapping(self) -> list[tuple[str, str, int, str]]:
|
||||
return fused_moe_make_expert_params_mapping(
|
||||
self,
|
||||
ckpt_gate_proj_name="w1",
|
||||
ckpt_down_proj_name="w2",
|
||||
ckpt_up_proj_name="w3",
|
||||
num_experts=self.config.num_local_experts,
|
||||
)
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
stacked_params_mapping = [
|
||||
# (param_name, shard_name, shard_id)
|
||||
("qkv_proj", "q_proj", "q"),
|
||||
("qkv_proj", "k_proj", "k"),
|
||||
("qkv_proj", "v_proj", "v"),
|
||||
]
|
||||
|
||||
# Params for weights, fp8 weight scales, fp8 activation scales
|
||||
# (param_name, weight_name, expert_id, shard_id)
|
||||
expert_params_mapping = self.get_expert_mapping()
|
||||
|
||||
params_dict = dict(self.named_parameters())
|
||||
loaded_params: set[str] = set()
|
||||
for name, loaded_weight in weights:
|
||||
if "rotary_emb.inv_freq" in name:
|
||||
continue
|
||||
|
||||
spec_layer = get_spec_layer_idx_from_weight_name(self.config, name)
|
||||
if spec_layer is not None:
|
||||
continue # skip spec decode layers for main model
|
||||
|
||||
for param_name, weight_name, shard_id in stacked_params_mapping:
|
||||
# Skip non-stacked layers and experts (experts handled below).
|
||||
if weight_name not in name:
|
||||
continue
|
||||
# We have mlp.experts[0].gate_proj in the checkpoint.
|
||||
# Since we handle the experts below in expert_params_mapping,
|
||||
# we need to skip here BEFORE we update the name, otherwise
|
||||
# name will be updated to mlp.experts[0].gate_up_proj, which
|
||||
# will then be updated below in expert_params_mapping
|
||||
# for mlp.experts[0].gate_gate_up_proj, which breaks load.
|
||||
if ("mlp.experts." in name) and name not in params_dict:
|
||||
continue
|
||||
name = name.replace(weight_name, param_name)
|
||||
# Skip loading extra bias for GPTQ models.
|
||||
if name.endswith(".bias") and name not in params_dict:
|
||||
continue
|
||||
|
||||
if is_pp_missing_parameter(name, self):
|
||||
continue
|
||||
|
||||
# Remap qkv_proj.[kv]_scale to attn.[kv]_scale
|
||||
if name.endswith((".k_scale", ".v_scale")):
|
||||
remapped_name = maybe_remap_kv_scale_name(name, params_dict)
|
||||
if remapped_name is not None and remapped_name in params_dict:
|
||||
param = params_dict[remapped_name]
|
||||
weight_loader = getattr(
|
||||
param, "weight_loader", default_weight_loader
|
||||
)
|
||||
weight_loader(param, loaded_weight)
|
||||
break
|
||||
|
||||
param = params_dict[name]
|
||||
weight_loader = param.weight_loader
|
||||
weight_loader(param, loaded_weight, shard_id)
|
||||
break
|
||||
else:
|
||||
for mapping in expert_params_mapping:
|
||||
param_name, weight_name, expert_id, shard_id = mapping
|
||||
if weight_name not in name:
|
||||
continue
|
||||
name = name.replace(weight_name, param_name)
|
||||
|
||||
if is_pp_missing_parameter(name, self):
|
||||
continue
|
||||
|
||||
param = params_dict[name]
|
||||
weight_loader = param.weight_loader
|
||||
weight_loader(
|
||||
param,
|
||||
loaded_weight,
|
||||
name,
|
||||
shard_id=shard_id,
|
||||
expert_id=expert_id,
|
||||
)
|
||||
break
|
||||
else:
|
||||
# Skip loading extra bias for GPTQ models.
|
||||
if name.endswith(".bias") and name not in params_dict:
|
||||
continue
|
||||
|
||||
# Remapping the name of FP8 kv-scale.
|
||||
name = maybe_remap_kv_scale_name(name, params_dict)
|
||||
if name is None:
|
||||
continue
|
||||
|
||||
if is_pp_missing_parameter(name, self):
|
||||
continue
|
||||
|
||||
param = params_dict[name]
|
||||
weight_loader = getattr(
|
||||
param, "weight_loader", default_weight_loader
|
||||
)
|
||||
weight_loader(param, loaded_weight)
|
||||
loaded_params.add(name)
|
||||
return loaded_params
|
||||
# Skip spec-decode (MTP) layers; they are appended after the main
|
||||
# decoder layers and have no destination in the main model.
|
||||
skip_prefixes = None
|
||||
num_mtp = getattr(self.config, "num_mtp_modules", 0)
|
||||
if num_mtp:
|
||||
base = self.config.num_hidden_layers
|
||||
skip_prefixes = [f"layers.{base + i}." for i in range(num_mtp)]
|
||||
loader = AutoWeightsLoader(
|
||||
self,
|
||||
skip_prefixes=skip_prefixes,
|
||||
)
|
||||
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
|
||||
|
||||
|
||||
class MiniMaxM2ForCausalLM(nn.Module, SupportsLoRA, SupportsPP, SupportsEagle3):
|
||||
hf_to_vllm_mapper = MiniMaxM2Model.hf_to_vllm_mapper
|
||||
packed_modules_mapping = {
|
||||
"qkv_proj": [
|
||||
"q_proj",
|
||||
@@ -577,17 +490,3 @@ class MiniMaxM2ForCausalLM(nn.Module, SupportsLoRA, SupportsPP, SupportsEagle3):
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
loader = AutoWeightsLoader(self)
|
||||
return loader.load_weights(weights)
|
||||
|
||||
def get_expert_mapping(self) -> list[tuple[str, str, int, str]]:
|
||||
return self.model.get_expert_mapping()
|
||||
|
||||
|
||||
def get_spec_layer_idx_from_weight_name(
|
||||
config: PretrainedConfig, weight_name: str
|
||||
) -> int | None:
|
||||
if hasattr(config, "num_mtp_modules") and (config.num_mtp_modules > 0):
|
||||
layer_idx = config.num_hidden_layers
|
||||
for i in range(config.num_mtp_modules):
|
||||
if weight_name.startswith(f"model.layers.{layer_idx + i}."):
|
||||
return layer_idx + i
|
||||
return None
|
||||
|
||||
@@ -24,8 +24,7 @@
|
||||
# limitations under the License.
|
||||
"""Inference-only Mixtral model."""
|
||||
|
||||
import typing
|
||||
from collections.abc import Callable, Iterable
|
||||
from collections.abc import Iterable
|
||||
from itertools import islice
|
||||
|
||||
import torch
|
||||
@@ -42,7 +41,6 @@ from vllm.distributed import (
|
||||
from vllm.model_executor.layers.attention import Attention
|
||||
from vllm.model_executor.layers.fused_moe import (
|
||||
FusedMoE,
|
||||
fused_moe_make_expert_params_mapping,
|
||||
)
|
||||
from vllm.model_executor.layers.layernorm import RMSNorm
|
||||
from vllm.model_executor.layers.linear import (
|
||||
@@ -57,17 +55,13 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
|
||||
ParallelLMHead,
|
||||
VocabParallelEmbedding,
|
||||
)
|
||||
from vllm.model_executor.model_loader.weight_utils import (
|
||||
default_weight_loader,
|
||||
maybe_remap_kv_scale_name,
|
||||
)
|
||||
from vllm.sequence import IntermediateTensors
|
||||
|
||||
from .interfaces import MixtureOfExperts, SupportsLoRA, SupportsPP
|
||||
from .utils import (
|
||||
AutoWeightsLoader,
|
||||
PPMissingLayer,
|
||||
is_pp_missing_parameter,
|
||||
WeightsMapper,
|
||||
make_empty_intermediate_tensors_factory,
|
||||
make_layers,
|
||||
maybe_prefix,
|
||||
@@ -142,6 +136,7 @@ class MixtralMoE(nn.Module):
|
||||
prefix=f"{prefix}.experts",
|
||||
enable_eplb=self.enable_eplb,
|
||||
num_redundant_experts=self.n_redundant_experts,
|
||||
ckpt_names=("w1", "w2", "w3"),
|
||||
)
|
||||
|
||||
def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
|
||||
@@ -295,6 +290,15 @@ class MixtralDecoderLayer(nn.Module):
|
||||
|
||||
@support_torch_compile
|
||||
class MixtralModel(nn.Module):
|
||||
hf_to_vllm_mapper = WeightsMapper(
|
||||
orig_to_new_stacked={
|
||||
# weight_name: (param_name, shard_id)
|
||||
".q_proj": (".qkv_proj", "q"),
|
||||
".k_proj": (".qkv_proj", "k"),
|
||||
".v_proj": (".qkv_proj", "v"),
|
||||
}
|
||||
)
|
||||
|
||||
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
|
||||
super().__init__()
|
||||
|
||||
@@ -364,114 +368,15 @@ class MixtralModel(nn.Module):
|
||||
hidden_states, _ = self.norm(hidden_states, residual)
|
||||
return hidden_states
|
||||
|
||||
def get_expert_mapping(self) -> list[tuple[str, str, int, str]]:
|
||||
# Params for weights, fp8 weight scales, fp8 activation scales
|
||||
# (param_name, weight_name, expert_id, shard_id)
|
||||
return fused_moe_make_expert_params_mapping(
|
||||
self,
|
||||
ckpt_gate_proj_name="w1",
|
||||
ckpt_down_proj_name="w2",
|
||||
ckpt_up_proj_name="w3",
|
||||
num_experts=self.config.num_local_experts,
|
||||
num_redundant_experts=self.num_redundant_experts,
|
||||
)
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
stacked_params_mapping = [
|
||||
# (param_name, shard_name, shard_id)
|
||||
("qkv_proj", "q_proj", "q"),
|
||||
("qkv_proj", "k_proj", "k"),
|
||||
("qkv_proj", "v_proj", "v"),
|
||||
]
|
||||
|
||||
params_dict = dict(self.named_parameters())
|
||||
loaded_params: set[str] = set()
|
||||
expert_params_mapping = self.get_expert_mapping()
|
||||
for name, loaded_weight in weights:
|
||||
for param_name, weight_name, shard_id in stacked_params_mapping:
|
||||
if weight_name not in name:
|
||||
continue
|
||||
name = name.replace(weight_name, param_name)
|
||||
# Skip loading extra bias for GPTQ models.
|
||||
if (
|
||||
name.endswith(".bias") or name.endswith("_bias")
|
||||
) and name not in params_dict:
|
||||
continue
|
||||
# Skip layers on other devices.
|
||||
if is_pp_missing_parameter(name, self):
|
||||
continue
|
||||
if name.endswith("scale"):
|
||||
# Remapping the name of FP8 kv-scale.
|
||||
name = maybe_remap_kv_scale_name(name, params_dict)
|
||||
if name is None:
|
||||
continue
|
||||
param = params_dict[name]
|
||||
weight_loader = param.weight_loader
|
||||
weight_loader(param, loaded_weight, shard_id)
|
||||
break
|
||||
else:
|
||||
is_expert_weight = False
|
||||
for mapping in expert_params_mapping:
|
||||
param_name, weight_name, expert_id, shard_id = mapping
|
||||
|
||||
if weight_name not in name:
|
||||
continue
|
||||
|
||||
is_expert_weight = True
|
||||
name_mapped = name.replace(weight_name, param_name)
|
||||
|
||||
# Skip layers on other devices.
|
||||
if is_pp_missing_parameter(name_mapped, self):
|
||||
continue
|
||||
|
||||
if (
|
||||
name_mapped.endswith(".bias") or name_mapped.endswith("_bias")
|
||||
) and name_mapped not in params_dict:
|
||||
continue
|
||||
|
||||
param = params_dict[name_mapped]
|
||||
weight_loader = typing.cast(
|
||||
Callable[..., bool], param.weight_loader
|
||||
)
|
||||
success = weight_loader(
|
||||
param,
|
||||
loaded_weight,
|
||||
name_mapped,
|
||||
shard_id=shard_id,
|
||||
expert_id=expert_id,
|
||||
return_success=True,
|
||||
)
|
||||
if success:
|
||||
name = name_mapped
|
||||
break
|
||||
else:
|
||||
if is_expert_weight:
|
||||
continue
|
||||
# Skip loading extra bias for GPTQ models.
|
||||
if (
|
||||
name.endswith(".bias") or name.endswith("_bias")
|
||||
) and name not in params_dict:
|
||||
continue
|
||||
# Skip layers on other devices.
|
||||
if is_pp_missing_parameter(name, self):
|
||||
continue
|
||||
# Remapping the name of FP8 kv-scale.
|
||||
name = maybe_remap_kv_scale_name(name, params_dict)
|
||||
if name is None:
|
||||
continue
|
||||
|
||||
param = params_dict[name]
|
||||
weight_loader = getattr(
|
||||
param, "weight_loader", default_weight_loader
|
||||
)
|
||||
weight_loader(param, loaded_weight)
|
||||
loaded_params.add(name)
|
||||
return loaded_params
|
||||
loader = AutoWeightsLoader(self)
|
||||
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
|
||||
|
||||
|
||||
class MixtralForCausalLM(nn.Module, SupportsLoRA, SupportsPP, MixtureOfExperts):
|
||||
fall_back_to_pt_during_load = False
|
||||
|
||||
hf_to_vllm_mapper = MixtralModel.hf_to_vllm_mapper
|
||||
packed_modules_mapping = {
|
||||
"qkv_proj": [
|
||||
"q_proj",
|
||||
@@ -582,6 +487,3 @@ class MixtralForCausalLM(nn.Module, SupportsLoRA, SupportsPP, MixtureOfExperts):
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
loader = AutoWeightsLoader(self)
|
||||
return loader.load_weights(weights)
|
||||
|
||||
def get_expert_mapping(self) -> list[tuple[str, str, int, str]]:
|
||||
return self.model.get_expert_mapping()
|
||||
|
||||
@@ -672,6 +672,14 @@ class MolmoDecoderNormAfterLayer(MolmoDecoderLayer):
|
||||
class MolmoVisionBackbone(nn.Module, SupportsQuant):
|
||||
packed_modules_mapping = {"merged_linear": ["gate_proj", "up_proj"]}
|
||||
|
||||
hf_to_vllm_mapper = WeightsMapper(
|
||||
orig_to_new_stacked={
|
||||
# image_projector gate_up merge
|
||||
"gate_proj": ("merged_linear", 0),
|
||||
"up_proj": ("merged_linear", 1),
|
||||
}
|
||||
)
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
config: PretrainedConfig,
|
||||
@@ -800,38 +808,8 @@ class MolmoVisionBackbone(nn.Module, SupportsQuant):
|
||||
return image_features
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
stacked_params_mapping = [
|
||||
# (param_name, shard_name, shard_id)
|
||||
("merged_linear", "gate_proj", 0),
|
||||
("merged_linear", "up_proj", 1),
|
||||
]
|
||||
params_dict = dict(self.named_parameters())
|
||||
loaded_params: set[str] = set()
|
||||
|
||||
for name, loaded_weight in weights:
|
||||
for param_name, weight_name, shard_id in stacked_params_mapping:
|
||||
if weight_name not in name:
|
||||
continue
|
||||
name = name.replace(weight_name, param_name)
|
||||
# Skip loading extra bias for GPTQ models.
|
||||
if name.endswith(".bias") and name not in params_dict:
|
||||
continue
|
||||
if is_pp_missing_parameter(name, self):
|
||||
continue
|
||||
param = params_dict[name]
|
||||
weight_loader = param.weight_loader
|
||||
weight_loader(param, loaded_weight, shard_id)
|
||||
break
|
||||
else:
|
||||
if name.endswith(".bias") and name not in params_dict:
|
||||
continue
|
||||
if is_pp_missing_parameter(name, self):
|
||||
continue
|
||||
param = params_dict[name]
|
||||
weight_loader = getattr(param, "weight_loader", default_weight_loader)
|
||||
weight_loader(param, loaded_weight)
|
||||
loaded_params.add(name)
|
||||
return loaded_params
|
||||
loader = AutoWeightsLoader(self)
|
||||
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
|
||||
|
||||
|
||||
@support_torch_compile
|
||||
|
||||
@@ -709,6 +709,20 @@ class Molmo2VisionBackbone(nn.Module, SupportsQuant):
|
||||
"merged_linear": ["gate_proj", "up_proj"],
|
||||
}
|
||||
|
||||
# Runs after the top-level mapper, so image_pooling_2d/image_projector
|
||||
# source names are already renamed to q/k/v_proj and gate/up_proj.
|
||||
hf_to_vllm_mapper = WeightsMapper(
|
||||
orig_to_new_stacked={
|
||||
"wq": ("merged_qkv", "q"),
|
||||
"wk": ("merged_qkv", "k"),
|
||||
"wv": ("merged_qkv", "v"),
|
||||
"k_proj": ("merged_kv", 0),
|
||||
"v_proj": ("merged_kv", 1),
|
||||
"gate_proj": ("merged_linear", 0),
|
||||
"up_proj": ("merged_linear", 1),
|
||||
},
|
||||
)
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
vit_config: VitConfig,
|
||||
@@ -839,43 +853,8 @@ class Molmo2VisionBackbone(nn.Module, SupportsQuant):
|
||||
]
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
stacked_params_mapping = [
|
||||
# (param_name, shard_name, shard_id)
|
||||
("merged_qkv", "wq", "q"),
|
||||
("merged_qkv", "wk", "k"),
|
||||
("merged_qkv", "wv", "v"),
|
||||
("merged_kv", "k_proj", 0),
|
||||
("merged_kv", "v_proj", 1),
|
||||
("merged_linear", "gate_proj", 0),
|
||||
("merged_linear", "up_proj", 1),
|
||||
]
|
||||
params_dict = dict(self.named_parameters())
|
||||
loaded_params: set[str] = set()
|
||||
|
||||
for name, loaded_weight in weights:
|
||||
for param_name, weight_name, shard_id in stacked_params_mapping:
|
||||
if weight_name not in name:
|
||||
continue
|
||||
name = name.replace(weight_name, param_name)
|
||||
# Skip loading extra bias for GPTQ models.
|
||||
if name.endswith(".bias") and name not in params_dict:
|
||||
continue
|
||||
if is_pp_missing_parameter(name, self):
|
||||
continue
|
||||
param = params_dict[name]
|
||||
weight_loader = param.weight_loader
|
||||
weight_loader(param, loaded_weight, shard_id)
|
||||
break
|
||||
else:
|
||||
if name.endswith(".bias") and name not in params_dict:
|
||||
continue
|
||||
if is_pp_missing_parameter(name, self):
|
||||
continue
|
||||
param = params_dict[name]
|
||||
weight_loader = getattr(param, "weight_loader", default_weight_loader)
|
||||
weight_loader(param, loaded_weight)
|
||||
loaded_params.add(name)
|
||||
return loaded_params
|
||||
loader = AutoWeightsLoader(self)
|
||||
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
|
||||
|
||||
|
||||
class Molmo2Attention(nn.Module):
|
||||
|
||||
@@ -33,7 +33,6 @@ from vllm.model_executor.layers.linear import (
|
||||
from vllm.model_executor.layers.logits_processor import LogitsProcessor
|
||||
from vllm.model_executor.layers.quantization import QuantizationConfig
|
||||
from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
|
||||
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
|
||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
from vllm.multimodal.inputs import (
|
||||
AudioItem,
|
||||
@@ -742,37 +741,6 @@ class GatedMLP(nn.Module):
|
||||
x, _ = self.down_proj(x)
|
||||
return x
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
stacked_params_mapping = [
|
||||
("gate_up_proj", "gate_proj", 0),
|
||||
("gate_up_proj", "up_proj", 1),
|
||||
]
|
||||
params_dict = dict(self.named_parameters())
|
||||
loaded_params: set[str] = set()
|
||||
|
||||
for name, loaded_weight in weights:
|
||||
target_name = name
|
||||
for param_name, weight_name, shard_id in stacked_params_mapping:
|
||||
components = target_name.split(".")
|
||||
if weight_name not in components:
|
||||
continue
|
||||
|
||||
target_name = ".".join(
|
||||
param_name if component == weight_name else component
|
||||
for component in components
|
||||
)
|
||||
param = params_dict[target_name]
|
||||
weight_loader = param.weight_loader
|
||||
weight_loader(param, loaded_weight, shard_id)
|
||||
break
|
||||
else:
|
||||
param = params_dict[target_name]
|
||||
weight_loader = getattr(param, "weight_loader", default_weight_loader)
|
||||
weight_loader(param, loaded_weight)
|
||||
|
||||
loaded_params.add(target_name)
|
||||
return loaded_params
|
||||
|
||||
|
||||
@support_torch_compile(
|
||||
dynamic_arg_dims={
|
||||
@@ -1478,7 +1446,11 @@ class MossAudioModel(nn.Module, SupportsMultiModal, SupportsPP, SupportsLoRA):
|
||||
"language_model.embed_tokens.": "language_model.model.embed_tokens.",
|
||||
"language_model.layers.": "language_model.model.layers.",
|
||||
"language_model.norm.": "language_model.model.norm.",
|
||||
}
|
||||
},
|
||||
orig_to_new_stacked={
|
||||
".gate_proj": (".gate_up_proj", 0),
|
||||
".up_proj": (".gate_up_proj", 1),
|
||||
},
|
||||
)
|
||||
|
||||
def get_mm_mapping(self) -> MultiModelKeys:
|
||||
|
||||
@@ -34,7 +34,6 @@ from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.attention import Attention
|
||||
from vllm.model_executor.layers.fused_moe import (
|
||||
FusedMoE,
|
||||
fused_moe_make_expert_params_mapping,
|
||||
)
|
||||
from vllm.model_executor.layers.layernorm import RMSNorm
|
||||
from vllm.model_executor.layers.linear import (
|
||||
@@ -49,13 +48,12 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
|
||||
ParallelLMHead,
|
||||
VocabParallelEmbedding,
|
||||
)
|
||||
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
|
||||
from vllm.sequence import IntermediateTensors
|
||||
|
||||
from .interfaces import SupportsLoRA, SupportsPP
|
||||
from .utils import (
|
||||
AutoWeightsLoader,
|
||||
is_pp_missing_parameter,
|
||||
WeightsMapper,
|
||||
make_empty_intermediate_tensors_factory,
|
||||
make_layers,
|
||||
maybe_prefix,
|
||||
@@ -338,106 +336,16 @@ class OlmoeModel(nn.Module):
|
||||
hidden_states = self.norm(hidden_states)
|
||||
return hidden_states
|
||||
|
||||
def get_expert_mapping(self) -> list[tuple[str, str, int, str]]:
|
||||
# Params for weights, fp8 weight scales, fp8 activation scales
|
||||
# (param_name, weight_name, expert_id, shard_id)
|
||||
return fused_moe_make_expert_params_mapping(
|
||||
self,
|
||||
ckpt_gate_proj_name="gate_proj",
|
||||
ckpt_down_proj_name="down_proj",
|
||||
ckpt_up_proj_name="up_proj",
|
||||
num_experts=self.config.num_experts,
|
||||
)
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
stacked_params_mapping = [
|
||||
# (param_name, shard_name, shard_id)
|
||||
("qkv_proj", "q_proj", "q"),
|
||||
("qkv_proj", "k_proj", "k"),
|
||||
("qkv_proj", "v_proj", "v"),
|
||||
]
|
||||
|
||||
params_dict = dict(self.named_parameters())
|
||||
loaded_params: set[str] = set()
|
||||
expert_params_mapping = self.get_expert_mapping()
|
||||
for name, loaded_weight in weights:
|
||||
for param_name, weight_name, shard_id in stacked_params_mapping:
|
||||
# Skip non-stacked layers and experts (experts handled below).
|
||||
if weight_name not in name:
|
||||
continue
|
||||
# We have mlp.experts[0].gate_proj in the checkpoint.
|
||||
# Since we handle the experts below in expert_params_mapping,
|
||||
# we need to skip here BEFORE we update the name, otherwise
|
||||
# name will be updated to mlp.experts[0].gate_up_proj, which
|
||||
# will then be updated below in expert_params_mapping
|
||||
# for mlp.experts[0].gate_gate_up_proj, which breaks load.
|
||||
if "mlp.experts" in name:
|
||||
continue
|
||||
name = name.replace(weight_name, param_name)
|
||||
# Skip loading extra bias for GPTQ models.
|
||||
if name.endswith(".bias") and name not in params_dict:
|
||||
continue
|
||||
# Skip layers on other devices.
|
||||
if is_pp_missing_parameter(name, self):
|
||||
continue
|
||||
if name not in params_dict:
|
||||
continue
|
||||
|
||||
param = params_dict[name]
|
||||
weight_loader = param.weight_loader
|
||||
weight_loader(param, loaded_weight, shard_id)
|
||||
break
|
||||
else:
|
||||
for mapping in expert_params_mapping:
|
||||
param_name, weight_name, expert_id, shard_id = mapping
|
||||
if weight_name not in name:
|
||||
continue
|
||||
name = name.replace(weight_name, param_name)
|
||||
# Skip layers on other devices.
|
||||
if is_pp_missing_parameter(name, self):
|
||||
continue
|
||||
param = params_dict[name]
|
||||
weight_loader = param.weight_loader
|
||||
weight_loader(
|
||||
param,
|
||||
loaded_weight,
|
||||
name,
|
||||
shard_id=shard_id,
|
||||
expert_id=expert_id,
|
||||
)
|
||||
break
|
||||
else:
|
||||
# Skip loading extra bias for GPTQ models.
|
||||
if name.endswith(".bias") and name not in params_dict:
|
||||
continue
|
||||
# Skip layers on other devices.
|
||||
if is_pp_missing_parameter(name, self):
|
||||
continue
|
||||
# Remapping the name of FP8 kv-scale.
|
||||
if name.endswith("kv_scale"):
|
||||
remapped_kv_scale_name = name.replace(
|
||||
".kv_scale", ".attn.kv_scale"
|
||||
)
|
||||
if remapped_kv_scale_name not in params_dict:
|
||||
logger.warning_once(
|
||||
"Found kv scale in the checkpoint (e.g. %s), but not found the expected name in the model (e.g. %s). kv-scale is not loaded.", # noqa: E501
|
||||
name,
|
||||
remapped_kv_scale_name,
|
||||
)
|
||||
continue
|
||||
else:
|
||||
name = remapped_kv_scale_name
|
||||
|
||||
param = params_dict[name]
|
||||
weight_loader = getattr(
|
||||
param, "weight_loader", default_weight_loader
|
||||
)
|
||||
weight_loader(param, loaded_weight)
|
||||
loaded_params.add(name)
|
||||
return loaded_params
|
||||
|
||||
|
||||
class OlmoeForCausalLM(nn.Module, SupportsPP, SupportsLoRA):
|
||||
hf_to_vllm_mapper = WeightsMapper(
|
||||
orig_to_new_stacked={
|
||||
# weight_name: (param_name, shard_id)
|
||||
".q_proj": (".qkv_proj", "q"),
|
||||
".k_proj": (".qkv_proj", "k"),
|
||||
".v_proj": (".qkv_proj", "v"),
|
||||
}
|
||||
)
|
||||
packed_modules_mapping = {
|
||||
"qkv_proj": [
|
||||
"q_proj",
|
||||
@@ -496,7 +404,4 @@ class OlmoeForCausalLM(nn.Module, SupportsPP, SupportsLoRA):
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
loader = AutoWeightsLoader(self)
|
||||
return loader.load_weights(weights)
|
||||
|
||||
def get_expert_mapping(self) -> list[tuple[str, str, int, str]]:
|
||||
return self.model.get_expert_mapping()
|
||||
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
|
||||
|
||||
@@ -48,10 +48,6 @@ from vllm.model_executor.layers.quantization import QuantizationConfig
|
||||
from vllm.model_executor.layers.rotary_embedding.common import (
|
||||
ApplyRotaryEmb,
|
||||
)
|
||||
from vllm.model_executor.model_loader.weight_utils import (
|
||||
default_weight_loader,
|
||||
maybe_remap_kv_scale_name,
|
||||
)
|
||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
from vllm.multimodal.inputs import (
|
||||
MultiModalFeatureSpec,
|
||||
@@ -81,7 +77,6 @@ from .utils import (
|
||||
AutoWeightsLoader,
|
||||
PPMissingLayer,
|
||||
WeightsMapper,
|
||||
is_pp_missing_parameter,
|
||||
maybe_prefix,
|
||||
)
|
||||
from .vision import get_vit_attn_backend
|
||||
@@ -884,6 +879,14 @@ class SiglipVisionTransformer(nn.Module):
|
||||
|
||||
|
||||
class SiglipVisionModel(nn.Module):
|
||||
hf_to_vllm_mapper = WeightsMapper(
|
||||
orig_to_new_stacked={
|
||||
".q_proj": (".qkv_proj", "q"),
|
||||
".k_proj": (".qkv_proj", "k"),
|
||||
".v_proj": (".qkv_proj", "v"),
|
||||
}
|
||||
)
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
config,
|
||||
@@ -928,55 +931,19 @@ class SiglipVisionModel(nn.Module):
|
||||
)
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
stacked_params_mapping = [
|
||||
("qkv_proj", "q_proj", "q"),
|
||||
("qkv_proj", "k_proj", "k"),
|
||||
("qkv_proj", "v_proj", "v"),
|
||||
]
|
||||
params_dict = dict(self.named_parameters(remove_duplicate=False))
|
||||
loaded_params: set[str] = set()
|
||||
for name, loaded_weight in weights:
|
||||
if "rotary_emb.inv_freq" in name:
|
||||
continue
|
||||
if "head.attention" in name or "head.layernorm" in name:
|
||||
continue
|
||||
if "head.mlp" in name or "head.probe" in name:
|
||||
continue
|
||||
if "packing_position_embedding" in name:
|
||||
continue
|
||||
for (
|
||||
param_name,
|
||||
weight_name,
|
||||
shard_id,
|
||||
) in stacked_params_mapping:
|
||||
if weight_name not in name:
|
||||
continue
|
||||
name = name.replace(weight_name, param_name)
|
||||
if name.endswith(".bias") and name not in params_dict:
|
||||
continue
|
||||
if is_pp_missing_parameter(name, self):
|
||||
continue
|
||||
param = params_dict[name]
|
||||
weight_loader = param.weight_loader
|
||||
weight_loader(param, loaded_weight, shard_id)
|
||||
break
|
||||
else:
|
||||
if name.endswith(".bias") and name not in params_dict:
|
||||
continue
|
||||
name = maybe_remap_kv_scale_name(name, params_dict)
|
||||
if name is None:
|
||||
continue
|
||||
if is_pp_missing_parameter(name, self):
|
||||
continue
|
||||
param = params_dict[name]
|
||||
weight_loader = getattr(
|
||||
param,
|
||||
"weight_loader",
|
||||
default_weight_loader,
|
||||
)
|
||||
weight_loader(param, loaded_weight)
|
||||
loaded_params.add(name)
|
||||
return loaded_params
|
||||
# Skip the SigLIP attention pooling head and packing pos embedding
|
||||
# present in the checkpoint but absent from this vision tower.
|
||||
loader = AutoWeightsLoader(
|
||||
self,
|
||||
skip_substrs=[
|
||||
"head.attention",
|
||||
"head.layernorm",
|
||||
"head.mlp",
|
||||
"head.probe",
|
||||
"packing_position_embedding",
|
||||
],
|
||||
)
|
||||
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
|
||||
|
||||
|
||||
@MULTIMODAL_REGISTRY.register_processor(
|
||||
|
||||
@@ -37,7 +37,6 @@ from vllm.distributed import get_pp_group, get_tensor_model_parallel_world_size
|
||||
from vllm.model_executor.layers.attention import Attention
|
||||
from vllm.model_executor.layers.fused_moe import (
|
||||
FusedMoE,
|
||||
fused_moe_make_expert_params_mapping,
|
||||
)
|
||||
from vllm.model_executor.layers.linear import (
|
||||
QKVParallelLinear,
|
||||
@@ -51,16 +50,12 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
|
||||
ParallelLMHead,
|
||||
VocabParallelEmbedding,
|
||||
)
|
||||
from vllm.model_executor.model_loader.weight_utils import (
|
||||
default_weight_loader,
|
||||
maybe_remap_kv_scale_name,
|
||||
)
|
||||
from vllm.sequence import IntermediateTensors
|
||||
|
||||
from .interfaces import SupportsLoRA, SupportsPP
|
||||
from .utils import (
|
||||
AutoWeightsLoader,
|
||||
is_pp_missing_parameter,
|
||||
WeightsMapper,
|
||||
make_empty_intermediate_tensors_factory,
|
||||
make_layers,
|
||||
maybe_prefix,
|
||||
@@ -289,6 +284,7 @@ class PhiMoE(nn.Module):
|
||||
tp_size=tp_size,
|
||||
custom_routing_function=phimoe_routing_function,
|
||||
prefix=f"{prefix}.experts",
|
||||
ckpt_names=("w1", "w2", "w3"),
|
||||
)
|
||||
|
||||
def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
|
||||
@@ -516,84 +512,18 @@ class PhiMoEModel(nn.Module):
|
||||
hidden_states = self.norm(hidden_states)
|
||||
return hidden_states
|
||||
|
||||
def get_expert_mapping(self) -> list[tuple[str, str, int, str]]:
|
||||
return fused_moe_make_expert_params_mapping(
|
||||
self,
|
||||
ckpt_gate_proj_name="w1",
|
||||
ckpt_down_proj_name="w2",
|
||||
ckpt_up_proj_name="w3",
|
||||
num_experts=self.config.num_local_experts,
|
||||
)
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
stacked_params_mapping = [
|
||||
# (param_name, shard_name, shard_id)
|
||||
("qkv_proj", "q_proj", "q"),
|
||||
("qkv_proj", "k_proj", "k"),
|
||||
("qkv_proj", "v_proj", "v"),
|
||||
]
|
||||
|
||||
params_dict = dict(self.named_parameters())
|
||||
loaded_params: set[str] = set()
|
||||
expert_params_mapping = self.get_expert_mapping()
|
||||
for name, loaded_weight in weights:
|
||||
for param_name, weight_name, shard_id in stacked_params_mapping:
|
||||
if weight_name not in name:
|
||||
continue
|
||||
name = name.replace(weight_name, param_name)
|
||||
# Skip loading extra bias for GPTQ models.
|
||||
if name.endswith(".bias") and name not in params_dict:
|
||||
continue
|
||||
# Skip layers on other devices.
|
||||
if is_pp_missing_parameter(name, self):
|
||||
continue
|
||||
param = params_dict[name]
|
||||
weight_loader = param.weight_loader
|
||||
weight_loader(param, loaded_weight, shard_id)
|
||||
break
|
||||
else:
|
||||
for mapping in expert_params_mapping:
|
||||
param_name, weight_name, expert_id, shard_id = mapping
|
||||
if weight_name not in name:
|
||||
continue
|
||||
name = name.replace(weight_name, param_name)
|
||||
# Skip layers on other devices.
|
||||
if is_pp_missing_parameter(name, self):
|
||||
continue
|
||||
param = params_dict[name]
|
||||
weight_loader = param.weight_loader
|
||||
weight_loader(
|
||||
param,
|
||||
loaded_weight,
|
||||
name,
|
||||
shard_id=shard_id,
|
||||
expert_id=expert_id,
|
||||
)
|
||||
break
|
||||
else:
|
||||
# Skip loading extra bias for GPTQ models.
|
||||
if name.endswith(".bias") and name not in params_dict:
|
||||
continue
|
||||
# Skip layers on other devices.
|
||||
if is_pp_missing_parameter(name, self):
|
||||
continue
|
||||
# Remapping the name of FP8 kv-scale.
|
||||
name = maybe_remap_kv_scale_name(name, params_dict)
|
||||
if name is None:
|
||||
continue
|
||||
|
||||
param = params_dict[name]
|
||||
weight_loader = getattr(
|
||||
param, "weight_loader", default_weight_loader
|
||||
)
|
||||
weight_loader(param, loaded_weight)
|
||||
loaded_params.add(name)
|
||||
return loaded_params
|
||||
|
||||
|
||||
class PhiMoEForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
|
||||
fall_back_to_pt_during_load = False
|
||||
|
||||
hf_to_vllm_mapper = WeightsMapper(
|
||||
orig_to_new_stacked={
|
||||
# weight_name: (param_name, shard_id)
|
||||
".q_proj": (".qkv_proj", "q"),
|
||||
".k_proj": (".qkv_proj", "k"),
|
||||
".v_proj": (".qkv_proj", "v"),
|
||||
}
|
||||
)
|
||||
packed_modules_mapping = {
|
||||
"qkv_proj": [
|
||||
"q_proj",
|
||||
@@ -654,7 +584,4 @@ class PhiMoEForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
loader = AutoWeightsLoader(self)
|
||||
return loader.load_weights(weights)
|
||||
|
||||
def get_expert_mapping(self) -> list[tuple[str, str, int, str]]:
|
||||
return self.model.get_expert_mapping()
|
||||
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
|
||||
|
||||
@@ -65,7 +65,6 @@ from vllm.model_executor.layers.rotary_embedding import get_rope
|
||||
from vllm.model_executor.layers.rotary_embedding.common import (
|
||||
ApplyRotaryEmb,
|
||||
)
|
||||
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
|
||||
from vllm.model_executor.models.module_mapping import MultiModelKeys
|
||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
from vllm.multimodal.evs import (
|
||||
@@ -613,6 +612,16 @@ class Qwen2_5_VisionPatchMerger(nn.Module):
|
||||
|
||||
|
||||
class Qwen2_5_VisionTransformer(nn.Module):
|
||||
hf_to_vllm_mapper = WeightsMapper(
|
||||
orig_to_new_stacked={
|
||||
".attn.q.": (".attn.qkv.", "q"),
|
||||
".attn.k.": (".attn.qkv.", "k"),
|
||||
".attn.v.": (".attn.qkv.", "v"),
|
||||
".mlp.gate_proj.": (".mlp.gate_up_proj.", 0),
|
||||
".mlp.up_proj.": (".mlp.gate_up_proj.", 1),
|
||||
}
|
||||
)
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
vision_config: Qwen2_5_VLVisionConfig,
|
||||
@@ -1116,32 +1125,8 @@ class Qwen2_5_VisionTransformer(nn.Module):
|
||||
return hidden_states
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
stacked_params_mapping = [
|
||||
# (param_name, shard_name, shard_id)
|
||||
("attn.qkv.", "attn.q.", "q"),
|
||||
("attn.qkv.", "attn.k.", "k"),
|
||||
("attn.qkv.", "attn.v.", "v"),
|
||||
("mlp.gate_up_proj.", "mlp.gate_proj.", 0),
|
||||
("mlp.gate_up_proj.", "mlp.up_proj.", 1),
|
||||
]
|
||||
params_dict = dict(self.named_parameters(remove_duplicate=False))
|
||||
loaded_params: set[str] = set()
|
||||
|
||||
for name, loaded_weight in weights:
|
||||
for param_name, weight_name, shard_id in stacked_params_mapping:
|
||||
if weight_name not in name:
|
||||
continue
|
||||
name = name.replace(weight_name, param_name)
|
||||
param = params_dict[name]
|
||||
weight_loader = param.weight_loader
|
||||
weight_loader(param, loaded_weight, shard_id)
|
||||
break
|
||||
else:
|
||||
param = params_dict[name]
|
||||
weight_loader = getattr(param, "weight_loader", default_weight_loader)
|
||||
weight_loader(param, loaded_weight)
|
||||
loaded_params.add(name)
|
||||
return loaded_params
|
||||
loader = AutoWeightsLoader(self)
|
||||
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
|
||||
|
||||
|
||||
class Qwen2_5_VLProcessingInfo(Qwen2VLProcessingInfo):
|
||||
|
||||
@@ -42,7 +42,6 @@ from vllm.model_executor.layers.activation import SiluAndMul
|
||||
from vllm.model_executor.layers.attention import Attention
|
||||
from vllm.model_executor.layers.fused_moe import (
|
||||
FusedMoE,
|
||||
fused_moe_make_expert_params_mapping,
|
||||
)
|
||||
from vllm.model_executor.layers.layernorm import RMSNorm
|
||||
from vllm.model_executor.layers.linear import (
|
||||
@@ -58,14 +57,13 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
|
||||
ParallelLMHead,
|
||||
VocabParallelEmbedding,
|
||||
)
|
||||
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
|
||||
from vllm.sequence import IntermediateTensors
|
||||
|
||||
from .interfaces import SupportsLoRA, SupportsPP
|
||||
from .utils import (
|
||||
AutoWeightsLoader,
|
||||
WeightsMapper,
|
||||
extract_layer_index,
|
||||
is_pp_missing_parameter,
|
||||
make_empty_intermediate_tensors_factory,
|
||||
make_layers,
|
||||
maybe_prefix,
|
||||
@@ -418,124 +416,22 @@ class Qwen2MoeModel(nn.Module):
|
||||
hidden_states, _ = self.norm(hidden_states, residual)
|
||||
return hidden_states
|
||||
|
||||
def get_expert_mapping(self) -> list[tuple[str, str, int, str]]:
|
||||
# Params for weights, fp8 weight scales, fp8 activation scales
|
||||
# (param_name, weight_name, expert_id, shard_id)
|
||||
return fused_moe_make_expert_params_mapping(
|
||||
self,
|
||||
ckpt_gate_proj_name="gate_proj",
|
||||
ckpt_down_proj_name="down_proj",
|
||||
ckpt_up_proj_name="up_proj",
|
||||
num_experts=self.config.num_experts,
|
||||
)
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
stacked_params_mapping = [
|
||||
# (param_name, shard_name, shard_id)
|
||||
("qkv_proj", "q_proj", "q"),
|
||||
("qkv_proj", "k_proj", "k"),
|
||||
("qkv_proj", "v_proj", "v"),
|
||||
("gate_up_proj", "gate_proj", 0),
|
||||
("gate_up_proj", "up_proj", 1),
|
||||
]
|
||||
|
||||
params_dict = dict(self.named_parameters())
|
||||
loaded_params: set[str] = set()
|
||||
expert_params_mapping = self.get_expert_mapping()
|
||||
for name, loaded_weight in weights:
|
||||
for param_name, weight_name, shard_id in stacked_params_mapping:
|
||||
# Skip non-stacked layers and experts (experts handled below).
|
||||
if weight_name not in name:
|
||||
continue
|
||||
# We have mlp.experts[0].gate_proj in the checkpoint.
|
||||
# Since we handle the experts below in expert_params_mapping,
|
||||
# we need to skip here BEFORE we update the name, otherwise
|
||||
# name will be updated to mlp.experts[0].gate_up_proj, which
|
||||
# will then be updated below in expert_params_mapping
|
||||
# for mlp.experts[0].gate_gate_up_proj, which breaks load.
|
||||
if "mlp.experts" in name:
|
||||
continue
|
||||
name = name.replace(weight_name, param_name)
|
||||
# Skip loading extra bias for GPTQ models.
|
||||
if (
|
||||
name.endswith(".bias") or name.endswith("_bias")
|
||||
) and name not in params_dict:
|
||||
continue
|
||||
# Skip layers on other devices.
|
||||
if is_pp_missing_parameter(name, self):
|
||||
continue
|
||||
if name not in params_dict:
|
||||
continue
|
||||
|
||||
param = params_dict[name]
|
||||
weight_loader = param.weight_loader
|
||||
weight_loader(param, loaded_weight, shard_id)
|
||||
break
|
||||
else:
|
||||
for mapping in expert_params_mapping:
|
||||
param_name, weight_name, expert_id, shard_id = mapping
|
||||
if weight_name not in name:
|
||||
continue
|
||||
name = name.replace(weight_name, param_name)
|
||||
|
||||
# Skip layers on other devices.
|
||||
if is_pp_missing_parameter(name, self):
|
||||
continue
|
||||
# Skip loading extra bias for GPTQ models.
|
||||
if (
|
||||
name.endswith(".bias") or name.endswith("_bias")
|
||||
) and name not in params_dict:
|
||||
continue
|
||||
param = params_dict[name]
|
||||
weight_loader = param.weight_loader
|
||||
weight_loader(
|
||||
param,
|
||||
loaded_weight,
|
||||
name,
|
||||
shard_id=shard_id,
|
||||
expert_id=expert_id,
|
||||
)
|
||||
break
|
||||
else:
|
||||
# Skip loading extra bias for GPTQ models.
|
||||
if (
|
||||
name.endswith(".bias") or name.endswith("_bias")
|
||||
) and name not in params_dict:
|
||||
continue
|
||||
# Skip layers on other devices.
|
||||
if is_pp_missing_parameter(name, self):
|
||||
continue
|
||||
# Remapping the name of FP8 kv-scale.
|
||||
if name.endswith("kv_scale"):
|
||||
remapped_kv_scale_name = name.replace(
|
||||
".kv_scale", ".attn.kv_scale"
|
||||
)
|
||||
if remapped_kv_scale_name not in params_dict:
|
||||
logger.warning_once(
|
||||
"Found kv_scale in the checkpoint (e.g. %s), but not found the expected name in the model (e.g. %s). kv_scale is not loaded.", # noqa: E501
|
||||
name,
|
||||
remapped_kv_scale_name,
|
||||
)
|
||||
continue
|
||||
else:
|
||||
name = remapped_kv_scale_name
|
||||
# GGUF: make sure that shared_expert_gate is a 2D tensor.
|
||||
if (
|
||||
"mlp.shared_expert_gate" in name
|
||||
and len(loaded_weight.shape) == 1
|
||||
):
|
||||
loaded_weight = loaded_weight[None, :]
|
||||
param = params_dict[name]
|
||||
weight_loader = getattr(
|
||||
param, "weight_loader", default_weight_loader
|
||||
)
|
||||
weight_loader(param, loaded_weight)
|
||||
loaded_params.add(name)
|
||||
return loaded_params
|
||||
|
||||
|
||||
class Qwen2MoeForCausalLM(nn.Module, SupportsPP, SupportsLoRA):
|
||||
fall_back_to_pt_during_load = False
|
||||
hf_to_vllm_mapper = WeightsMapper(
|
||||
orig_to_new_stacked={
|
||||
# weight_name: (param_name, shard_id)
|
||||
".q_proj": (".qkv_proj", "q"),
|
||||
".k_proj": (".qkv_proj", "k"),
|
||||
".v_proj": (".qkv_proj", "v"),
|
||||
# .experts.gate_up_proj must be handled by MoERunner.load_weights for EP
|
||||
".mlp.gate_proj": (".mlp.gate_up_proj", 0),
|
||||
".mlp.up_proj": (".mlp.gate_up_proj", 1),
|
||||
".shared_expert.gate_proj": (".shared_expert.gate_up_proj", 0),
|
||||
".shared_expert.up_proj": (".shared_expert.gate_up_proj", 1),
|
||||
}
|
||||
)
|
||||
packed_modules_mapping = {
|
||||
"qkv_proj": [
|
||||
"q_proj",
|
||||
@@ -596,8 +492,16 @@ class Qwen2MoeForCausalLM(nn.Module, SupportsPP, SupportsLoRA):
|
||||
return logits
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
loader = AutoWeightsLoader(self)
|
||||
return loader.load_weights(weights)
|
||||
def _maybe_reshape(
|
||||
weights: Iterable[tuple[str, torch.Tensor]],
|
||||
) -> Iterable[tuple[str, torch.Tensor]]:
|
||||
for name, loaded_weight in weights:
|
||||
# GGUF: make sure that shared_expert_gate is a 2D tensor.
|
||||
if "mlp.shared_expert_gate" in name and loaded_weight.dim() == 1:
|
||||
loaded_weight = loaded_weight[None, :]
|
||||
yield name, loaded_weight
|
||||
|
||||
def get_expert_mapping(self) -> list[tuple[str, str, int, str]]:
|
||||
return self.model.get_expert_mapping()
|
||||
loader = AutoWeightsLoader(self)
|
||||
return loader.load_weights(
|
||||
_maybe_reshape(weights), mapper=self.hf_to_vllm_mapper
|
||||
)
|
||||
|
||||
@@ -60,7 +60,6 @@ from vllm.model_executor.layers.rotary_embedding import get_rope
|
||||
from vllm.model_executor.layers.rotary_embedding.common import (
|
||||
ApplyRotaryEmb,
|
||||
)
|
||||
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
|
||||
from vllm.model_executor.models.module_mapping import MultiModelKeys
|
||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
from vllm.multimodal.inputs import (
|
||||
@@ -522,6 +521,15 @@ class Qwen2VisionPatchMerger(nn.Module):
|
||||
|
||||
|
||||
class Qwen2VisionTransformer(nn.Module):
|
||||
# Vision checkpoints store qkv pre-fused; merge separate q/k/v into qkv.
|
||||
hf_to_vllm_mapper = WeightsMapper(
|
||||
orig_to_new_stacked={
|
||||
".q_proj": (".qkv", "q"),
|
||||
".k_proj": (".qkv", "k"),
|
||||
".v_proj": (".qkv", "v"),
|
||||
}
|
||||
)
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
vision_config: Qwen2VLVisionConfig,
|
||||
@@ -742,31 +750,8 @@ class Qwen2VisionTransformer(nn.Module):
|
||||
return x
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
stacked_params_mapping = [
|
||||
# (param_name, shard_name, shard_id)
|
||||
("qkv_proj", "q_proj", "q"),
|
||||
("qkv_proj", "k_proj", "k"),
|
||||
("qkv_proj", "v_proj", "v"),
|
||||
]
|
||||
params_dict = dict(self.named_parameters(remove_duplicate=False))
|
||||
loaded_params: set[str] = set()
|
||||
|
||||
for name, loaded_weight in weights:
|
||||
for param_name, weight_name, shard_id in stacked_params_mapping:
|
||||
if weight_name not in name:
|
||||
continue
|
||||
name = name.replace(weight_name, param_name)
|
||||
|
||||
param = params_dict[name]
|
||||
weight_loader = param.weight_loader
|
||||
weight_loader(param, loaded_weight, shard_id)
|
||||
break
|
||||
else:
|
||||
param = params_dict[name]
|
||||
weight_loader = getattr(param, "weight_loader", default_weight_loader)
|
||||
weight_loader(param, loaded_weight)
|
||||
loaded_params.add(name)
|
||||
return loaded_params
|
||||
loader = AutoWeightsLoader(self)
|
||||
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
|
||||
|
||||
|
||||
def _create_qwen2vl_field_factory(
|
||||
|
||||
@@ -24,8 +24,7 @@
|
||||
# limitations under the License.
|
||||
"""Inference-only Qwen3.5 Series compatible with HuggingFace weights."""
|
||||
|
||||
import typing
|
||||
from collections.abc import Callable, Iterable
|
||||
from collections.abc import Iterable
|
||||
|
||||
import torch
|
||||
from torch import nn
|
||||
@@ -37,12 +36,7 @@ from vllm.distributed import (
|
||||
get_pp_group,
|
||||
)
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.fused_moe import (
|
||||
fused_moe_make_expert_params_mapping,
|
||||
)
|
||||
from vllm.model_executor.layers.layernorm import (
|
||||
GemmaRMSNorm as Qwen3_5RMSNorm,
|
||||
)
|
||||
from vllm.model_executor.layers.layernorm import GemmaRMSNorm as Qwen3_5RMSNorm
|
||||
from vllm.model_executor.layers.logits_processor import LogitsProcessor
|
||||
from vllm.model_executor.layers.mamba.gdn.qwen_gdn_linear_attn import (
|
||||
QwenGatedDeltaNetAttention,
|
||||
@@ -57,16 +51,9 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
|
||||
ParallelLMHead,
|
||||
VocabParallelEmbedding,
|
||||
)
|
||||
from vllm.model_executor.model_loader.weight_utils import (
|
||||
default_weight_loader,
|
||||
maybe_remap_kv_scale_name,
|
||||
)
|
||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
from vllm.sequence import IntermediateTensors
|
||||
from vllm.transformers_utils.configs.qwen3_5 import (
|
||||
Qwen3_5Config,
|
||||
Qwen3_5TextConfig,
|
||||
)
|
||||
from vllm.transformers_utils.configs.qwen3_5 import Qwen3_5Config, Qwen3_5TextConfig
|
||||
from vllm.transformers_utils.configs.qwen3_5_moe import (
|
||||
Qwen3_5MoeConfig,
|
||||
Qwen3_5MoeTextConfig,
|
||||
@@ -89,6 +76,7 @@ from .qwen3_next import (
|
||||
Qwen3NextModel,
|
||||
Qwen3NextSparseMoeBlock,
|
||||
QwenNextMixtureOfExperts,
|
||||
_is_shared_expert_fse_compatible,
|
||||
)
|
||||
from .qwen3_vl import (
|
||||
Qwen3_VisionTransformer,
|
||||
@@ -100,9 +88,9 @@ from .qwen3_vl import (
|
||||
from .utils import (
|
||||
AutoWeightsLoader,
|
||||
PPMissingLayer,
|
||||
WeightsMapper,
|
||||
_merge_multimodal_embeddings,
|
||||
extract_layer_index,
|
||||
is_pp_missing_parameter,
|
||||
make_empty_intermediate_tensors_factory,
|
||||
make_layers,
|
||||
maybe_prefix,
|
||||
@@ -210,6 +198,17 @@ class Qwen3_5DecoderLayer(Qwen3NextDecoderLayer):
|
||||
}
|
||||
)
|
||||
class Qwen3_5Model(Qwen3NextModel):
|
||||
# Qwen3.5 ships the GDN in_proj checkpoints separately (qwen3-next
|
||||
# pre-fuses them); fuse them on top of the qwen3-next QKV/gate_up mapping.
|
||||
hf_to_vllm_mapper = Qwen3NextModel.hf_to_vllm_mapper | WeightsMapper(
|
||||
orig_to_new_stacked={
|
||||
".in_proj_qkv": (".in_proj_qkvz", (0, 1, 2)),
|
||||
".in_proj_z": (".in_proj_qkvz", 3),
|
||||
".in_proj_b": (".in_proj_ba", 0),
|
||||
".in_proj_a": (".in_proj_ba", 1),
|
||||
}
|
||||
)
|
||||
|
||||
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
|
||||
super(Qwen3NextModel, self).__init__()
|
||||
|
||||
@@ -222,6 +221,7 @@ class Qwen3_5Model(Qwen3NextModel):
|
||||
self.num_redundant_experts = eplb_config.num_redundant_experts
|
||||
|
||||
self.config = config
|
||||
self.quant_config = vllm_config.quant_config
|
||||
|
||||
self.vocab_size = config.vocab_size
|
||||
|
||||
@@ -251,212 +251,21 @@ class Qwen3_5Model(Qwen3NextModel):
|
||||
|
||||
self.aux_hidden_state_layers: tuple[int, ...] = ()
|
||||
|
||||
def load_fused_expert_weights(
|
||||
self,
|
||||
name: str,
|
||||
params_dict: dict,
|
||||
loaded_weight: torch.Tensor,
|
||||
shard_id: str,
|
||||
num_experts: int,
|
||||
) -> bool:
|
||||
param = params_dict[name]
|
||||
weight_loader = typing.cast(Callable[..., bool], param.weight_loader)
|
||||
loaded_local_expert = False
|
||||
for expert_id in range(num_experts):
|
||||
curr_expert_weight = loaded_weight[expert_id]
|
||||
success = weight_loader(
|
||||
param,
|
||||
curr_expert_weight,
|
||||
name,
|
||||
shard_id=shard_id,
|
||||
expert_id=expert_id,
|
||||
return_success=True,
|
||||
)
|
||||
if success:
|
||||
loaded_local_expert = True
|
||||
|
||||
return loaded_local_expert
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
stacked_params_mapping = [
|
||||
# (param_name, shard_name, shard_id)
|
||||
# GDN
|
||||
("in_proj_qkvz", "in_proj_qkv", (0, 1, 2)),
|
||||
("in_proj_qkvz", "in_proj_z", 3),
|
||||
# self attention
|
||||
("qkv_proj", "q_proj", "q"),
|
||||
("qkv_proj", "k_proj", "k"),
|
||||
("qkv_proj", "v_proj", "v"),
|
||||
# mlp
|
||||
("gate_up_proj", "gate_proj", 0),
|
||||
("gate_up_proj", "up_proj", 1),
|
||||
("in_proj_ba", "in_proj_b", 0),
|
||||
("in_proj_ba", "in_proj_a", 1),
|
||||
]
|
||||
|
||||
params_dict = dict(self.named_parameters())
|
||||
loaded_params: set[str] = set()
|
||||
expert_params_mapping = self.get_expert_mapping()
|
||||
is_fused_expert = False
|
||||
fused_expert_params_mapping: list[tuple[str, str, int, str]] = []
|
||||
for param_name, ckpt_name, _, shard_id in fused_moe_make_expert_params_mapping(
|
||||
self,
|
||||
ckpt_gate_proj_name="gate_up_proj",
|
||||
ckpt_down_proj_name="down_proj",
|
||||
ckpt_up_proj_name="gate_up_proj",
|
||||
num_experts=1,
|
||||
):
|
||||
if shard_id == "w3":
|
||||
continue
|
||||
parts = ckpt_name.split(".")
|
||||
fused_expert_params_mapping.append(
|
||||
(f"{param_name}weight", f"{parts[0]}.{parts[2]}", 0, shard_id)
|
||||
mapper = self.hf_to_vllm_mapper
|
||||
# FSE must match construction (Qwen3NextSparseMoeBlock): reroute the
|
||||
# shared expert into the extra fused slot only when AITER FSE is both
|
||||
# requested and compatible with the quant spec.
|
||||
is_fse = rocm_aiter_ops.is_fusion_moe_shared_experts_enabled() and (
|
||||
_is_shared_expert_fse_compatible(self.quant_config)
|
||||
)
|
||||
if is_fse:
|
||||
num_routed = self.config.num_experts
|
||||
mapper = mapper | WeightsMapper(
|
||||
orig_to_new_substr={"mlp.shared_expert.": f"mlp.experts.{num_routed}."}
|
||||
)
|
||||
num_experts = (
|
||||
self.config.num_experts if hasattr(self.config, "num_experts") else 0
|
||||
)
|
||||
from vllm.config import get_current_vllm_config
|
||||
|
||||
from .qwen3_next import _is_shared_expert_fse_compatible
|
||||
|
||||
is_fse = (
|
||||
rocm_aiter_ops.is_fusion_moe_shared_experts_enabled()
|
||||
and _is_shared_expert_fse_compatible(get_current_vllm_config().quant_config)
|
||||
)
|
||||
|
||||
for name, loaded_weight in weights:
|
||||
if "rotary_emb.inv_freq" in name:
|
||||
continue
|
||||
|
||||
if name.startswith("mtp."):
|
||||
continue
|
||||
|
||||
# Remapping the name of FP8 kv-scale.
|
||||
if name.endswith("scale"):
|
||||
name = maybe_remap_kv_scale_name(name, params_dict)
|
||||
if name is None:
|
||||
continue
|
||||
|
||||
# FSE: remap shared_expert weights to fused expert slot
|
||||
if is_fse and "mlp.shared_expert." in name:
|
||||
name = name.replace(
|
||||
"mlp.shared_expert.",
|
||||
f"mlp.experts.{num_experts}.",
|
||||
)
|
||||
is_fused_expert = False
|
||||
expert_params_mapping = self.get_expert_mapping()
|
||||
|
||||
for param_name, weight_name, shard_id in stacked_params_mapping:
|
||||
if "experts.gate_up_proj" in name or "experts.down_proj" in name:
|
||||
is_fused_expert = True
|
||||
expert_params_mapping = fused_expert_params_mapping
|
||||
|
||||
if weight_name not in name:
|
||||
continue
|
||||
|
||||
if "mlp.experts" in name:
|
||||
continue
|
||||
|
||||
name = name.replace(weight_name, param_name)
|
||||
# Skip loading extra bias for GPTQ models.
|
||||
if name.endswith(".bias") and name not in params_dict:
|
||||
continue
|
||||
# Skip layers on other devices.
|
||||
if is_pp_missing_parameter(name, self):
|
||||
continue
|
||||
# name = apply_attn_prefix(name, params_dict)
|
||||
if name not in params_dict:
|
||||
continue
|
||||
param = params_dict[name]
|
||||
weight_loader = param.weight_loader
|
||||
weight_loader(param, loaded_weight, shard_id)
|
||||
break
|
||||
else:
|
||||
is_expert_weight = False
|
||||
for mapping in expert_params_mapping:
|
||||
param_name, weight_name, expert_id, shard_id = mapping
|
||||
if weight_name not in name:
|
||||
continue
|
||||
is_expert_weight = True
|
||||
name_mapped = name.replace(weight_name, param_name)
|
||||
# Skip layers on other devices.
|
||||
if is_pp_missing_parameter(name_mapped, self):
|
||||
continue
|
||||
if is_fused_expert:
|
||||
# qwen3.5 no need to transpose
|
||||
# loaded_weight = loaded_weight.transpose(-1, -2)
|
||||
if "experts.gate_up_proj" in name:
|
||||
loaded_weight = loaded_weight.chunk(2, dim=-2)
|
||||
success_w1 = self.load_fused_expert_weights(
|
||||
name_mapped,
|
||||
params_dict,
|
||||
loaded_weight[0],
|
||||
"w1",
|
||||
num_experts,
|
||||
)
|
||||
success_w3 = self.load_fused_expert_weights(
|
||||
name_mapped,
|
||||
params_dict,
|
||||
loaded_weight[1],
|
||||
"w3",
|
||||
num_experts,
|
||||
)
|
||||
success = success_w1 and success_w3
|
||||
else:
|
||||
# down_proj
|
||||
success = self.load_fused_expert_weights(
|
||||
name_mapped,
|
||||
params_dict,
|
||||
loaded_weight,
|
||||
shard_id,
|
||||
num_experts,
|
||||
)
|
||||
if success:
|
||||
name = name_mapped
|
||||
break
|
||||
else:
|
||||
# Skip loading extra bias for GPTQ models.
|
||||
if (
|
||||
name_mapped.endswith(".bias")
|
||||
or name_mapped.endswith("_bias")
|
||||
) and name_mapped not in params_dict:
|
||||
continue
|
||||
param = params_dict[name_mapped]
|
||||
weight_loader = param.weight_loader
|
||||
success = weight_loader(
|
||||
param,
|
||||
loaded_weight,
|
||||
name_mapped,
|
||||
shard_id=shard_id,
|
||||
expert_id=expert_id,
|
||||
return_success=True,
|
||||
)
|
||||
if success:
|
||||
name = name_mapped
|
||||
break
|
||||
else:
|
||||
if is_expert_weight:
|
||||
# We've checked that this is an expert weight
|
||||
# However it's not mapped locally to this rank
|
||||
# So we simply skip it
|
||||
continue
|
||||
# Skip loading extra bias for GPTQ models.
|
||||
if name.endswith(".bias") and name not in params_dict:
|
||||
continue
|
||||
if is_pp_missing_parameter(name, self):
|
||||
continue
|
||||
if name not in params_dict:
|
||||
logger.warning_once(
|
||||
f"Parameter {name} not found in params_dict, skip loading"
|
||||
)
|
||||
continue
|
||||
param = params_dict[name]
|
||||
weight_loader = getattr(
|
||||
param, "weight_loader", default_weight_loader
|
||||
)
|
||||
weight_loader(param, loaded_weight)
|
||||
loaded_params.add(name)
|
||||
return loaded_params
|
||||
loader = AutoWeightsLoader(self)
|
||||
return loader.load_weights(weights, mapper=mapper)
|
||||
|
||||
|
||||
class Qwen3_5ForCausalLMBase(
|
||||
@@ -566,9 +375,6 @@ class Qwen3_5MoeForCausalLM(Qwen3_5ForCausalLMBase, QwenNextMixtureOfExperts):
|
||||
# set MoE hyperparameters
|
||||
self.set_moe_parameters()
|
||||
|
||||
def get_expert_mapping(self) -> list[tuple[str, str, int, str]]:
|
||||
return self.model.get_expert_mapping()
|
||||
|
||||
|
||||
########################################################
|
||||
# Qwen3_5-Dense
|
||||
|
||||
@@ -2,29 +2,32 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Inference-only Qwen3_5 MTP model."""
|
||||
|
||||
import typing
|
||||
from collections.abc import Callable, Iterable
|
||||
from collections.abc import Iterable
|
||||
|
||||
import torch
|
||||
from torch import nn
|
||||
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
from vllm.compilation.decorators import support_torch_compile
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.config import VllmConfig, get_current_vllm_config
|
||||
from vllm.distributed.parallel_state import get_pp_group
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.fused_moe import (
|
||||
fused_moe_make_expert_params_mapping,
|
||||
)
|
||||
from vllm.model_executor.layers.linear import ColumnParallelLinear
|
||||
from vllm.model_executor.layers.logits_processor import LogitsProcessor
|
||||
from vllm.model_executor.layers.vocab_parallel_embedding import (
|
||||
ParallelLMHead,
|
||||
VocabParallelEmbedding,
|
||||
)
|
||||
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
|
||||
from vllm.model_executor.models.interfaces import LocalArgmaxMixin
|
||||
from vllm.model_executor.models.qwen3_5 import Qwen3_5DecoderLayer, Qwen3_5RMSNorm
|
||||
from vllm.model_executor.models.qwen3_next import QwenNextMixtureOfExperts
|
||||
from vllm.model_executor.models.qwen3_5 import (
|
||||
Qwen3_5DecoderLayer,
|
||||
Qwen3_5Model,
|
||||
Qwen3_5RMSNorm,
|
||||
)
|
||||
from vllm.model_executor.models.qwen3_next import (
|
||||
QwenNextMixtureOfExperts,
|
||||
_is_shared_expert_fse_compatible,
|
||||
)
|
||||
from vllm.sequence import IntermediateTensors
|
||||
from vllm.transformers_utils.configs.qwen3_5 import Qwen3_5TextConfig
|
||||
from vllm.transformers_utils.configs.qwen3_5_moe import Qwen3_5MoeTextConfig
|
||||
@@ -37,8 +40,8 @@ from .interfaces import (
|
||||
from .utils import (
|
||||
AutoWeightsLoader,
|
||||
PPMissingLayer,
|
||||
WeightsMapper,
|
||||
_merge_multimodal_embeddings,
|
||||
is_pp_missing_parameter,
|
||||
make_empty_intermediate_tensors_factory,
|
||||
maybe_prefix,
|
||||
)
|
||||
@@ -58,6 +61,8 @@ logger = init_logger(__name__)
|
||||
}
|
||||
)
|
||||
class Qwen3_5MultiTokenPredictor(nn.Module):
|
||||
hf_to_vllm_mapper = Qwen3_5Model.hf_to_vllm_mapper
|
||||
|
||||
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
|
||||
super().__init__()
|
||||
|
||||
@@ -159,188 +164,18 @@ class Qwen3_5MultiTokenPredictor(nn.Module):
|
||||
hidden_states, _ = self.norm(hidden_states, residual)
|
||||
return hidden_states
|
||||
|
||||
def load_fused_expert_weights(
|
||||
self,
|
||||
name: str,
|
||||
params_dict: dict,
|
||||
loaded_weight: torch.Tensor,
|
||||
shard_id: str,
|
||||
num_experts: int,
|
||||
) -> bool:
|
||||
param = params_dict[name]
|
||||
weight_loader = typing.cast(Callable[..., bool], param.weight_loader)
|
||||
loaded_local_expert = False
|
||||
for expert_id in range(num_experts):
|
||||
curr_expert_weight = loaded_weight[expert_id]
|
||||
success = weight_loader(
|
||||
param,
|
||||
curr_expert_weight,
|
||||
name,
|
||||
shard_id=shard_id,
|
||||
expert_id=expert_id,
|
||||
return_success=True,
|
||||
)
|
||||
if success:
|
||||
loaded_local_expert = True
|
||||
|
||||
return loaded_local_expert
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
stacked_params_mapping = [
|
||||
# (param_name, shard_name, shard_id)
|
||||
("qkv_proj", "q_proj", "q"),
|
||||
("qkv_proj", "k_proj", "k"),
|
||||
("qkv_proj", "v_proj", "v"),
|
||||
("gate_up_proj", "gate_proj", 0),
|
||||
("gate_up_proj", "up_proj", 1),
|
||||
]
|
||||
|
||||
# Params for weights, fp8 weight scales, fp8 activation scales
|
||||
# (param_name, weight_name, expert_id, shard_id)
|
||||
expert_params_mapping = fused_moe_make_expert_params_mapping(
|
||||
self,
|
||||
ckpt_gate_proj_name="gate_proj",
|
||||
ckpt_down_proj_name="down_proj",
|
||||
ckpt_up_proj_name="up_proj",
|
||||
num_experts=self.config.num_experts
|
||||
if hasattr(self.config, "num_experts")
|
||||
else 0,
|
||||
mapper = self.hf_to_vllm_mapper
|
||||
is_fse = rocm_aiter_ops.is_fusion_moe_shared_experts_enabled() and (
|
||||
_is_shared_expert_fse_compatible(get_current_vllm_config().quant_config)
|
||||
)
|
||||
|
||||
params_dict = dict(self.named_parameters())
|
||||
loaded_params: set[str] = set()
|
||||
is_fused_expert = False
|
||||
fused_expert_params_mapping: list[tuple[str, str, int, str]] = []
|
||||
for param_name, ckpt_name, _, shard_id in fused_moe_make_expert_params_mapping(
|
||||
self,
|
||||
ckpt_gate_proj_name="gate_up_proj",
|
||||
ckpt_down_proj_name="down_proj",
|
||||
ckpt_up_proj_name="gate_up_proj",
|
||||
num_experts=1,
|
||||
):
|
||||
if shard_id == "w3":
|
||||
continue
|
||||
parts = ckpt_name.split(".")
|
||||
fused_expert_params_mapping.append(
|
||||
(f"{param_name}weight", f"{parts[0]}.{parts[2]}", 0, shard_id)
|
||||
if is_fse:
|
||||
num_routed = getattr(self.config, "num_experts", 0)
|
||||
mapper = mapper | WeightsMapper(
|
||||
orig_to_new_substr={"mlp.shared_expert.": f"mlp.experts.{num_routed}."}
|
||||
)
|
||||
num_experts = (
|
||||
self.config.num_experts if hasattr(self.config, "num_experts") else 0
|
||||
)
|
||||
for name, loaded_weight in weights:
|
||||
if "rotary_emb.inv_freq" in name:
|
||||
continue
|
||||
|
||||
for param_name, weight_name, shard_id in stacked_params_mapping:
|
||||
if "experts.gate_up_proj" in name or "experts.down_proj" in name:
|
||||
is_fused_expert = True
|
||||
expert_params_mapping = fused_expert_params_mapping
|
||||
|
||||
if weight_name not in name:
|
||||
continue
|
||||
|
||||
if "mlp.experts" in name:
|
||||
continue
|
||||
|
||||
name = name.replace(weight_name, param_name)
|
||||
# Skip loading extra bias for GPTQ models.
|
||||
if name.endswith(".bias") and name not in params_dict:
|
||||
continue
|
||||
# Skip layers on other devices.
|
||||
if is_pp_missing_parameter(name, self):
|
||||
continue
|
||||
if name not in params_dict:
|
||||
continue
|
||||
param = params_dict[name]
|
||||
weight_loader = param.weight_loader
|
||||
weight_loader(param, loaded_weight, shard_id)
|
||||
break
|
||||
else:
|
||||
is_expert_weight = False
|
||||
for mapping in expert_params_mapping:
|
||||
param_name, weight_name, expert_id, shard_id = mapping
|
||||
if weight_name not in name:
|
||||
continue
|
||||
is_expert_weight = True
|
||||
name_mapped = name.replace(weight_name, param_name)
|
||||
# Skip layers on other devices.
|
||||
if is_pp_missing_parameter(name_mapped, self):
|
||||
continue
|
||||
if is_fused_expert:
|
||||
# qwen3.5 no need to transpose
|
||||
# loaded_weight = loaded_weight.transpose(-1, -2)
|
||||
if "experts.gate_up_proj" in name:
|
||||
loaded_weight = loaded_weight.chunk(2, dim=-2)
|
||||
success_w1 = self.load_fused_expert_weights(
|
||||
name_mapped,
|
||||
params_dict,
|
||||
loaded_weight[0],
|
||||
"w1",
|
||||
num_experts,
|
||||
)
|
||||
success_w3 = self.load_fused_expert_weights(
|
||||
name_mapped,
|
||||
params_dict,
|
||||
loaded_weight[1],
|
||||
"w3",
|
||||
num_experts,
|
||||
)
|
||||
success = success_w1 and success_w3
|
||||
else:
|
||||
# down_proj
|
||||
success = self.load_fused_expert_weights(
|
||||
name_mapped,
|
||||
params_dict,
|
||||
loaded_weight,
|
||||
shard_id,
|
||||
num_experts,
|
||||
)
|
||||
if success:
|
||||
name = name_mapped
|
||||
break
|
||||
else:
|
||||
# Skip loading extra bias for GPTQ models.
|
||||
if (
|
||||
name_mapped.endswith(".bias")
|
||||
or name_mapped.endswith("_bias")
|
||||
) and name_mapped not in params_dict:
|
||||
continue
|
||||
param = params_dict[name_mapped]
|
||||
weight_loader = param.weight_loader
|
||||
success = weight_loader(
|
||||
param,
|
||||
loaded_weight,
|
||||
name_mapped,
|
||||
shard_id=shard_id,
|
||||
expert_id=expert_id,
|
||||
return_success=True,
|
||||
)
|
||||
if success:
|
||||
name = name_mapped
|
||||
break
|
||||
else:
|
||||
if is_expert_weight:
|
||||
# We've checked that this is an expert weight
|
||||
# However it's not mapped locally to this rank
|
||||
# So we simply skip it
|
||||
continue
|
||||
# Skip loading extra bias for GPTQ models.
|
||||
if name.endswith(".bias") and name not in params_dict:
|
||||
continue
|
||||
if is_pp_missing_parameter(name, self):
|
||||
continue
|
||||
if name not in params_dict:
|
||||
logger.warning_once(
|
||||
f"Parameter {name} not found in params_dict, skip loading"
|
||||
)
|
||||
continue
|
||||
param = params_dict[name]
|
||||
weight_loader = getattr(
|
||||
param, "weight_loader", default_weight_loader
|
||||
)
|
||||
weight_loader(param, loaded_weight)
|
||||
loaded_params.add(name)
|
||||
return loaded_params
|
||||
loader = AutoWeightsLoader(self)
|
||||
return loader.load_weights(weights, mapper=mapper)
|
||||
|
||||
|
||||
@support_torch_compile(
|
||||
|
||||
@@ -17,15 +17,12 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
|
||||
ParallelLMHead,
|
||||
VocabParallelEmbedding,
|
||||
)
|
||||
from vllm.model_executor.model_loader.weight_utils import (
|
||||
default_weight_loader,
|
||||
maybe_remap_kv_scale_name,
|
||||
)
|
||||
from vllm.model_executor.models.qwen3 import Qwen3DecoderLayer, Qwen3ForCausalLM
|
||||
from vllm.multimodal.inputs import NestedTensors
|
||||
|
||||
from .utils import (
|
||||
AutoWeightsLoader,
|
||||
WeightsMapper,
|
||||
get_draft_quant_config,
|
||||
maybe_prefix,
|
||||
process_eagle_weight,
|
||||
@@ -132,6 +129,17 @@ class Qwen3Eagle3DecoderLayer(Qwen3DecoderLayer):
|
||||
}
|
||||
)
|
||||
class Qwen3Eagle3Model(nn.Module):
|
||||
hf_to_vllm_mapper = WeightsMapper(
|
||||
orig_to_new_substr={"midlayer.": "layers.0."},
|
||||
orig_to_new_stacked={
|
||||
".q_proj": (".qkv_proj", "q"),
|
||||
".k_proj": (".qkv_proj", "k"),
|
||||
".v_proj": (".qkv_proj", "v"),
|
||||
".gate_proj": (".gate_up_proj", 0),
|
||||
".up_proj": (".gate_up_proj", 1),
|
||||
},
|
||||
)
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
@@ -256,38 +264,8 @@ class Qwen3Eagle3Model(nn.Module):
|
||||
return hidden_states, aux_output
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
stacked_params_mapping = [
|
||||
# (param_name, shard_name, shard_id)
|
||||
(".qkv_proj", ".q_proj", "q"),
|
||||
(".qkv_proj", ".k_proj", "k"),
|
||||
(".qkv_proj", ".v_proj", "v"),
|
||||
(".gate_up_proj", ".gate_proj", 0),
|
||||
(".gate_up_proj", ".up_proj", 1),
|
||||
]
|
||||
params_dict = dict(self.named_parameters())
|
||||
loaded_params: set[str] = set()
|
||||
for name, loaded_weight in weights:
|
||||
if "midlayer." in name:
|
||||
name = name.replace("midlayer.", "layers.0.")
|
||||
# Remapping the name FP8 kv-scale or zero point.
|
||||
if "scale" in name or "zero_point" in name:
|
||||
name = maybe_remap_kv_scale_name(name, params_dict)
|
||||
if name is None:
|
||||
continue
|
||||
for param_name, weight_name, shard_id in stacked_params_mapping:
|
||||
if weight_name not in name:
|
||||
continue
|
||||
name = name.replace(weight_name, param_name)
|
||||
param = params_dict[name]
|
||||
weight_loader = param.weight_loader
|
||||
weight_loader(param, loaded_weight, shard_id)
|
||||
break
|
||||
else:
|
||||
param = params_dict[name]
|
||||
weight_loader = getattr(param, "weight_loader", default_weight_loader)
|
||||
weight_loader(param, loaded_weight)
|
||||
loaded_params.add(name)
|
||||
return loaded_params
|
||||
loader = AutoWeightsLoader(self)
|
||||
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
|
||||
|
||||
|
||||
class Eagle3Qwen3ForCausalLM(Qwen3ForCausalLM):
|
||||
|
||||
@@ -23,8 +23,7 @@
|
||||
# limitations under the License.
|
||||
"""Inference-only Qwen3MoE model compatible with HuggingFace weights."""
|
||||
|
||||
import typing
|
||||
from collections.abc import Callable, Iterable
|
||||
from collections.abc import Iterable
|
||||
from itertools import islice
|
||||
from typing import Any
|
||||
|
||||
@@ -43,10 +42,7 @@ from vllm.distributed import (
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.activation import SiluAndMul
|
||||
from vllm.model_executor.layers.attention import Attention
|
||||
from vllm.model_executor.layers.fused_moe import (
|
||||
FusedMoE,
|
||||
fused_moe_make_expert_params_mapping,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe import FusedMoE
|
||||
from vllm.model_executor.layers.layernorm import RMSNorm
|
||||
from vllm.model_executor.layers.linear import (
|
||||
MergedColumnParallelLinear,
|
||||
@@ -61,10 +57,6 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
|
||||
ParallelLMHead,
|
||||
VocabParallelEmbedding,
|
||||
)
|
||||
from vllm.model_executor.model_loader.weight_utils import (
|
||||
default_weight_loader,
|
||||
maybe_remap_kv_scale_name,
|
||||
)
|
||||
from vllm.model_executor.models.utils import sequence_parallel_chunk
|
||||
from vllm.sequence import IntermediateTensors
|
||||
|
||||
@@ -75,12 +67,13 @@ from .interfaces import (
|
||||
SupportsEagle3,
|
||||
SupportsLoRA,
|
||||
SupportsPP,
|
||||
SupportsQuant,
|
||||
)
|
||||
from .utils import (
|
||||
AutoWeightsLoader,
|
||||
PPMissingLayer,
|
||||
WeightsMapper,
|
||||
extract_layer_index,
|
||||
is_pp_missing_parameter,
|
||||
make_empty_intermediate_tensors_factory,
|
||||
make_layers,
|
||||
maybe_prefix,
|
||||
@@ -438,6 +431,20 @@ class Qwen3MoeDecoderLayer(nn.Module):
|
||||
|
||||
@support_torch_compile
|
||||
class Qwen3MoeModel(nn.Module, EagleModelMixin):
|
||||
hf_to_vllm_mapper = WeightsMapper(
|
||||
orig_to_new_stacked={
|
||||
# weight_name: (param_name, shard_id)
|
||||
".q_proj": (".qkv_proj", "q"),
|
||||
".k_proj": (".qkv_proj", "k"),
|
||||
".v_proj": (".qkv_proj", "v"),
|
||||
# .experts.gate_up_proj must be handled by MoERunner.load_weights for EP
|
||||
".mlp.gate_proj": (".mlp.gate_up_proj", 0),
|
||||
".mlp.up_proj": (".mlp.gate_up_proj", 1),
|
||||
".shared_expert.gate_proj": (".shared_expert.gate_up_proj", 0),
|
||||
".shared_expert.up_proj": (".shared_expert.gate_up_proj", 1),
|
||||
}
|
||||
)
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
@@ -516,152 +523,31 @@ class Qwen3MoeModel(nn.Module, EagleModelMixin):
|
||||
return hidden_states, aux_hidden_states
|
||||
return hidden_states
|
||||
|
||||
def get_expert_mapping(self) -> list[tuple[str, str, int, str]]:
|
||||
# Params for weights, fp8 weight scales, fp8 activation scales
|
||||
# (param_name, weight_name, expert_id, shard_id)
|
||||
return fused_moe_make_expert_params_mapping(
|
||||
self,
|
||||
ckpt_gate_proj_name="gate_proj",
|
||||
ckpt_down_proj_name="down_proj",
|
||||
ckpt_up_proj_name="up_proj",
|
||||
num_experts=self.config.num_experts,
|
||||
num_redundant_experts=self.num_redundant_experts,
|
||||
)
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
stacked_params_mapping = [
|
||||
# (param_name, shard_name, shard_id)
|
||||
("qkv_proj", "q_proj", "q"),
|
||||
("qkv_proj", "k_proj", "k"),
|
||||
("qkv_proj", "v_proj", "v"),
|
||||
("gate_up_proj", "gate_proj", 0),
|
||||
("gate_up_proj", "up_proj", 1),
|
||||
]
|
||||
|
||||
# Skip loading extra parameters for GPTQ/modelopt models.
|
||||
ignore_suffixes = (
|
||||
".bias",
|
||||
"_bias",
|
||||
".weight_scale",
|
||||
"_weight_scale",
|
||||
".input_scale",
|
||||
"_input_scale",
|
||||
loader = AutoWeightsLoader(
|
||||
self,
|
||||
ignore_unexpected_suffixes=[
|
||||
".bias",
|
||||
"_bias",
|
||||
".weight_scale",
|
||||
"_weight_scale",
|
||||
".input_scale",
|
||||
"_input_scale",
|
||||
],
|
||||
)
|
||||
|
||||
params_dict = dict(self.named_parameters())
|
||||
loaded_params: set[str] = set()
|
||||
expert_params_mapping = self.get_expert_mapping()
|
||||
for name, loaded_weight in weights:
|
||||
if "scale" in name or "zero_point" in name:
|
||||
name = maybe_remap_kv_scale_name(name, params_dict)
|
||||
if name is None:
|
||||
continue
|
||||
for param_name, weight_name, shard_id in stacked_params_mapping:
|
||||
# Skip non-stacked layers and experts (experts handled below).
|
||||
if weight_name not in name:
|
||||
continue
|
||||
# We have mlp.experts[0].gate_proj in the checkpoint.
|
||||
# Since we handle the experts below in expert_params_mapping,
|
||||
# we need to skip here BEFORE we update the name, otherwise
|
||||
# name will be updated to mlp.experts[0].gate_up_proj, which
|
||||
# will then be updated below in expert_params_mapping
|
||||
# for mlp.experts[0].gate_gate_up_proj, which breaks load.
|
||||
if "mlp.experts" in name:
|
||||
continue
|
||||
name = name.replace(weight_name, param_name)
|
||||
|
||||
# Skip loading extra parameters for GPTQ/modelopt models.
|
||||
if name.endswith(ignore_suffixes) and name not in params_dict:
|
||||
continue
|
||||
|
||||
# Skip layers on other devices.
|
||||
if is_pp_missing_parameter(name, self):
|
||||
continue
|
||||
if name.endswith("scale"):
|
||||
# Remapping the name of FP8 kv-scale.
|
||||
name = maybe_remap_kv_scale_name(name, params_dict)
|
||||
if name is None:
|
||||
continue
|
||||
if name not in params_dict:
|
||||
continue
|
||||
|
||||
param = params_dict[name]
|
||||
weight_loader = getattr(param, "weight_loader", default_weight_loader)
|
||||
if weight_loader == default_weight_loader:
|
||||
weight_loader(param, loaded_weight)
|
||||
else:
|
||||
weight_loader(param, loaded_weight, shard_id)
|
||||
break
|
||||
else:
|
||||
is_expert_weight = False
|
||||
for mapping in expert_params_mapping:
|
||||
param_name, weight_name, expert_id, shard_id = mapping
|
||||
if weight_name not in name:
|
||||
continue
|
||||
|
||||
# Anyway, this is an expert weight and should not be
|
||||
# attempted to load as other weights later
|
||||
is_expert_weight = True
|
||||
|
||||
# Do not modify `name` since the loop may continue here
|
||||
# Instead, create a new variable
|
||||
name_mapped = name.replace(weight_name, param_name)
|
||||
|
||||
if is_pp_missing_parameter(name_mapped, self):
|
||||
continue
|
||||
|
||||
# Skip loading extra parameters for GPTQ/modelopt models.
|
||||
if (
|
||||
name_mapped.endswith(ignore_suffixes)
|
||||
and name_mapped not in params_dict
|
||||
):
|
||||
continue
|
||||
|
||||
param = params_dict[name_mapped]
|
||||
# We should ask the weight loader to return success or not
|
||||
# here since otherwise we may skip experts with other
|
||||
# available replicas.
|
||||
weight_loader = typing.cast(
|
||||
Callable[..., bool], param.weight_loader
|
||||
)
|
||||
success = weight_loader(
|
||||
param,
|
||||
loaded_weight,
|
||||
name_mapped,
|
||||
shard_id=shard_id,
|
||||
expert_id=expert_id,
|
||||
return_success=True,
|
||||
)
|
||||
if success:
|
||||
name = name_mapped
|
||||
break
|
||||
else:
|
||||
if is_expert_weight:
|
||||
# We've checked that this is an expert weight
|
||||
# However it's not mapped locally to this rank
|
||||
# So we simply skip it
|
||||
continue
|
||||
|
||||
# Skip loading extra parameters for GPTQ/modelopt models.
|
||||
if name.endswith(ignore_suffixes) and name not in params_dict:
|
||||
continue
|
||||
# Skip layers on other devices.
|
||||
if is_pp_missing_parameter(name, self):
|
||||
continue
|
||||
if name not in params_dict:
|
||||
continue
|
||||
param = params_dict[name]
|
||||
weight_loader = getattr(
|
||||
param, "weight_loader", default_weight_loader
|
||||
)
|
||||
weight_loader(param, loaded_weight)
|
||||
loaded_params.add(name)
|
||||
return loaded_params
|
||||
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
|
||||
|
||||
|
||||
class Qwen3MoeForCausalLM(
|
||||
nn.Module, SupportsPP, SupportsLoRA, SupportsEagle, SupportsEagle3, MixtureOfExperts
|
||||
nn.Module,
|
||||
SupportsPP,
|
||||
SupportsLoRA,
|
||||
SupportsEagle,
|
||||
SupportsEagle3,
|
||||
MixtureOfExperts,
|
||||
SupportsQuant,
|
||||
):
|
||||
hf_to_vllm_mapper = Qwen3MoeModel.hf_to_vllm_mapper
|
||||
packed_modules_mapping = {
|
||||
"qkv_proj": [
|
||||
"q_proj",
|
||||
@@ -769,6 +655,3 @@ class Qwen3MoeForCausalLM(
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
loader = AutoWeightsLoader(self)
|
||||
return loader.load_weights(weights)
|
||||
|
||||
def get_expert_mapping(self) -> list[tuple[str, str, int, str]]:
|
||||
return self.model.get_expert_mapping()
|
||||
|
||||
@@ -10,12 +10,7 @@ from torch import nn
|
||||
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
from vllm.compilation.decorators import support_torch_compile
|
||||
from vllm.config import (
|
||||
CacheConfig,
|
||||
ModelConfig,
|
||||
VllmConfig,
|
||||
get_current_vllm_config,
|
||||
)
|
||||
from vllm.config import CacheConfig, ModelConfig, VllmConfig
|
||||
from vllm.distributed import (
|
||||
get_ep_group,
|
||||
get_pp_group,
|
||||
@@ -26,7 +21,6 @@ from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.attention import Attention
|
||||
from vllm.model_executor.layers.fused_moe import (
|
||||
FusedMoE,
|
||||
fused_moe_make_expert_params_mapping,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_qk_norm_rope import fused_qk_rmsnorm_rope_gate
|
||||
from vllm.model_executor.layers.layernorm import (
|
||||
@@ -53,10 +47,6 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
|
||||
ParallelLMHead,
|
||||
VocabParallelEmbedding,
|
||||
)
|
||||
from vllm.model_executor.model_loader.weight_utils import (
|
||||
default_weight_loader,
|
||||
maybe_remap_kv_scale_name,
|
||||
)
|
||||
from vllm.model_executor.models.qwen2_moe import Qwen2MoeMLP as Qwen3NextMLP
|
||||
from vllm.model_executor.models.utils import sequence_parallel_chunk
|
||||
from vllm.platforms import current_platform
|
||||
@@ -76,8 +66,8 @@ from .interfaces import (
|
||||
from .utils import (
|
||||
AutoWeightsLoader,
|
||||
PPMissingLayer,
|
||||
WeightsMapper,
|
||||
extract_layer_index,
|
||||
is_pp_missing_parameter,
|
||||
make_empty_intermediate_tensors_factory,
|
||||
make_layers,
|
||||
maybe_prefix,
|
||||
@@ -132,7 +122,6 @@ class Qwen3NextSparseMoeBlock(nn.Module):
|
||||
)
|
||||
|
||||
# Load balancing settings.
|
||||
vllm_config = get_current_vllm_config()
|
||||
eplb_config = vllm_config.parallel_config.eplb_config
|
||||
self.enable_eplb = parallel_config.enable_eplb
|
||||
|
||||
@@ -544,6 +533,19 @@ class Qwen3NextDecoderLayer(nn.Module):
|
||||
|
||||
@support_torch_compile
|
||||
class Qwen3NextModel(nn.Module, EagleModelMixin):
|
||||
hf_to_vllm_mapper = WeightsMapper(
|
||||
orig_to_new_stacked={
|
||||
# weight_name: (param_name, shard_id)
|
||||
".q_proj": (".qkv_proj", "q"),
|
||||
".k_proj": (".qkv_proj", "k"),
|
||||
".v_proj": (".qkv_proj", "v"),
|
||||
".mlp.gate_proj": (".mlp.gate_up_proj", 0),
|
||||
".mlp.up_proj": (".mlp.gate_up_proj", 1),
|
||||
".shared_expert.gate_proj": (".shared_expert.gate_up_proj", 0),
|
||||
".shared_expert.up_proj": (".shared_expert.gate_up_proj", 1),
|
||||
}
|
||||
)
|
||||
|
||||
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
|
||||
super().__init__()
|
||||
|
||||
@@ -625,123 +627,18 @@ class Qwen3NextModel(nn.Module, EagleModelMixin):
|
||||
return hidden_states, aux_hidden_states
|
||||
return hidden_states
|
||||
|
||||
def get_expert_mapping(self) -> list[tuple[str, str, int, str]]:
|
||||
# Params for weights, fp8 weight scales, fp8 activation scales
|
||||
# (param_name, weight_name, expert_id, shard_id)
|
||||
num_experts = getattr(self.config, "num_experts", 0)
|
||||
if rocm_aiter_ops.is_fusion_moe_shared_experts_enabled():
|
||||
num_experts += 1
|
||||
return fused_moe_make_expert_params_mapping(
|
||||
self,
|
||||
ckpt_gate_proj_name="gate_proj",
|
||||
ckpt_down_proj_name="down_proj",
|
||||
ckpt_up_proj_name="up_proj",
|
||||
num_experts=num_experts,
|
||||
num_redundant_experts=self.num_redundant_experts,
|
||||
)
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
stacked_params_mapping = [
|
||||
# (param_name, shard_name, shard_id)
|
||||
("qkv_proj", "q_proj", "q"),
|
||||
("qkv_proj", "k_proj", "k"),
|
||||
("qkv_proj", "v_proj", "v"),
|
||||
("gate_up_proj", "gate_proj", 0),
|
||||
("gate_up_proj", "up_proj", 1),
|
||||
]
|
||||
|
||||
params_dict = dict(self.named_parameters())
|
||||
loaded_params: set[str] = set()
|
||||
expert_params_mapping = self.get_expert_mapping()
|
||||
|
||||
is_fse = rocm_aiter_ops.is_fusion_moe_shared_experts_enabled()
|
||||
num_routed = getattr(self.config, "num_experts", 0)
|
||||
|
||||
for name, loaded_weight in weights:
|
||||
if "rotary_emb.inv_freq" in name:
|
||||
continue
|
||||
|
||||
if name.startswith("mtp."):
|
||||
continue
|
||||
|
||||
# Remapping the name of FP8 kv-scale.
|
||||
if name.endswith("scale"):
|
||||
name = maybe_remap_kv_scale_name(name, params_dict)
|
||||
if name is None:
|
||||
continue
|
||||
|
||||
# FSE: remap shared_expert weights to the fused expert slot
|
||||
if is_fse and "mlp.shared_expert." in name:
|
||||
name = name.replace(
|
||||
"mlp.shared_expert.",
|
||||
f"mlp.experts.{num_routed}.",
|
||||
)
|
||||
|
||||
for param_name, weight_name, shard_id in stacked_params_mapping:
|
||||
if weight_name not in name:
|
||||
continue
|
||||
|
||||
if "mlp.experts" in name:
|
||||
continue
|
||||
|
||||
name = name.replace(weight_name, param_name)
|
||||
# Skip loading extra bias for GPTQ models.
|
||||
if name.endswith(".bias") and name not in params_dict:
|
||||
continue
|
||||
# Skip layers on other devices.
|
||||
if is_pp_missing_parameter(name, self):
|
||||
continue
|
||||
# name = apply_attn_prefix(name, params_dict)
|
||||
if name not in params_dict:
|
||||
continue
|
||||
param = params_dict[name]
|
||||
weight_loader = param.weight_loader
|
||||
weight_loader(param, loaded_weight, shard_id)
|
||||
break
|
||||
else:
|
||||
for mapping in expert_params_mapping:
|
||||
param_name, weight_name, expert_id, shard_id = mapping
|
||||
if weight_name not in name:
|
||||
continue
|
||||
name = name.replace(weight_name, param_name)
|
||||
# Skip layers on other devices.
|
||||
if is_pp_missing_parameter(name, self):
|
||||
continue
|
||||
# Skip loading extra bias for GPTQ models.
|
||||
if (
|
||||
name.endswith(".bias") or name.endswith("_bias")
|
||||
) and name not in params_dict:
|
||||
continue
|
||||
if name not in params_dict:
|
||||
continue
|
||||
param = params_dict[name]
|
||||
weight_loader = param.weight_loader
|
||||
weight_loader(
|
||||
param,
|
||||
loaded_weight,
|
||||
name,
|
||||
shard_id=shard_id,
|
||||
expert_id=expert_id,
|
||||
)
|
||||
break
|
||||
else:
|
||||
# Skip loading extra bias for GPTQ models.
|
||||
if name.endswith(".bias") and name not in params_dict:
|
||||
continue
|
||||
if is_pp_missing_parameter(name, self):
|
||||
continue
|
||||
if name not in params_dict:
|
||||
logger.warning_once(
|
||||
f"Parameter {name} not found in params_dict, skip loading"
|
||||
)
|
||||
continue
|
||||
param = params_dict[name]
|
||||
weight_loader = getattr(
|
||||
param, "weight_loader", default_weight_loader
|
||||
)
|
||||
weight_loader(param, loaded_weight)
|
||||
loaded_params.add(name)
|
||||
return loaded_params
|
||||
mapper = self.hf_to_vllm_mapper
|
||||
if rocm_aiter_ops.is_fusion_moe_shared_experts_enabled():
|
||||
# AITER fused-shared-experts: route the shared_expert checkpoint
|
||||
# weights into the extra fused expert slot. Merge (not mutate) so the
|
||||
# shared class mapper isn't permanently altered.
|
||||
num_routed = getattr(self.config, "num_experts", 0)
|
||||
mapper = mapper | WeightsMapper(
|
||||
orig_to_new_substr={"mlp.shared_expert.": f"mlp.experts.{num_routed}."}
|
||||
)
|
||||
loader = AutoWeightsLoader(self)
|
||||
return loader.load_weights(weights, mapper=mapper)
|
||||
|
||||
|
||||
class QwenNextMixtureOfExperts(MixtureOfExperts):
|
||||
@@ -901,11 +798,5 @@ class Qwen3NextForCausalLM(
|
||||
return self.logits_processor(self.lm_head, hidden_states)
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
loader = AutoWeightsLoader(
|
||||
self,
|
||||
skip_prefixes=["mtp."],
|
||||
)
|
||||
loader = AutoWeightsLoader(self, skip_prefixes=["mtp."])
|
||||
return loader.load_weights(weights)
|
||||
|
||||
def get_expert_mapping(self) -> list[tuple[str, str, int, str]]:
|
||||
return self.model.get_expert_mapping()
|
||||
|
||||
@@ -12,18 +12,15 @@ from vllm.compilation.decorators import support_torch_compile
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.distributed.parallel_state import get_pp_group
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.fused_moe import (
|
||||
fused_moe_make_expert_params_mapping,
|
||||
)
|
||||
from vllm.model_executor.layers.linear import ColumnParallelLinear
|
||||
from vllm.model_executor.layers.logits_processor import LogitsProcessor
|
||||
from vllm.model_executor.layers.vocab_parallel_embedding import (
|
||||
ParallelLMHead,
|
||||
VocabParallelEmbedding,
|
||||
)
|
||||
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
|
||||
from vllm.model_executor.models.qwen3_next import (
|
||||
Qwen3NextDecoderLayer,
|
||||
Qwen3NextModel,
|
||||
Qwen3NextRMSNorm,
|
||||
QwenNextMixtureOfExperts,
|
||||
)
|
||||
@@ -32,7 +29,7 @@ from vllm.transformers_utils.configs.qwen3_next import Qwen3NextConfig
|
||||
|
||||
from .utils import (
|
||||
AutoWeightsLoader,
|
||||
is_pp_missing_parameter,
|
||||
WeightsMapper,
|
||||
make_empty_intermediate_tensors_factory,
|
||||
maybe_prefix,
|
||||
)
|
||||
@@ -44,6 +41,8 @@ KVCache = tuple[torch.Tensor, torch.Tensor]
|
||||
|
||||
@support_torch_compile
|
||||
class Qwen3NextMultiTokenPredictor(nn.Module):
|
||||
hf_to_vllm_mapper = Qwen3NextModel.hf_to_vllm_mapper
|
||||
|
||||
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
|
||||
super().__init__()
|
||||
|
||||
@@ -146,101 +145,16 @@ class Qwen3NextMultiTokenPredictor(nn.Module):
|
||||
return hidden_states
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
stacked_params_mapping = [
|
||||
# (param_name, shard_name, shard_id)
|
||||
("qkv_proj", "q_proj", "q"),
|
||||
("qkv_proj", "k_proj", "k"),
|
||||
("qkv_proj", "v_proj", "v"),
|
||||
("gate_up_proj", "gate_proj", 0),
|
||||
("gate_up_proj", "up_proj", 1),
|
||||
]
|
||||
|
||||
# Params for weights, fp8 weight scales, fp8 activation scales
|
||||
# (param_name, weight_name, expert_id, shard_id)
|
||||
is_fse = rocm_aiter_ops.is_fusion_moe_shared_experts_enabled()
|
||||
num_experts = self.config.num_experts
|
||||
if is_fse:
|
||||
num_experts += 1
|
||||
expert_params_mapping = fused_moe_make_expert_params_mapping(
|
||||
self,
|
||||
ckpt_gate_proj_name="gate_proj",
|
||||
ckpt_down_proj_name="down_proj",
|
||||
ckpt_up_proj_name="up_proj",
|
||||
num_experts=num_experts,
|
||||
)
|
||||
num_routed = self.config.num_experts
|
||||
|
||||
params_dict = dict(self.named_parameters())
|
||||
loaded_params: set[str] = set()
|
||||
for name, loaded_weight in weights:
|
||||
if "rotary_emb.inv_freq" in name:
|
||||
continue
|
||||
|
||||
# FSE: remap shared_expert weights to the fused expert slot
|
||||
if is_fse and "mlp.shared_expert." in name:
|
||||
name = name.replace(
|
||||
"mlp.shared_expert.",
|
||||
f"mlp.experts.{num_routed}.",
|
||||
)
|
||||
|
||||
for param_name, weight_name, shard_id in stacked_params_mapping:
|
||||
if weight_name not in name:
|
||||
continue
|
||||
|
||||
if "mlp.experts" in name:
|
||||
continue
|
||||
|
||||
name = name.replace(weight_name, param_name)
|
||||
# Skip loading extra bias for GPTQ models.
|
||||
if name.endswith(".bias") and name not in params_dict:
|
||||
continue
|
||||
# Skip layers on other devices.
|
||||
if is_pp_missing_parameter(name, self):
|
||||
continue
|
||||
if name not in params_dict:
|
||||
continue
|
||||
param = params_dict[name]
|
||||
weight_loader = param.weight_loader
|
||||
weight_loader(param, loaded_weight, shard_id)
|
||||
break
|
||||
else:
|
||||
for mapping in expert_params_mapping:
|
||||
param_name, weight_name, expert_id, shard_id = mapping
|
||||
if weight_name not in name:
|
||||
continue
|
||||
name = name.replace(weight_name, param_name)
|
||||
# Skip layers on other devices.
|
||||
if is_pp_missing_parameter(name, self):
|
||||
continue
|
||||
# Skip loading extra bias for GPTQ models.
|
||||
if (
|
||||
name.endswith(".bias") or name.endswith("_bias")
|
||||
) and name not in params_dict:
|
||||
continue
|
||||
param = params_dict[name]
|
||||
weight_loader = param.weight_loader
|
||||
weight_loader(
|
||||
param,
|
||||
loaded_weight,
|
||||
name,
|
||||
shard_id=shard_id,
|
||||
expert_id=expert_id,
|
||||
)
|
||||
break
|
||||
else:
|
||||
# Skip loading extra bias for GPTQ models.
|
||||
if name.endswith(".bias") and name not in params_dict:
|
||||
continue
|
||||
if is_pp_missing_parameter(name, self):
|
||||
continue
|
||||
|
||||
param = params_dict[name]
|
||||
weight_loader = getattr(
|
||||
param, "weight_loader", default_weight_loader
|
||||
)
|
||||
weight_loader(param, loaded_weight)
|
||||
loaded_params.add(name)
|
||||
return loaded_params
|
||||
mapper = self.hf_to_vllm_mapper
|
||||
if rocm_aiter_ops.is_fusion_moe_shared_experts_enabled():
|
||||
# AITER fused-shared-experts: route the shared_expert checkpoint
|
||||
# weights into the extra fused expert slot.
|
||||
num_routed = self.config.num_experts
|
||||
mapper = mapper | WeightsMapper(
|
||||
orig_to_new_substr={"mlp.shared_expert.": f"mlp.experts.{num_routed}."}
|
||||
)
|
||||
loader = AutoWeightsLoader(self)
|
||||
return loader.load_weights(weights, mapper=mapper)
|
||||
|
||||
|
||||
@support_torch_compile
|
||||
|
||||
@@ -63,7 +63,6 @@ from vllm.model_executor.layers.logits_processor import LogitsProcessor
|
||||
from vllm.model_executor.layers.quantization import QuantizationConfig
|
||||
from vllm.model_executor.layers.rotary_embedding import get_rope
|
||||
from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
|
||||
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
|
||||
from vllm.model_executor.models.module_mapping import MultiModelKeys
|
||||
from vllm.model_executor.models.qwen2_audio import Qwen2AudioProcessingInfo
|
||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
@@ -319,6 +318,14 @@ class Qwen3OmniMoeAudioEncoderLayer(nn.Module):
|
||||
class Qwen3OmniMoeAudioEncoder(nn.Module):
|
||||
"""vLLM-native Qwen3-Omni Audio Encoder."""
|
||||
|
||||
hf_to_vllm_mapper = WeightsMapper(
|
||||
orig_to_new_stacked={
|
||||
".self_attn.q_proj.": (".self_attn.qkv.", "q"),
|
||||
".self_attn.k_proj.": (".self_attn.qkv.", "k"),
|
||||
".self_attn.v_proj.": (".self_attn.qkv.", "v"),
|
||||
}
|
||||
)
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
config: Qwen3OmniMoeAudioEncoderConfig,
|
||||
@@ -531,35 +538,8 @@ class Qwen3OmniMoeAudioEncoder(nn.Module):
|
||||
return lengths
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
"""Load weights with mapping from HuggingFace format."""
|
||||
stacked_params_mapping = [
|
||||
# (param_name, shard_name, shard_id)
|
||||
("self_attn.qkv.", "self_attn.q_proj.", "q"),
|
||||
("self_attn.qkv.", "self_attn.k_proj.", "k"),
|
||||
("self_attn.qkv.", "self_attn.v_proj.", "v"),
|
||||
]
|
||||
params_dict = dict(self.named_parameters(remove_duplicate=False))
|
||||
loaded_params: set[str] = set()
|
||||
|
||||
for name, loaded_weight in weights:
|
||||
for param_name, weight_name, shard_id in stacked_params_mapping:
|
||||
if weight_name not in name:
|
||||
continue
|
||||
name = name.replace(weight_name, param_name)
|
||||
|
||||
param = params_dict[name]
|
||||
weight_loader = param.weight_loader
|
||||
weight_loader(param, loaded_weight, shard_id)
|
||||
break
|
||||
else:
|
||||
param = params_dict.get(name)
|
||||
if param is not None:
|
||||
weight_loader = getattr(
|
||||
param, "weight_loader", default_weight_loader
|
||||
)
|
||||
weight_loader(param, loaded_weight)
|
||||
loaded_params.add(name)
|
||||
return loaded_params
|
||||
loader = AutoWeightsLoader(self)
|
||||
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
|
||||
|
||||
|
||||
class Qwen3_VisionPatchEmbed(nn.Module):
|
||||
@@ -737,6 +717,14 @@ class Qwen3_VisionPatchMerger(nn.Module):
|
||||
|
||||
|
||||
class Qwen3Omni_VisionTransformer(nn.Module):
|
||||
hf_to_vllm_mapper = WeightsMapper(
|
||||
orig_to_new_stacked={
|
||||
".attn.q.": (".attn.qkv.", "q"),
|
||||
".attn.k.": (".attn.qkv.", "k"),
|
||||
".attn.v.": (".attn.qkv.", "v"),
|
||||
}
|
||||
)
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
vision_config,
|
||||
@@ -1044,31 +1032,8 @@ class Qwen3Omni_VisionTransformer(nn.Module):
|
||||
return hidden_states
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
stacked_params_mapping = [
|
||||
# (param_name, shard_name, shard_id)
|
||||
("attn.qkv.", "attn.q.", "q"),
|
||||
("attn.qkv.", "attn.k.", "k"),
|
||||
("attn.qkv.", "attn.v.", "v"),
|
||||
]
|
||||
params_dict = dict(self.named_parameters(remove_duplicate=False))
|
||||
loaded_params: set[str] = set()
|
||||
|
||||
for name, loaded_weight in weights:
|
||||
for param_name, weight_name, shard_id in stacked_params_mapping:
|
||||
if weight_name not in name:
|
||||
continue
|
||||
name = name.replace(weight_name, param_name)
|
||||
|
||||
param = params_dict[name]
|
||||
weight_loader = param.weight_loader
|
||||
weight_loader(param, loaded_weight, shard_id)
|
||||
break
|
||||
else:
|
||||
param = params_dict[name]
|
||||
weight_loader = getattr(param, "weight_loader", default_weight_loader)
|
||||
weight_loader(param, loaded_weight)
|
||||
loaded_params.add(name)
|
||||
return loaded_params
|
||||
loader = AutoWeightsLoader(self)
|
||||
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
|
||||
|
||||
|
||||
@support_torch_compile(
|
||||
|
||||
@@ -67,7 +67,6 @@ from vllm.model_executor.layers.logits_processor import LogitsProcessor
|
||||
from vllm.model_executor.layers.quantization import QuantizationConfig
|
||||
from vllm.model_executor.layers.rotary_embedding import get_rope
|
||||
from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
|
||||
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
|
||||
from vllm.model_executor.models.module_mapping import MultiModelKeys
|
||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
from vllm.multimodal.evs import (
|
||||
@@ -518,6 +517,14 @@ class Qwen3_VisionPatchMerger(nn.Module):
|
||||
|
||||
|
||||
class Qwen3_VisionTransformer(nn.Module):
|
||||
hf_to_vllm_mapper = WeightsMapper(
|
||||
orig_to_new_stacked={
|
||||
"attn.q.": ("attn.qkv.", "q"),
|
||||
"attn.k.": ("attn.qkv.", "k"),
|
||||
"attn.v.": ("attn.qkv.", "v"),
|
||||
}
|
||||
)
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
vision_config: Qwen3VLVisionConfig,
|
||||
@@ -834,31 +841,8 @@ class Qwen3_VisionTransformer(nn.Module):
|
||||
return hidden_states
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
stacked_params_mapping = [
|
||||
# (param_name, shard_name, shard_id)
|
||||
("attn.qkv.", "attn.q.", "q"),
|
||||
("attn.qkv.", "attn.k.", "k"),
|
||||
("attn.qkv.", "attn.v.", "v"),
|
||||
]
|
||||
params_dict = dict(self.named_parameters(remove_duplicate=False))
|
||||
loaded_params: set[str] = set()
|
||||
|
||||
for name, loaded_weight in weights:
|
||||
for param_name, weight_name, shard_id in stacked_params_mapping:
|
||||
if weight_name not in name:
|
||||
continue
|
||||
name = name.replace(weight_name, param_name)
|
||||
|
||||
param = params_dict[name]
|
||||
weight_loader = param.weight_loader
|
||||
weight_loader(param, loaded_weight, shard_id)
|
||||
break
|
||||
else:
|
||||
param = params_dict[name]
|
||||
weight_loader = getattr(param, "weight_loader", default_weight_loader)
|
||||
weight_loader(param, loaded_weight)
|
||||
loaded_params.add(name)
|
||||
return loaded_params
|
||||
loader = AutoWeightsLoader(self)
|
||||
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
|
||||
|
||||
|
||||
class Qwen3VLProcessingInfo(Qwen2VLProcessingInfo):
|
||||
|
||||
@@ -24,8 +24,6 @@
|
||||
# limitations under the License.
|
||||
"""Inference-only Qwen3-VL-MoE model compatible with HuggingFace weights."""
|
||||
|
||||
import typing
|
||||
from collections.abc import Callable, Iterable
|
||||
from itertools import islice
|
||||
|
||||
import torch
|
||||
@@ -39,10 +37,6 @@ from vllm.distributed import get_pp_group
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.logits_processor import LogitsProcessor
|
||||
from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
|
||||
from vllm.model_executor.model_loader.weight_utils import (
|
||||
default_weight_loader,
|
||||
maybe_remap_kv_scale_name,
|
||||
)
|
||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
from vllm.sequence import IntermediateTensors
|
||||
from vllm.tokenizers.registry import cached_tokenizer_from_config
|
||||
@@ -60,7 +54,7 @@ from .qwen3_vl import (
|
||||
Qwen3VLMultiModalProcessor,
|
||||
Qwen3VLProcessingInfo,
|
||||
)
|
||||
from .utils import is_pp_missing_parameter, maybe_prefix
|
||||
from .utils import maybe_prefix
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
@@ -135,202 +129,6 @@ class Qwen3MoeLLMModel(Qwen3MoeModel):
|
||||
return hidden_states, aux_hidden_states
|
||||
return hidden_states
|
||||
|
||||
def load_fused_expert_weights(
|
||||
self,
|
||||
name: str,
|
||||
params_dict: dict,
|
||||
loaded_weight: torch.Tensor,
|
||||
shard_id: str,
|
||||
num_experts: int,
|
||||
) -> bool:
|
||||
param = params_dict[name]
|
||||
weight_loader = typing.cast(Callable[..., bool], param.weight_loader)
|
||||
loaded_local_expert = False
|
||||
for expert_id in range(num_experts):
|
||||
curr_expert_weight = loaded_weight[expert_id]
|
||||
success = weight_loader(
|
||||
param,
|
||||
curr_expert_weight,
|
||||
name,
|
||||
shard_id=shard_id,
|
||||
expert_id=expert_id,
|
||||
return_success=True,
|
||||
)
|
||||
if success:
|
||||
loaded_local_expert = True
|
||||
|
||||
return loaded_local_expert
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
stacked_params_mapping = [
|
||||
# (param_name, shard_name, shard_id)
|
||||
("qkv_proj", "q_proj", "q"),
|
||||
("qkv_proj", "k_proj", "k"),
|
||||
("qkv_proj", "v_proj", "v"),
|
||||
("gate_up_proj", "gate_proj", 0),
|
||||
("gate_up_proj", "up_proj", 1),
|
||||
]
|
||||
# Skip loading extra parameters for GPTQ/modelopt models.
|
||||
ignore_suffixes = (
|
||||
".bias",
|
||||
"_bias",
|
||||
".weight_scale",
|
||||
"_weight_scale",
|
||||
".input_scale",
|
||||
"_input_scale",
|
||||
)
|
||||
params_dict = dict(self.named_parameters())
|
||||
loaded_params: set[str] = set()
|
||||
expert_params_mapping = self.get_expert_mapping()
|
||||
is_fused_expert = False
|
||||
base_layer = (
|
||||
"base_layer." if any(".base_layer." in name for name in params_dict) else ""
|
||||
)
|
||||
fused_expert_params_mapping = [
|
||||
(
|
||||
f"experts.routed_experts.{base_layer}w13_weight",
|
||||
"experts.gate_up_proj",
|
||||
0,
|
||||
"w1",
|
||||
),
|
||||
(
|
||||
f"experts.routed_experts.{base_layer}w2_weight",
|
||||
"experts.down_proj",
|
||||
0,
|
||||
"w2",
|
||||
),
|
||||
]
|
||||
num_experts = self.config.num_experts
|
||||
for name, loaded_weight in weights:
|
||||
if "scale" in name or "zero_point" in name:
|
||||
name = maybe_remap_kv_scale_name(name, params_dict)
|
||||
if name is None:
|
||||
continue
|
||||
|
||||
for param_name, weight_name, shard_id in stacked_params_mapping:
|
||||
if "experts.gate_up_proj" in name or "experts.down_proj" in name:
|
||||
is_fused_expert = True
|
||||
expert_params_mapping = fused_expert_params_mapping
|
||||
|
||||
# Skip non-stacked layers and experts (experts handled below).
|
||||
if weight_name not in name:
|
||||
continue
|
||||
# We have mlp.experts[0].gate_proj in the checkpoint.
|
||||
# Since we handle the experts below in expert_params_mapping,
|
||||
# we need to skip here BEFORE we update the name, otherwise
|
||||
# name will be updated to mlp.experts[0].gate_up_proj, which
|
||||
# will then be updated below in expert_params_mapping
|
||||
# for mlp.experts[0].gate_gate_up_proj, which breaks load.
|
||||
if "mlp.experts" in name:
|
||||
continue
|
||||
name = name.replace(weight_name, param_name)
|
||||
# Skip loading extra parameters for GPTQ/modelopt models.
|
||||
if name.endswith(ignore_suffixes) and name not in params_dict:
|
||||
continue
|
||||
# Skip layers on other devices.
|
||||
if is_pp_missing_parameter(name, self):
|
||||
continue
|
||||
if name.endswith("scale"):
|
||||
# Remapping the name of FP8 kv-scale.
|
||||
name = maybe_remap_kv_scale_name(name, params_dict)
|
||||
if name is None:
|
||||
continue
|
||||
if name not in params_dict:
|
||||
continue
|
||||
param = params_dict[name]
|
||||
weight_loader = getattr(param, "weight_loader", default_weight_loader)
|
||||
if weight_loader == default_weight_loader:
|
||||
weight_loader(param, loaded_weight)
|
||||
else:
|
||||
weight_loader(param, loaded_weight, shard_id)
|
||||
break
|
||||
else:
|
||||
is_expert_weight = False
|
||||
for mapping in expert_params_mapping:
|
||||
param_name, weight_name, expert_id, shard_id = mapping
|
||||
if weight_name not in name:
|
||||
continue
|
||||
# Anyway, this is an expert weight and should not be
|
||||
# attempted to load as other weights later
|
||||
is_expert_weight = True
|
||||
name_mapped = name.replace(weight_name, param_name)
|
||||
if is_pp_missing_parameter(name_mapped, self):
|
||||
continue
|
||||
if is_fused_expert:
|
||||
loaded_weight = loaded_weight.transpose(-1, -2) # no bias
|
||||
if "experts.gate_up_proj" in name:
|
||||
loaded_weight = loaded_weight.chunk(2, dim=-2)
|
||||
success_w1 = self.load_fused_expert_weights(
|
||||
name_mapped,
|
||||
params_dict,
|
||||
loaded_weight[0],
|
||||
"w1",
|
||||
num_experts,
|
||||
)
|
||||
success_w3 = self.load_fused_expert_weights(
|
||||
name_mapped,
|
||||
params_dict,
|
||||
loaded_weight[1],
|
||||
"w3",
|
||||
num_experts,
|
||||
)
|
||||
success = success_w1 and success_w3
|
||||
else:
|
||||
# down_proj
|
||||
success = self.load_fused_expert_weights(
|
||||
name_mapped,
|
||||
params_dict,
|
||||
loaded_weight,
|
||||
shard_id,
|
||||
num_experts,
|
||||
)
|
||||
else:
|
||||
# Skip loading extra parameters for GPTQ/modelopt models
|
||||
if (
|
||||
name_mapped.endswith(ignore_suffixes)
|
||||
and name_mapped not in params_dict
|
||||
):
|
||||
continue
|
||||
param = params_dict[name_mapped]
|
||||
# We should ask the weight loader to return success or
|
||||
# not here since otherwise we may skip experts with
|
||||
# other available replicas.
|
||||
weight_loader = typing.cast(
|
||||
Callable[..., bool], param.weight_loader
|
||||
)
|
||||
success = weight_loader(
|
||||
param,
|
||||
loaded_weight,
|
||||
name_mapped,
|
||||
shard_id=shard_id,
|
||||
expert_id=expert_id,
|
||||
return_success=True,
|
||||
)
|
||||
if success:
|
||||
name = name_mapped
|
||||
break
|
||||
else:
|
||||
if is_expert_weight:
|
||||
# We've checked that this is an expert weight
|
||||
# However it's not mapped locally to this rank
|
||||
# So we simply skip it
|
||||
continue
|
||||
# Skip loading extra parameters for GPTQ/modelopt models.
|
||||
if name.endswith(ignore_suffixes) and name not in params_dict:
|
||||
continue
|
||||
# Skip layers on other devices.
|
||||
if is_pp_missing_parameter(name, self):
|
||||
continue
|
||||
if name not in params_dict:
|
||||
continue
|
||||
param = params_dict[name]
|
||||
weight_loader = getattr(
|
||||
param, "weight_loader", default_weight_loader
|
||||
)
|
||||
weight_loader(param, loaded_weight)
|
||||
loaded_params.add(name)
|
||||
return loaded_params
|
||||
|
||||
|
||||
class Qwen3MoeLLMForCausalLM(Qwen3MoeForCausalLM):
|
||||
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
|
||||
|
||||
@@ -33,10 +33,6 @@ from vllm.model_executor.layers.linear import (
|
||||
from vllm.model_executor.layers.pooler import DispatchPooler
|
||||
from vllm.model_executor.layers.quantization import QuantizationConfig
|
||||
from vllm.model_executor.layers.vocab_parallel_embedding import VocabParallelEmbedding
|
||||
from vllm.model_executor.model_loader.weight_utils import (
|
||||
default_weight_loader,
|
||||
maybe_remap_kv_scale_name,
|
||||
)
|
||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
from vllm.multimodal.inputs import (
|
||||
MultiModalFieldConfig,
|
||||
@@ -62,7 +58,7 @@ from vllm.utils.tensor_schema import TensorSchema, TensorShape
|
||||
|
||||
from .interfaces import MultiModalEmbeddings, SupportsMultiModal, SupportsQuant
|
||||
from .interfaces_base import default_pooling_type
|
||||
from .utils import AutoWeightsLoader, maybe_prefix
|
||||
from .utils import AutoWeightsLoader, WeightsMapper, maybe_prefix
|
||||
from .vision import (
|
||||
VisionEncoderInfo,
|
||||
VisionFeatureSelectStrategy,
|
||||
@@ -571,6 +567,14 @@ class SiglipEncoder(nn.Module):
|
||||
|
||||
|
||||
class SiglipTextTransformer(nn.Module):
|
||||
hf_to_vllm_mapper = WeightsMapper(
|
||||
orig_to_new_stacked={
|
||||
".q_proj": (".qkv_proj", "q"),
|
||||
".k_proj": (".qkv_proj", "k"),
|
||||
".v_proj": (".qkv_proj", "v"),
|
||||
}
|
||||
)
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
config: SiglipTextConfig,
|
||||
@@ -615,30 +619,8 @@ class SiglipTextTransformer(nn.Module):
|
||||
return last_hidden_state
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
stacked_params_mapping = [
|
||||
# (param_name, shard_name, shard_id)
|
||||
("qkv_proj", "q_proj", "q"),
|
||||
("qkv_proj", "k_proj", "k"),
|
||||
("qkv_proj", "v_proj", "v"),
|
||||
]
|
||||
params_dict = dict(self.named_parameters())
|
||||
loaded_params: set[str] = set()
|
||||
|
||||
for name, loaded_weight in weights:
|
||||
for param_name, weight_name, shard_id in stacked_params_mapping:
|
||||
if weight_name not in name:
|
||||
continue
|
||||
name = name.replace(weight_name, param_name)
|
||||
param = params_dict[name]
|
||||
weight_loader = param.weight_loader
|
||||
weight_loader(param, loaded_weight, shard_id)
|
||||
break
|
||||
else:
|
||||
param = params_dict[name]
|
||||
weight_loader = getattr(param, "weight_loader", default_weight_loader)
|
||||
weight_loader(param, loaded_weight)
|
||||
loaded_params.add(name)
|
||||
return loaded_params
|
||||
loader = AutoWeightsLoader(self)
|
||||
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
|
||||
|
||||
|
||||
class SiglipMultiheadAttentionPoolingHead(nn.Module):
|
||||
@@ -682,6 +664,14 @@ class SiglipMultiheadAttentionPoolingHead(nn.Module):
|
||||
|
||||
|
||||
class SiglipVisionTransformer(nn.Module):
|
||||
hf_to_vllm_mapper = WeightsMapper(
|
||||
orig_to_new_stacked={
|
||||
".q_proj": (".qkv_proj", "q"),
|
||||
".k_proj": (".qkv_proj", "k"),
|
||||
".v_proj": (".qkv_proj", "v"),
|
||||
}
|
||||
)
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
config: SiglipVisionConfig,
|
||||
@@ -806,46 +796,25 @@ class SiglipVisionTransformer(nn.Module):
|
||||
return encoder_outputs
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
stacked_params_mapping = [
|
||||
# (param_name, shard_name, shard_id)
|
||||
("qkv_proj", "q_proj", "q"),
|
||||
("qkv_proj", "k_proj", "k"),
|
||||
("qkv_proj", "v_proj", "v"),
|
||||
]
|
||||
params_dict = dict(self.named_parameters())
|
||||
loaded_params: set[str] = set()
|
||||
skip_prefixes = []
|
||||
if self.post_layernorm is None:
|
||||
skip_prefixes.append("post_layernorm.")
|
||||
if self.head is None:
|
||||
skip_prefixes.append("head.")
|
||||
loader = AutoWeightsLoader(self, skip_prefixes=skip_prefixes)
|
||||
|
||||
layer_count = len(self.encoder.layers)
|
||||
|
||||
for name, loaded_weight in weights:
|
||||
# post_layernorm is not needed in SiglipVisionTransformer
|
||||
if name.startswith("post_layernorm") and self.post_layernorm is None:
|
||||
continue
|
||||
|
||||
# if the model configuration is not going to use
|
||||
# the pooling head for inference, don't load its weights
|
||||
if self.head is None and name.startswith("head"):
|
||||
continue
|
||||
|
||||
# omit layers when num_hidden_layers_override is set
|
||||
if name.startswith("encoder.layers"):
|
||||
layer_idx = int(name.split(".")[2])
|
||||
if layer_idx >= layer_count:
|
||||
def _filter(ws):
|
||||
for name, w in ws:
|
||||
# omit layers when num_hidden_layers_override is set
|
||||
if name.startswith("encoder.layers.") and (
|
||||
int(name.split(".")[2]) >= layer_count
|
||||
):
|
||||
continue
|
||||
yield name, w
|
||||
|
||||
for param_name, weight_name, shard_id in stacked_params_mapping:
|
||||
if weight_name not in name:
|
||||
continue
|
||||
name = name.replace(weight_name, param_name)
|
||||
param = params_dict[name]
|
||||
weight_loader = param.weight_loader
|
||||
weight_loader(param, loaded_weight, shard_id)
|
||||
break
|
||||
else:
|
||||
param = params_dict[name]
|
||||
weight_loader = getattr(param, "weight_loader", default_weight_loader)
|
||||
weight_loader(param, loaded_weight)
|
||||
loaded_params.add(name)
|
||||
return loaded_params
|
||||
return loader.load_weights(_filter(weights), mapper=self.hf_to_vllm_mapper)
|
||||
|
||||
|
||||
class SiglipVisionModel(nn.Module):
|
||||
@@ -896,67 +865,6 @@ class SiglipVisionModel(nn.Module):
|
||||
feature_select_strategy=feature_select_strategy,
|
||||
)
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
stacked_params_mapping = [
|
||||
# (param_name, shard_name, shard_id)
|
||||
("qkv_proj", "q_proj", "q"),
|
||||
("qkv_proj", "k_proj", "k"),
|
||||
("qkv_proj", "v_proj", "v"),
|
||||
]
|
||||
params_dict = dict(self.named_parameters())
|
||||
loaded_params: set[str] = set()
|
||||
layer_count = len(self.vision_model.encoder.layers)
|
||||
|
||||
for name, loaded_weight in weights:
|
||||
# post_layernorm is optional in SiglipVisionModel
|
||||
if (
|
||||
name.startswith("vision_model.post_layernorm")
|
||||
and self.vision_model.post_layernorm is None
|
||||
):
|
||||
continue
|
||||
|
||||
# if the model configuration is not going to use
|
||||
# the pooling head for inference, don't load its weights
|
||||
if self.vision_model.head is None and name.startswith("vision_model.head"):
|
||||
continue
|
||||
|
||||
# omit layers when num_hidden_layers_override is set
|
||||
if name.startswith("vision_model.encoder.layers"):
|
||||
layer_idx = int(name.split(".")[3])
|
||||
if layer_idx >= layer_count:
|
||||
continue
|
||||
|
||||
# Check if this is a scale parameter that needs remapping first
|
||||
if name.endswith((".k_scale", ".v_scale", ".q_scale", ".prob_scale")):
|
||||
# Try to remap the scale name first
|
||||
remapped_name = maybe_remap_kv_scale_name(name, params_dict)
|
||||
if remapped_name is not None and remapped_name in params_dict:
|
||||
# Successfully remapped, use the remapped name
|
||||
param = params_dict[remapped_name]
|
||||
weight_loader = getattr(
|
||||
param, "weight_loader", default_weight_loader
|
||||
)
|
||||
weight_loader(param, loaded_weight)
|
||||
loaded_params.add(remapped_name)
|
||||
continue
|
||||
# If remapping failed, continue with normal processing
|
||||
|
||||
for param_name, weight_name, shard_id in stacked_params_mapping:
|
||||
if weight_name not in name:
|
||||
continue
|
||||
name = name.replace(weight_name, param_name)
|
||||
|
||||
param = params_dict[name]
|
||||
weight_loader = param.weight_loader
|
||||
weight_loader(param, loaded_weight, shard_id)
|
||||
break
|
||||
else:
|
||||
param = params_dict[name]
|
||||
weight_loader = getattr(param, "weight_loader", default_weight_loader)
|
||||
weight_loader(param, loaded_weight)
|
||||
loaded_params.add(name)
|
||||
return loaded_params
|
||||
|
||||
|
||||
# Adapted from: https://github.com/huggingface/transformers/blob/v4.54.1/src/transformers/models/siglip/modeling_siglip.py#L200
|
||||
class SiglipTextEmbeddings(nn.Module):
|
||||
|
||||
@@ -26,10 +26,9 @@ from vllm.model_executor.layers.quantization import QuantizationConfig
|
||||
from vllm.model_executor.layers.rotary_embedding.common import (
|
||||
ApplyRotaryEmb,
|
||||
)
|
||||
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
from .utils import maybe_prefix
|
||||
from .utils import AutoWeightsLoader, WeightsMapper, maybe_prefix
|
||||
from .vision import is_vit_use_data_parallel
|
||||
|
||||
|
||||
@@ -588,6 +587,14 @@ class Siglip2VisionTransformer(nn.Module):
|
||||
|
||||
|
||||
class Siglip2NavitModel(torch.nn.Module):
|
||||
hf_to_vllm_mapper = WeightsMapper(
|
||||
orig_to_new_stacked={
|
||||
".q_proj": (".qkv_proj", "q"),
|
||||
".k_proj": (".qkv_proj", "k"),
|
||||
".v_proj": (".qkv_proj", "v"),
|
||||
}
|
||||
)
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
config: Siglip2VisionConfig,
|
||||
@@ -613,28 +620,5 @@ class Siglip2NavitModel(torch.nn.Module):
|
||||
)
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
stacked_params_mapping = [
|
||||
# (param_name, shard_name, shard_id)
|
||||
("qkv_proj", "q_proj", "q"),
|
||||
("qkv_proj", "k_proj", "k"),
|
||||
("qkv_proj", "v_proj", "v"),
|
||||
]
|
||||
params_dict = dict(self.named_parameters())
|
||||
loaded_params: set[str] = set()
|
||||
|
||||
for name, loaded_weight in weights:
|
||||
for param_name, weight_name, shard_id in stacked_params_mapping:
|
||||
if weight_name not in name:
|
||||
continue
|
||||
name = name.replace(weight_name, param_name)
|
||||
|
||||
param = params_dict[name]
|
||||
weight_loader = param.weight_loader
|
||||
weight_loader(param, loaded_weight, shard_id)
|
||||
break
|
||||
else:
|
||||
param = params_dict[name]
|
||||
weight_loader = getattr(param, "weight_loader", default_weight_loader)
|
||||
weight_loader(param, loaded_weight)
|
||||
loaded_params.add(name)
|
||||
return loaded_params
|
||||
loader = AutoWeightsLoader(self)
|
||||
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
|
||||
|
||||
@@ -16,7 +16,6 @@
|
||||
# limitations under the License.
|
||||
"""Transformers modeling backend mixin for Mixture of Experts (MoE) models."""
|
||||
|
||||
from collections.abc import Iterable
|
||||
from dataclasses import dataclass
|
||||
from functools import partial
|
||||
from typing import TYPE_CHECKING, Any
|
||||
@@ -82,11 +81,6 @@ class TransformersMoERunner(MoERunner):
|
||||
) -> torch.Tensor:
|
||||
return super().forward(hidden_states, topk_weights)
|
||||
|
||||
def load_weights(
|
||||
self, weights: Iterable[tuple[str, torch.Tensor]]
|
||||
) -> Iterable[str]:
|
||||
return self.routed_experts.load_weights(weights)
|
||||
|
||||
|
||||
def _transformers_moe_forward(
|
||||
hidden_states: torch.Tensor,
|
||||
@@ -119,6 +113,20 @@ direct_register_custom_op(
|
||||
)
|
||||
|
||||
|
||||
class TransformersRoutedExperts(RoutedExperts):
|
||||
def get_expert_mapping(
|
||||
self, include_fused: bool = False
|
||||
) -> list[tuple[str, str, int, str]]:
|
||||
common_names = ("gate_proj", "down_proj", "up_proj")
|
||||
common_map = super().get_expert_mapping(*common_names, include_fused)
|
||||
mixtral_map = super().get_expert_mapping("w1", "w2", "w3", include_fused)
|
||||
if not include_fused:
|
||||
return common_map + mixtral_map
|
||||
common_fused, common_unfused = common_map[:3], common_map[3:]
|
||||
mixtral_fused, mixtral_unfused = mixtral_map[:3], mixtral_map[3:]
|
||||
return common_fused + mixtral_fused + common_unfused + mixtral_unfused
|
||||
|
||||
|
||||
class MoEMixin(MixtureOfExperts):
|
||||
def __init__(self, *, vllm_config: "VllmConfig", prefix: str = ""):
|
||||
self.check_version("5.0.0", "MoE models support")
|
||||
@@ -140,43 +148,6 @@ class MoEMixin(MixtureOfExperts):
|
||||
mlp.n_redundant_experts = self.num_redundant_experts
|
||||
mlp.experts.update_expert_map()
|
||||
|
||||
def get_expert_mapping(self) -> list[tuple[str, str, int, str]]:
|
||||
"""
|
||||
Params for weights, fp8 weight scales, fp8 activation scales
|
||||
(param_name, weight_name, expert_id, shard_id)
|
||||
"""
|
||||
# Models saved with fused experts. These are checkpoints released:
|
||||
# - After Transformers v5
|
||||
# - Before Transformers v5, but re-saved with save_original_format=False
|
||||
# In the fused experts case, we repurpose the expert_id as shard_idx for
|
||||
# deconcatenating w1 and w3 in FusedMoE.load_weights.
|
||||
expert_mapping = [
|
||||
("experts.w13_weight", "experts.gate_up_proj", 0, "w1"),
|
||||
("experts.w13_weight", "experts.gate_up_proj", 1, "w3"),
|
||||
("experts.w2_weight", "experts.down_proj", 0, "w2"),
|
||||
]
|
||||
# Models saved with ModuleList experts
|
||||
ckpt_names = [
|
||||
# (ckpt_gate_proj_name, ckpt_down_proj_name, ckpt_up_proj_name)
|
||||
("gate_proj", "down_proj", "up_proj"), # Most common MoE style
|
||||
("w1", "w2", "w3"), # Granite, Mixtral, Phi MoE style
|
||||
]
|
||||
num_experts = self.model_config.get_num_experts()
|
||||
num_redundant_experts = self.parallel_config.eplb_config.num_redundant_experts
|
||||
for gate_proj, down_proj, up_proj in ckpt_names:
|
||||
expert_mapping.extend(
|
||||
RoutedExperts.make_expert_params_mapping(
|
||||
self,
|
||||
ckpt_gate_proj_name=gate_proj,
|
||||
ckpt_down_proj_name=down_proj,
|
||||
ckpt_up_proj_name=up_proj,
|
||||
num_experts=num_experts,
|
||||
num_redundant_experts=num_redundant_experts,
|
||||
routed_experts_prefix="",
|
||||
)
|
||||
)
|
||||
return expert_mapping
|
||||
|
||||
def recursive_replace(self):
|
||||
"""Initialize the MoE layers."""
|
||||
text_config = self.text_config
|
||||
@@ -218,9 +189,6 @@ class MoEMixin(MixtureOfExperts):
|
||||
if "gptoss" in wrapped_arch:
|
||||
activation = "swigluoai"
|
||||
|
||||
# Expert mapping for `AutoWeightsLoader`
|
||||
expert_mapping = self.get_expert_mapping()
|
||||
|
||||
# Expert parallel load balancing kwargs
|
||||
enable_eplb = self.parallel_config.enable_eplb
|
||||
num_redundant_experts = self.parallel_config.eplb_config.num_redundant_experts
|
||||
@@ -308,12 +276,12 @@ class MoEMixin(MixtureOfExperts):
|
||||
enable_eplb=enable_eplb,
|
||||
num_redundant_experts=num_redundant_experts,
|
||||
has_bias=has_bias,
|
||||
expert_mapping=expert_mapping,
|
||||
custom_routing_function=partial(
|
||||
custom_routing_function,
|
||||
moe_state=moe_state,
|
||||
),
|
||||
runner_cls=TransformersMoERunner,
|
||||
routed_experts_cls=TransformersRoutedExperts,
|
||||
runner_args={"moe_state": moe_state},
|
||||
)
|
||||
mlp.experts = fused_experts
|
||||
|
||||
@@ -43,7 +43,6 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
|
||||
ParallelLMHead,
|
||||
VocabParallelEmbedding,
|
||||
)
|
||||
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
|
||||
from vllm.sequence import IntermediateTensors
|
||||
|
||||
from .interfaces import HasInnerState, IsHybrid, SupportsMambaPrefixCaching
|
||||
@@ -804,34 +803,6 @@ class Zamba2Model(nn.Module):
|
||||
hidden_states = self.final_layernorm(hidden_states)
|
||||
return hidden_states
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
stacked_params_mapping = [
|
||||
# (param_name, shard_name, shard_id)
|
||||
("qkv_proj", "q_proj", "q"),
|
||||
("qkv_proj", "k_proj", "k"),
|
||||
("qkv_proj", "v_proj", "v"),
|
||||
]
|
||||
|
||||
params_dict = dict(self.named_parameters())
|
||||
loaded_params: set[str] = set()
|
||||
for chkpt_weight_name, loaded_weight in weights:
|
||||
for param_name, weight_name, shard_id in stacked_params_mapping:
|
||||
if weight_name not in chkpt_weight_name:
|
||||
continue
|
||||
chkpt_weight_name = chkpt_weight_name.replace(weight_name, param_name)
|
||||
param = params_dict[chkpt_weight_name]
|
||||
weight_loader = param.weight_loader
|
||||
weight_loader(param, loaded_weight, shard_id)
|
||||
break
|
||||
else:
|
||||
if chkpt_weight_name not in params_dict:
|
||||
continue
|
||||
param = params_dict[chkpt_weight_name]
|
||||
weight_loader = getattr(param, "weight_loader", default_weight_loader)
|
||||
weight_loader(param, loaded_weight)
|
||||
loaded_params.add(chkpt_weight_name)
|
||||
return loaded_params
|
||||
|
||||
|
||||
class Zamba2ForCausalLM(nn.Module, HasInnerState, IsHybrid, SupportsMambaPrefixCaching):
|
||||
"""Zamba2 model with causal language modeling head.
|
||||
@@ -849,7 +820,12 @@ class Zamba2ForCausalLM(nn.Module, HasInnerState, IsHybrid, SupportsMambaPrefixC
|
||||
"A_log": "A",
|
||||
"0.weight": "A.weight",
|
||||
"1.weight": "B.weight",
|
||||
}
|
||||
},
|
||||
orig_to_new_stacked={
|
||||
".self_attn.q_proj": (".self_attn.qkv_proj", "q"),
|
||||
".self_attn.k_proj": (".self_attn.qkv_proj", "k"),
|
||||
".self_attn.v_proj": (".self_attn.qkv_proj", "v"),
|
||||
},
|
||||
)
|
||||
|
||||
@classmethod
|
||||
|
||||
@@ -124,15 +124,16 @@ def get_packed_modules_mapping(model: torch.nn.Module) -> dict[str, list[str]]:
|
||||
def get_moe_expert_mapping(
|
||||
model: torch.nn.Module,
|
||||
) -> list[tuple[str, str, int, str]]:
|
||||
if parent_map := getattr(model, "get_expert_mapping", None):
|
||||
return parent_map()
|
||||
else:
|
||||
# We only check main components instead of whole model submodules
|
||||
for child in model.children():
|
||||
child_map = getattr(child, "get_expert_mapping", None)
|
||||
if child_map is not None:
|
||||
return child_map()
|
||||
return []
|
||||
"""Get the expert mapping from a model.
|
||||
|
||||
It will be retrieved from the first module that has a `get_expert_mapping` method.
|
||||
If the model manually implements `get_expert_mapping`, it will be used.
|
||||
Otherwise, it will use the first RoutedExperts layer."""
|
||||
for _, module in model.named_modules():
|
||||
get_mapping = getattr(module, "get_expert_mapping", None)
|
||||
if get_mapping is not None:
|
||||
return get_mapping()
|
||||
raise ValueError("No module in the model has a `get_expert_mapping` method.")
|
||||
|
||||
|
||||
def maybe_disable_graph_partition(current_backend: str) -> dict[str, bool]:
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import typing
|
||||
from collections.abc import Callable, Iterable, MutableSequence, Sequence
|
||||
from collections.abc import Callable, Iterable
|
||||
from itertools import islice
|
||||
|
||||
import regex as re
|
||||
@@ -1365,7 +1365,6 @@ class DeepseekV4ForCausalLM(nn.Module, SupportsPP, DeepseekV4MixtureOfExperts):
|
||||
self.set_moe_parameters()
|
||||
|
||||
def set_moe_parameters(self) -> None:
|
||||
self.expert_weights: MutableSequence[Sequence[torch.Tensor]] = []
|
||||
self.num_expert_groups = getattr(self.config, "n_group", 1)
|
||||
self.num_moe_layers = self.config.num_hidden_layers
|
||||
self.moe_layers: list[nn.Module] = []
|
||||
|
||||
Reference in New Issue
Block a user