Remove even more unnecessary load_weights methods (#48496)
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
This commit is contained in:
@@ -79,6 +79,7 @@ from .interfaces import MixtureOfExperts, SupportsEagle, SupportsLoRA, SupportsP
|
||||
from .utils import (
|
||||
AutoWeightsLoader,
|
||||
PPMissingLayer,
|
||||
get_spec_layer_idx_from_weight_name,
|
||||
is_pp_missing_parameter,
|
||||
make_empty_intermediate_tensors_factory,
|
||||
make_layers,
|
||||
@@ -1143,16 +1144,3 @@ class AXK1ForCausalLM(
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
loader = AutoWeightsLoader(self)
|
||||
return loader.load_weights(weights)
|
||||
|
||||
|
||||
def get_spec_layer_idx_from_weight_name(
|
||||
config: AXK1Config, weight_name: str
|
||||
) -> int | None:
|
||||
if config.num_nextn_predict_layers and config.num_nextn_predict_layers > 0:
|
||||
layer_idx = config.num_hidden_layers
|
||||
for i in range(config.num_nextn_predict_layers):
|
||||
if weight_name.startswith(
|
||||
f"model.layers.{layer_idx + i}."
|
||||
) or weight_name.startswith(f"layers.{layer_idx + i}."):
|
||||
return layer_idx + i
|
||||
return None
|
||||
|
||||
@@ -11,21 +11,13 @@ from transformers.models.aria.processing_aria import AriaProcessor
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.config.multimodal import BaseDummyOptions
|
||||
from vllm.distributed import get_tensor_model_parallel_rank
|
||||
from vllm.inputs import MultiModalDataDict
|
||||
from vllm.model_executor.layers.activation import get_act_fn
|
||||
from vllm.model_executor.layers.fused_moe import (
|
||||
FusedMoE,
|
||||
RoutedExperts,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe import FusedMoE
|
||||
from vllm.model_executor.layers.linear import ColumnParallelLinear, RowParallelLinear
|
||||
from vllm.model_executor.layers.logits_processor import LogitsProcessor
|
||||
from vllm.model_executor.layers.quantization import QuantizationConfig
|
||||
from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
|
||||
from vllm.model_executor.model_loader.weight_utils import (
|
||||
default_weight_loader,
|
||||
maybe_remap_kv_scale_name,
|
||||
)
|
||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
from vllm.multimodal.inputs import (
|
||||
MultiModalFieldConfig,
|
||||
@@ -51,7 +43,6 @@ from .llama import LlamaDecoderLayer, LlamaMLP, LlamaModel
|
||||
from .utils import (
|
||||
AutoWeightsLoader,
|
||||
WeightsMapper,
|
||||
is_pp_missing_parameter,
|
||||
maybe_prefix,
|
||||
)
|
||||
|
||||
@@ -94,34 +85,18 @@ class AriaVisionTransformer(Idefics3VisionTransformer, SupportsQuant):
|
||||
# Identity layer
|
||||
self.post_layernorm = nn.Identity()
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
stacked_params_mapping = [
|
||||
# (param_name, shard_name, shard_id)
|
||||
("qkv_proj", "q_proj", "q"),
|
||||
("qkv_proj", "k_proj", "k"),
|
||||
("qkv_proj", "v_proj", "v"),
|
||||
]
|
||||
params_dict = dict(self.named_parameters())
|
||||
loaded_params: set[str] = set()
|
||||
for name, loaded_weight in weights:
|
||||
# NOTE: post_layernorm is not used in Aria
|
||||
if "post_layernorm" in name:
|
||||
continue
|
||||
hf_to_vllm_mapper = WeightsMapper(
|
||||
orig_to_new_stacked={
|
||||
".q_proj": (".qkv_proj", "q"),
|
||||
".k_proj": (".qkv_proj", "k"),
|
||||
".v_proj": (".qkv_proj", "v"),
|
||||
}
|
||||
)
|
||||
|
||||
for param_name, weight_name, shard_id in stacked_params_mapping:
|
||||
if weight_name not in name:
|
||||
continue
|
||||
name = name.replace(weight_name, param_name)
|
||||
param = params_dict[name]
|
||||
weight_loader = param.weight_loader
|
||||
weight_loader(param, loaded_weight, shard_id)
|
||||
break
|
||||
else:
|
||||
param = params_dict[name]
|
||||
weight_loader = getattr(param, "weight_loader", default_weight_loader)
|
||||
weight_loader(param, loaded_weight)
|
||||
loaded_params.add(name)
|
||||
return loaded_params
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
# NOTE: post_layernorm is not used in Aria.
|
||||
loader = AutoWeightsLoader(self, skip_substrs=["post_layernorm"])
|
||||
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
|
||||
|
||||
|
||||
class AriaProjectorMLP(nn.Module):
|
||||
@@ -217,39 +192,6 @@ class AriaProjector(nn.Module):
|
||||
return out
|
||||
|
||||
|
||||
class AriaRoutedExperts(RoutedExperts):
|
||||
def weight_loader(
|
||||
self, param: nn.Parameter, loaded_weight: torch.Tensor, shard_id: str
|
||||
) -> None:
|
||||
# Override the weight_loader to handle the expert weights in the Aria
|
||||
# model, which are already packed with experts, and merge the gate and
|
||||
# up weights for each expert.
|
||||
# Note: Loading expert weights with quantization is not supported
|
||||
tp_rank = get_tensor_model_parallel_rank()
|
||||
tp_size = self.moe_config.tp_size
|
||||
if shard_id == "w13":
|
||||
# the shape of loaded_weight is
|
||||
# (num_experts, hidden_size, 2 * moe_intermediate_size)
|
||||
if tp_size > 1:
|
||||
up, gate = loaded_weight.chunk(2, dim=-1)
|
||||
up_current_rank = up.chunk(tp_size, dim=-1)[tp_rank]
|
||||
gate_current_rank = gate.chunk(tp_size, dim=-1)[tp_rank]
|
||||
up_and_gate = torch.cat(
|
||||
[up_current_rank, gate_current_rank], dim=-1
|
||||
).transpose(1, 2)
|
||||
param.data.copy_(up_and_gate)
|
||||
else:
|
||||
param.data.copy_(loaded_weight.transpose(1, 2))
|
||||
elif shard_id == "w2":
|
||||
# the shape of loaded_weight is
|
||||
# (num_experts, moe_intermediate_size, hidden_size)
|
||||
if tp_size > 1:
|
||||
down_current_rank = loaded_weight.chunk(tp_size, dim=1)[tp_rank]
|
||||
param.data.copy_(down_current_rank.transpose(1, 2))
|
||||
else:
|
||||
param.data.copy_(loaded_weight.transpose(1, 2))
|
||||
|
||||
|
||||
class AriaTextMoELayer(nn.Module):
|
||||
"""
|
||||
Mixture of Experts (MoE) Layer for the AriaMoE model.
|
||||
@@ -288,7 +230,6 @@ class AriaTextMoELayer(nn.Module):
|
||||
intermediate_size=config.intermediate_size,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.experts",
|
||||
routed_experts_cls=AriaRoutedExperts,
|
||||
)
|
||||
|
||||
def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
|
||||
@@ -335,70 +276,23 @@ class AriaTextModel(LlamaModel, SupportsQuant):
|
||||
packed_modules_mapping = {
|
||||
"qkv_proj": ["q_proj", "k_proj", "v_proj"],
|
||||
"gate_up_proj": ["gate_proj", "up_proj"],
|
||||
"experts.routed_experts.w13_weight": ["experts.fc1.weight"],
|
||||
"experts.routed_experts.w2_weight": ["experts.fc2.weight"],
|
||||
}
|
||||
|
||||
# Aria packs all experts into single (transposed) fc1/fc2 tensors, which is
|
||||
# exactly the pre-fused checkpoint layout FusedMoE self-loads once fc1/fc2
|
||||
# are renamed to the fused gate_up_proj/down_proj names.
|
||||
hf_to_vllm_mapper = LlamaModel.hf_to_vllm_mapper | WeightsMapper(
|
||||
orig_to_new_substr={
|
||||
"experts.fc1": "experts.gate_up_proj",
|
||||
"experts.fc2": "experts.down_proj",
|
||||
}
|
||||
)
|
||||
|
||||
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
|
||||
super().__init__(
|
||||
vllm_config=vllm_config, prefix=prefix, layer_type=AriaTextDecoderLayer
|
||||
)
|
||||
|
||||
# Adapted from LlamaModel.load_weights with the modification of adding
|
||||
# the expert weights mapping to `stacked_params_mapping`
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
stacked_params_mapping = [
|
||||
# (param_name, shard_name, shard_id)
|
||||
(".qkv_proj", ".q_proj", "q"),
|
||||
(".qkv_proj", ".k_proj", "k"),
|
||||
(".qkv_proj", ".v_proj", "v"),
|
||||
(".gate_up_proj", ".gate_proj", 0),
|
||||
(".gate_up_proj", ".up_proj", 1),
|
||||
("experts.routed_experts.w13_weight", "experts.fc1.weight", "w13"),
|
||||
("experts.routed_experts.w2_weight", "experts.fc2.weight", "w2"),
|
||||
]
|
||||
params_dict = dict(self.named_parameters())
|
||||
loaded_params: set[str] = set()
|
||||
for name, loaded_weight in weights:
|
||||
if "rotary_emb.inv_freq" in name:
|
||||
continue
|
||||
if "rotary_emb.cos_cached" in name or "rotary_emb.sin_cached" in name:
|
||||
# Models trained using ColossalAI may include these tensors in
|
||||
# the checkpoint. Skip them.
|
||||
continue
|
||||
for param_name, weight_name, shard_id in stacked_params_mapping:
|
||||
if weight_name not in name:
|
||||
continue
|
||||
name = name.replace(weight_name, param_name)
|
||||
# Skip loading extra bias for GPTQ models.
|
||||
if name.endswith(".bias") and name not in params_dict:
|
||||
continue
|
||||
|
||||
if is_pp_missing_parameter(name, self):
|
||||
continue
|
||||
|
||||
param = params_dict[name]
|
||||
weight_loader = param.weight_loader
|
||||
weight_loader(param, loaded_weight, shard_id)
|
||||
break
|
||||
else:
|
||||
# Skip loading extra bias for GPTQ models.
|
||||
if name.endswith(".bias") and name not in params_dict:
|
||||
continue
|
||||
# Remapping the name of FP8 kv-scale.
|
||||
name = maybe_remap_kv_scale_name(name, params_dict)
|
||||
if name is None:
|
||||
continue
|
||||
|
||||
if is_pp_missing_parameter(name, self):
|
||||
continue
|
||||
|
||||
param = params_dict[name]
|
||||
weight_loader = getattr(param, "weight_loader", default_weight_loader)
|
||||
weight_loader(param, loaded_weight)
|
||||
loaded_params.add(name)
|
||||
return loaded_params
|
||||
|
||||
|
||||
class AriaProcessingInfo(BaseProcessingInfo):
|
||||
def get_hf_config(self):
|
||||
|
||||
@@ -41,10 +41,7 @@ from vllm.distributed import (
|
||||
)
|
||||
from vllm.model_executor.layers.activation import SiluAndMul
|
||||
from vllm.model_executor.layers.attention import Attention
|
||||
from vllm.model_executor.layers.fused_moe import (
|
||||
FusedMoE,
|
||||
fused_moe_make_expert_params_mapping,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe import FusedMoE
|
||||
from vllm.model_executor.layers.layernorm import RMSNorm
|
||||
from vllm.model_executor.layers.linear import (
|
||||
MergedColumnParallelLinear,
|
||||
@@ -58,14 +55,13 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
|
||||
ParallelLMHead,
|
||||
VocabParallelEmbedding,
|
||||
)
|
||||
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
|
||||
from vllm.sequence import IntermediateTensors
|
||||
|
||||
from .interfaces import SupportsLoRA, SupportsPP
|
||||
from .utils import (
|
||||
AutoWeightsLoader,
|
||||
PPMissingLayer,
|
||||
is_pp_missing_parameter,
|
||||
WeightsMapper,
|
||||
make_empty_intermediate_tensors_factory,
|
||||
make_layers,
|
||||
maybe_prefix,
|
||||
@@ -379,6 +375,16 @@ class BailingMoeBlock(nn.Module):
|
||||
|
||||
@support_torch_compile
|
||||
class BailingMoeModel(nn.Module):
|
||||
hf_to_vllm_mapper = WeightsMapper(
|
||||
orig_to_new_stacked={
|
||||
# .experts.gate_up_proj must be handled by MoERunner.load_weights for EP
|
||||
".mlp.gate_proj": (".mlp.gate_up_proj", 0),
|
||||
".mlp.up_proj": (".mlp.gate_up_proj", 1),
|
||||
".shared_experts.gate_proj": (".shared_experts.gate_up_proj", 0),
|
||||
".shared_experts.up_proj": (".shared_experts.gate_up_proj", 1),
|
||||
}
|
||||
)
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
@@ -468,89 +474,9 @@ class BailingMoeModel(nn.Module):
|
||||
hidden_states, _ = self.norm(hidden_states, residual)
|
||||
return hidden_states
|
||||
|
||||
def get_expert_mapping(self) -> list[tuple[str, str, int, str]]:
|
||||
return fused_moe_make_expert_params_mapping(
|
||||
self,
|
||||
ckpt_gate_proj_name="gate_proj",
|
||||
ckpt_down_proj_name="down_proj",
|
||||
ckpt_up_proj_name="up_proj",
|
||||
num_experts=self.config.num_experts,
|
||||
)
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
stacked_params_mapping = [
|
||||
# (param_name, shard_name, shard_id)
|
||||
("gate_up_proj", "gate_proj", 0),
|
||||
("gate_up_proj", "up_proj", 1),
|
||||
]
|
||||
|
||||
params_dict = dict(self.named_parameters(remove_duplicate=False))
|
||||
loaded_params: set[str] = set()
|
||||
expert_params_mapping = self.get_expert_mapping()
|
||||
for name, loaded_weight in weights:
|
||||
if (
|
||||
hasattr(self.config, "norm_head")
|
||||
and self.config.norm_head
|
||||
and "lm_head.weight" in name
|
||||
):
|
||||
loaded_weight = F.normalize(loaded_weight, dim=0, p=2, eps=1e-7)
|
||||
|
||||
for param_name, weight_name, shard_id in stacked_params_mapping:
|
||||
if weight_name not in name:
|
||||
continue
|
||||
if "mlp.experts" in name:
|
||||
continue
|
||||
name = name.replace(weight_name, param_name)
|
||||
# Skip loading extra bias for GPTQ models.
|
||||
if name.endswith(".bias") and name not in params_dict:
|
||||
continue
|
||||
if name not in params_dict:
|
||||
continue
|
||||
|
||||
if is_pp_missing_parameter(name, self):
|
||||
continue
|
||||
|
||||
param = params_dict[name]
|
||||
weight_loader = param.weight_loader
|
||||
weight_loader(param, loaded_weight, shard_id)
|
||||
break
|
||||
else:
|
||||
for mapping in expert_params_mapping:
|
||||
param_name, weight_name, expert_id, shard_id = mapping
|
||||
if weight_name not in name:
|
||||
continue
|
||||
name = name.replace(weight_name, param_name)
|
||||
|
||||
if is_pp_missing_parameter(name, self):
|
||||
continue
|
||||
if name not in params_dict:
|
||||
continue
|
||||
param = params_dict[name]
|
||||
weight_loader = param.weight_loader
|
||||
weight_loader(
|
||||
param,
|
||||
loaded_weight,
|
||||
name,
|
||||
shard_id=shard_id,
|
||||
expert_id=expert_id,
|
||||
)
|
||||
break
|
||||
else:
|
||||
if name.endswith(".bias") and name not in params_dict:
|
||||
continue
|
||||
if name not in params_dict:
|
||||
continue
|
||||
|
||||
if is_pp_missing_parameter(name, self):
|
||||
continue
|
||||
|
||||
param = params_dict[name]
|
||||
weight_loader = getattr(
|
||||
param, "weight_loader", default_weight_loader
|
||||
)
|
||||
weight_loader(param, loaded_weight)
|
||||
loaded_params.add(name)
|
||||
return loaded_params
|
||||
loader = AutoWeightsLoader(self)
|
||||
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
|
||||
|
||||
|
||||
class BailingMoeForCausalLM(nn.Module, SupportsPP, SupportsLoRA):
|
||||
@@ -622,15 +548,21 @@ class BailingMoeForCausalLM(nn.Module, SupportsPP, SupportsLoRA):
|
||||
logits = self.logits_processor(self.lm_head, hidden_states)
|
||||
return logits
|
||||
|
||||
def _normalize_lm_head(
|
||||
self, weights: Iterable[tuple[str, torch.Tensor]]
|
||||
) -> Iterable[tuple[str, torch.Tensor]]:
|
||||
norm_head = getattr(self.config, "norm_head", False)
|
||||
for name, loaded_weight in weights:
|
||||
if norm_head and "lm_head.weight" in name:
|
||||
loaded_weight = F.normalize(loaded_weight, dim=0, p=2, eps=1e-7)
|
||||
yield name, loaded_weight
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
loader = AutoWeightsLoader(
|
||||
self,
|
||||
skip_prefixes=(["lm_head."] if self.tie_word_embeddings else None),
|
||||
)
|
||||
return loader.load_weights(weights)
|
||||
|
||||
def get_expert_mapping(self) -> list[tuple[str, str, int, str]]:
|
||||
return self.model.get_expert_mapping()
|
||||
return loader.load_weights(self._normalize_lm_head(weights))
|
||||
|
||||
|
||||
class BailingMoeV2ForCausalLM(BailingMoeForCausalLM):
|
||||
|
||||
@@ -47,13 +47,11 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
|
||||
ParallelLMHead,
|
||||
VocabParallelEmbedding,
|
||||
)
|
||||
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
|
||||
from vllm.sequence import IntermediateTensors
|
||||
|
||||
from .interfaces import SupportsPP, SupportsQuant
|
||||
from .utils import (
|
||||
AutoWeightsLoader,
|
||||
is_pp_missing_parameter,
|
||||
make_empty_intermediate_tensors_factory,
|
||||
make_layers,
|
||||
maybe_prefix,
|
||||
@@ -297,36 +295,23 @@ class BloomModel(nn.Module):
|
||||
hidden_states = self.ln_f(hidden_states)
|
||||
return hidden_states
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
params_dict = dict(self.named_parameters(remove_duplicate=False))
|
||||
loaded_params: set[str] = set()
|
||||
def _repack_qkv(
|
||||
self, weights: Iterable[tuple[str, torch.Tensor]]
|
||||
) -> Iterable[tuple[str, torch.Tensor]]:
|
||||
# BLOOM's fused QKV is laid out as (num_heads * 3 * head_size) on its
|
||||
# output dim (0), while vLLM expects (3 * num_heads * head_size).
|
||||
num_heads = self.config.num_attention_heads
|
||||
for name, loaded_weight in weights:
|
||||
if is_pp_missing_parameter(name, self):
|
||||
continue
|
||||
param = params_dict[name]
|
||||
|
||||
if "query_key_value" in name:
|
||||
# NOTE: BLOOM's fused QKV's output_dim has the shape of
|
||||
# (num_heads * 3 * head_size), while the
|
||||
# required shape is (3 * num_heads * head_size).
|
||||
# Thus, we need weight conversion.
|
||||
output_dim = getattr(param, "output_dim", None)
|
||||
num_heads = self.config.num_attention_heads
|
||||
if output_dim is not None:
|
||||
loaded_weight_shape = loaded_weight.shape
|
||||
loaded_weight = loaded_weight.view(
|
||||
loaded_weight_shape[:output_dim]
|
||||
+ (num_heads, 3, -1)
|
||||
+ loaded_weight_shape[output_dim + 1 :]
|
||||
)
|
||||
loaded_weight = loaded_weight.transpose(output_dim, output_dim + 1)
|
||||
loaded_weight = loaded_weight.reshape(loaded_weight_shape)
|
||||
shape = loaded_weight.shape
|
||||
loaded_weight = loaded_weight.view((num_heads, 3, -1) + shape[1:])
|
||||
loaded_weight = loaded_weight.transpose(0, 1)
|
||||
loaded_weight = loaded_weight.reshape(shape)
|
||||
yield name, loaded_weight
|
||||
|
||||
weight_loader = getattr(param, "weight_loader", default_weight_loader)
|
||||
weight_loader(param, loaded_weight)
|
||||
loaded_params.add(name)
|
||||
|
||||
return loaded_params
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
loader = AutoWeightsLoader(self)
|
||||
return loader.load_weights(self._repack_qkv(weights))
|
||||
|
||||
|
||||
class BloomForCausalLM(nn.Module, SupportsPP, SupportsQuant):
|
||||
|
||||
@@ -16,9 +16,7 @@ from vllm.distributed import (
|
||||
)
|
||||
from vllm.model_executor.layers.activation import SiluAndMul
|
||||
from vllm.model_executor.layers.attention import Attention
|
||||
from vllm.model_executor.layers.fused_moe import (
|
||||
FusedMoE,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe import FusedMoE
|
||||
from vllm.model_executor.layers.linear import (
|
||||
MergedColumnParallelLinear,
|
||||
QKVParallelLinear,
|
||||
|
||||
@@ -22,9 +22,9 @@ from vllm.model_executor.custom_op import PluggableLayer
|
||||
from vllm.model_executor.layers.attention import MMEncoderAttention
|
||||
from vllm.model_executor.layers.conv import Conv2dLayer
|
||||
from vllm.model_executor.layers.quantization import QuantizationConfig
|
||||
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
|
||||
|
||||
from .clip import CLIPEncoder, CLIPVisionEmbeddings
|
||||
from .utils import AutoWeightsLoader
|
||||
|
||||
|
||||
class MLPBlock(nn.Module):
|
||||
@@ -671,12 +671,5 @@ class DeepCLIPVisionTransformer(nn.Module):
|
||||
return encoder_outputs
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
params_dict = dict(self.named_parameters())
|
||||
loaded_params: set[str] = set()
|
||||
|
||||
for name, loaded_weight in weights:
|
||||
param = params_dict[name]
|
||||
weight_loader = getattr(param, "weight_loader", default_weight_loader)
|
||||
weight_loader(param, loaded_weight)
|
||||
loaded_params.add(name)
|
||||
return loaded_params
|
||||
loader = AutoWeightsLoader(self)
|
||||
return loader.load_weights(weights)
|
||||
|
||||
@@ -33,9 +33,12 @@ from .deepseek_v2 import (
|
||||
DeepseekV2MixtureOfExperts,
|
||||
DeepseekV2MoE,
|
||||
_try_load_fp8_indexer_wk,
|
||||
get_spec_layer_idx_from_weight_name,
|
||||
)
|
||||
from .utils import get_pp_missing_layer_names, maybe_prefix
|
||||
from .utils import (
|
||||
get_pp_missing_layer_names,
|
||||
get_spec_layer_idx_from_weight_name,
|
||||
maybe_prefix,
|
||||
)
|
||||
|
||||
|
||||
def _restore_full_token_layout_if_needed(
|
||||
|
||||
@@ -108,6 +108,7 @@ from .interfaces import (
|
||||
from .utils import (
|
||||
PPMissingLayer,
|
||||
get_pp_missing_layer_names,
|
||||
get_spec_layer_idx_from_weight_name,
|
||||
is_pp_missing_parameter,
|
||||
make_empty_intermediate_tensors_factory,
|
||||
make_layers,
|
||||
@@ -1918,21 +1919,3 @@ class DeepseekV3ForCausalLM(DeepseekV2ForCausalLM):
|
||||
|
||||
class GlmMoeDsaForCausalLM(DeepseekV2ForCausalLM):
|
||||
pass
|
||||
|
||||
|
||||
# Compatibility with
|
||||
# https://huggingface.co/deepseek-ai/DeepSeek-V3-Base/blob/main/configuration_deepseek.py
|
||||
def get_spec_layer_idx_from_weight_name(
|
||||
config: DeepseekV2Config | DeepseekV3Config, weight_name: str
|
||||
) -> int | None:
|
||||
if (
|
||||
hasattr(config, "num_nextn_predict_layers")
|
||||
and config.num_nextn_predict_layers > 0
|
||||
):
|
||||
layer_idx = config.num_hidden_layers
|
||||
for i in range(config.num_nextn_predict_layers):
|
||||
if weight_name.startswith(
|
||||
f"model.layers.{layer_idx + i}."
|
||||
) or weight_name.startswith(f"layers.{layer_idx + i}."):
|
||||
return layer_idx + i
|
||||
return None
|
||||
|
||||
@@ -23,8 +23,7 @@
|
||||
# limitations under the License.
|
||||
"""Inference-only ErineMoE model compatible with HuggingFace weights."""
|
||||
|
||||
import typing
|
||||
from collections.abc import Callable, Iterable
|
||||
from collections.abc import Iterable
|
||||
from itertools import islice
|
||||
from typing import Any
|
||||
|
||||
@@ -42,11 +41,7 @@ from vllm.distributed import (
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.activation import SiluAndMul
|
||||
from vllm.model_executor.layers.attention import Attention
|
||||
from vllm.model_executor.layers.fused_moe import (
|
||||
FusedMoE,
|
||||
MoERunner,
|
||||
fused_moe_make_expert_params_mapping,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe import FusedMoE, MoERunner
|
||||
from vllm.model_executor.layers.layernorm import RMSNorm
|
||||
from vllm.model_executor.layers.linear import (
|
||||
MergedColumnParallelLinear,
|
||||
@@ -61,10 +56,6 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
|
||||
ParallelLMHead,
|
||||
VocabParallelEmbedding,
|
||||
)
|
||||
from vllm.model_executor.model_loader.weight_utils import (
|
||||
default_weight_loader,
|
||||
maybe_remap_kv_scale_name,
|
||||
)
|
||||
from vllm.sequence import IntermediateTensors
|
||||
from vllm.transformers_utils.config import set_default_rope_theta
|
||||
|
||||
@@ -72,8 +63,8 @@ from .interfaces import MixtureOfExperts, SupportsLoRA, SupportsPP
|
||||
from .utils import (
|
||||
AutoWeightsLoader,
|
||||
PPMissingLayer,
|
||||
WeightsMapper,
|
||||
extract_layer_index,
|
||||
is_pp_missing_parameter,
|
||||
make_empty_intermediate_tensors_factory,
|
||||
make_layers,
|
||||
maybe_prefix,
|
||||
@@ -407,6 +398,19 @@ class Ernie4_5_MoeDecoderLayer(nn.Module):
|
||||
|
||||
@support_torch_compile
|
||||
class Ernie4_5_MoeModel(nn.Module):
|
||||
hf_to_vllm_mapper = WeightsMapper(
|
||||
orig_to_new_stacked={
|
||||
".q_proj": (".qkv_proj", "q"),
|
||||
".k_proj": (".qkv_proj", "k"),
|
||||
".v_proj": (".qkv_proj", "v"),
|
||||
# .experts.gate_up_proj must be handled by MoERunner.load_weights for EP
|
||||
".mlp.gate_proj": (".mlp.gate_up_proj", 0),
|
||||
".mlp.up_proj": (".mlp.gate_up_proj", 1),
|
||||
".shared_experts.gate_proj": (".shared_experts.gate_up_proj", 0),
|
||||
".shared_experts.up_proj": (".shared_experts.gate_up_proj", 1),
|
||||
}
|
||||
)
|
||||
|
||||
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
|
||||
super().__init__()
|
||||
|
||||
@@ -486,132 +490,26 @@ class Ernie4_5_MoeModel(nn.Module):
|
||||
|
||||
return hidden_states
|
||||
|
||||
def get_expert_mapping(self) -> list[tuple[str, str, int, str]]:
|
||||
# Params for weights, fp8 weight scales, fp8 activation scales
|
||||
# (param_name, weight_name, expert_id, shard_id)
|
||||
return fused_moe_make_expert_params_mapping(
|
||||
self,
|
||||
ckpt_gate_proj_name="gate_proj",
|
||||
ckpt_down_proj_name="down_proj",
|
||||
ckpt_up_proj_name="up_proj",
|
||||
num_experts=self.config.moe_num_experts,
|
||||
num_redundant_experts=self.num_redundant_experts,
|
||||
)
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
stacked_params_mapping = [
|
||||
# (param_name, shard_name, shard_id)
|
||||
("qkv_proj", "q_proj", "q"),
|
||||
("qkv_proj", "k_proj", "k"),
|
||||
("qkv_proj", "v_proj", "v"),
|
||||
("gate_up_proj", "gate_proj", 0),
|
||||
("gate_up_proj", "up_proj", 1),
|
||||
]
|
||||
|
||||
params_dict = dict(self.named_parameters())
|
||||
loaded_params: set[str] = set()
|
||||
expert_params_mapping = self.get_expert_mapping()
|
||||
def _preprocess(
|
||||
self, weights: Iterable[tuple[str, torch.Tensor]]
|
||||
) -> Iterable[tuple[str, torch.Tensor]]:
|
||||
for name, loaded_weight in weights:
|
||||
if self.config.tie_word_embeddings and name.endswith("lm_head.weight"):
|
||||
continue
|
||||
# MTP will be supported soon.
|
||||
if "mtp" in name:
|
||||
continue
|
||||
|
||||
# moe_statics.e_score_correction_bias is stored with a leading
|
||||
# singleton dim and under a different module name.
|
||||
if "e_score_correction_bias" in name:
|
||||
name = name.replace("moe_statics", "gate")
|
||||
loaded_weight = loaded_weight.squeeze(0)
|
||||
yield name, loaded_weight
|
||||
|
||||
for param_name, weight_name, shard_id in stacked_params_mapping:
|
||||
# Skip non-stacked layers and experts (experts handled below).
|
||||
if weight_name not in name:
|
||||
continue
|
||||
|
||||
if ("mlp.experts." in name) and name not in params_dict:
|
||||
continue
|
||||
name = name.replace(weight_name, param_name)
|
||||
# Skip loading extra bias for GPTQ models.
|
||||
if (
|
||||
name.endswith(".bias") or name.endswith("_bias")
|
||||
) and name not in params_dict:
|
||||
continue
|
||||
# Skip layers on other devices.
|
||||
if is_pp_missing_parameter(name, self):
|
||||
continue
|
||||
|
||||
param = params_dict[name]
|
||||
weight_loader = param.weight_loader
|
||||
weight_loader(param, loaded_weight, shard_id)
|
||||
break
|
||||
else:
|
||||
is_expert_weight = False
|
||||
for mapping in expert_params_mapping:
|
||||
param_name, weight_name, expert_id, shard_id = mapping
|
||||
|
||||
if weight_name not in name:
|
||||
continue
|
||||
|
||||
# Anyway, this is an expert weight and should not be
|
||||
# attempted to load as other weights later
|
||||
is_expert_weight = True
|
||||
|
||||
# Do not modify `name` since the loop may continue here
|
||||
# Instead, create a new variable
|
||||
name_mapped = name.replace(weight_name, param_name)
|
||||
# Skip layers on other devices.
|
||||
if is_pp_missing_parameter(name_mapped, self):
|
||||
continue
|
||||
|
||||
# Skip loading extra bias for GPTQ models.
|
||||
if (
|
||||
name_mapped.endswith(".bias") or name_mapped.endswith("_bias")
|
||||
) and name_mapped not in params_dict:
|
||||
continue
|
||||
param = params_dict[name_mapped]
|
||||
# We should ask the weight loader to return success or not
|
||||
# here since otherwise we may skip experts with other
|
||||
# available replicas.
|
||||
weight_loader = typing.cast(
|
||||
Callable[..., bool], param.weight_loader
|
||||
)
|
||||
success = weight_loader(
|
||||
param,
|
||||
loaded_weight,
|
||||
name_mapped,
|
||||
shard_id=shard_id,
|
||||
expert_id=expert_id,
|
||||
return_success=True,
|
||||
)
|
||||
if success:
|
||||
name = name_mapped
|
||||
break
|
||||
else:
|
||||
if is_expert_weight:
|
||||
# We've checked that this is an expert weight
|
||||
# However it's not mapped locally to this rank
|
||||
# So we simply skip it
|
||||
continue
|
||||
|
||||
# Skip loading extra bias for GPTQ models.
|
||||
if (
|
||||
name.endswith(".bias") or name.endswith("_bias")
|
||||
) and name not in params_dict:
|
||||
continue
|
||||
# Skip layers on other devices.
|
||||
if is_pp_missing_parameter(name, self):
|
||||
continue
|
||||
# Remapping the name of FP8 kv-scale.
|
||||
name = maybe_remap_kv_scale_name(name, params_dict)
|
||||
if name is None:
|
||||
continue
|
||||
|
||||
param = params_dict[name]
|
||||
weight_loader = getattr(
|
||||
param, "weight_loader", default_weight_loader
|
||||
)
|
||||
weight_loader(param, loaded_weight)
|
||||
loaded_params.add(name)
|
||||
return loaded_params
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
loader = AutoWeightsLoader(
|
||||
self,
|
||||
skip_substrs=["mtp"],
|
||||
ignore_unexpected_suffixes=[".bias", "_bias"],
|
||||
)
|
||||
return loader.load_weights(
|
||||
self._preprocess(weights), mapper=self.hf_to_vllm_mapper
|
||||
)
|
||||
|
||||
|
||||
class Ernie4_5_MoeForCausalLM(nn.Module, SupportsPP, SupportsLoRA, MixtureOfExperts):
|
||||
@@ -741,6 +639,3 @@ class Ernie4_5_MoeForCausalLM(nn.Module, SupportsPP, SupportsLoRA, MixtureOfExpe
|
||||
skip_prefixes=(["lm_head."] if self.config.tie_word_embeddings else None),
|
||||
)
|
||||
return loader.load_weights(weights)
|
||||
|
||||
def get_expert_mapping(self) -> list[tuple[str, str, int, str]]:
|
||||
return self.model.get_expert_mapping()
|
||||
|
||||
@@ -30,7 +30,6 @@ from vllm.config.multimodal import BaseDummyOptions
|
||||
from vllm.inputs import MultiModalDataDict
|
||||
from vllm.model_executor.layers.attention.mm_encoder_attention import MMEncoderAttention
|
||||
from vllm.model_executor.layers.linear import QKVParallelLinear, RowParallelLinear
|
||||
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
|
||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
from vllm.multimodal.inputs import (
|
||||
MultiModalFieldConfig,
|
||||
@@ -53,7 +52,12 @@ from vllm.sequence import IntermediateTensors
|
||||
from vllm.utils.import_utils import _has_module
|
||||
|
||||
from .interfaces import MultiModalEmbeddings, SupportsMultiModal, SupportsPP
|
||||
from .utils import AutoWeightsLoader, init_vllm_registered_model, maybe_prefix
|
||||
from .utils import (
|
||||
AutoWeightsLoader,
|
||||
WeightsMapper,
|
||||
init_vllm_registered_model,
|
||||
maybe_prefix,
|
||||
)
|
||||
|
||||
|
||||
class _SinusoidsPositionEmbedding(nn.Module):
|
||||
@@ -79,6 +83,14 @@ class _SinusoidsPositionEmbedding(nn.Module):
|
||||
class FunAudioChatAudioAttention(nn.Module):
|
||||
"""Multi-headed attention used inside the continuous audio tower."""
|
||||
|
||||
hf_to_vllm_mapper = WeightsMapper(
|
||||
orig_to_new_stacked={
|
||||
".q_proj": (".qkv_proj", "q"),
|
||||
".k_proj": (".qkv_proj", "k"),
|
||||
".v_proj": (".qkv_proj", "v"),
|
||||
}
|
||||
)
|
||||
|
||||
def __init__(self, config: Any):
|
||||
super().__init__()
|
||||
self.embed_dim = int(config.d_model)
|
||||
@@ -123,42 +135,15 @@ class FunAudioChatAudioAttention(nn.Module):
|
||||
bias=True,
|
||||
)
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
stacked_params_mapping = [
|
||||
("qkv_proj", "q_proj", "q"),
|
||||
("qkv_proj", "k_proj", "k"),
|
||||
("qkv_proj", "v_proj", "v"),
|
||||
]
|
||||
|
||||
params_dict = dict(self.named_parameters())
|
||||
with torch.no_grad():
|
||||
if self.qkv_proj.bias is not None:
|
||||
# HF FunAudioChat uses bias=False for k_proj. Ensure the missing
|
||||
# shard starts as zeros, while allowing q/v shards to load.
|
||||
self.qkv_proj.bias.zero_()
|
||||
|
||||
loaded_params: set[str] = set()
|
||||
for name, loaded_weight in weights:
|
||||
for param_name, shard_name, shard_id in stacked_params_mapping:
|
||||
if shard_name not in name:
|
||||
continue
|
||||
name = name.replace(shard_name, param_name)
|
||||
param = params_dict[name]
|
||||
weight_loader = getattr(param, "weight_loader", default_weight_loader)
|
||||
weight_loader(param, loaded_weight, shard_id)
|
||||
break
|
||||
else:
|
||||
# Skip loading extra bias for GPTQ models.
|
||||
if name.endswith(".bias") and name not in params_dict:
|
||||
continue
|
||||
|
||||
param = params_dict[name]
|
||||
weight_loader = getattr(param, "weight_loader", default_weight_loader)
|
||||
weight_loader(param, loaded_weight)
|
||||
|
||||
loaded_params.add(name)
|
||||
|
||||
return loaded_params
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
loader = AutoWeightsLoader(self)
|
||||
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
|
||||
@@ -44,18 +44,14 @@ from vllm.model_executor.layers.logits_processor import LogitsProcessor
|
||||
from vllm.model_executor.layers.quantization import QuantizationConfig
|
||||
from vllm.model_executor.layers.rotary_embedding import get_rope
|
||||
from vllm.model_executor.layers.vocab_parallel_embedding import VocabParallelEmbedding
|
||||
from vllm.model_executor.model_loader.weight_utils import (
|
||||
default_weight_loader,
|
||||
maybe_remap_kv_scale_name,
|
||||
)
|
||||
from vllm.sequence import IntermediateTensors
|
||||
from vllm.v1.attention.backends.utils import KVSharingFastPrefillMetadata
|
||||
|
||||
from .interfaces import SupportsQuant
|
||||
from .utils import (
|
||||
AutoWeightsLoader,
|
||||
WeightsMapper,
|
||||
extract_layer_index,
|
||||
is_pp_missing_parameter,
|
||||
make_layers,
|
||||
maybe_prefix,
|
||||
)
|
||||
@@ -788,6 +784,25 @@ class Gemma3nCrossDecoder(nn.Module):
|
||||
enable_if=lambda vllm_config: not vllm_config.cache_config.kv_sharing_fast_prefill
|
||||
)
|
||||
class Gemma3nTextModel(nn.Module, SupportsQuant):
|
||||
# Decoder layers, altup_unembed_projections and norm live on the text
|
||||
# model; every other submodule lives under self_decoder.
|
||||
hf_to_vllm_mapper = WeightsMapper(
|
||||
orig_to_new_prefix={
|
||||
"embed_tokens.": "self_decoder.embed_tokens.",
|
||||
"embed_tokens_per_layer.": "self_decoder.embed_tokens_per_layer.",
|
||||
"per_layer_model_projection.": "self_decoder.per_layer_model_projection.",
|
||||
"per_layer_projection_norm.": "self_decoder.per_layer_projection_norm.",
|
||||
"altup_projections.": "self_decoder.altup_projections.",
|
||||
},
|
||||
orig_to_new_stacked={
|
||||
".q_proj": (".qkv_proj", "q"),
|
||||
".k_proj": (".qkv_proj", "k"),
|
||||
".v_proj": (".qkv_proj", "v"),
|
||||
".gate_proj": (".gate_up_proj", 0),
|
||||
".up_proj": (".gate_up_proj", 1),
|
||||
},
|
||||
)
|
||||
|
||||
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
|
||||
super().__init__()
|
||||
config = vllm_config.model_config.hf_config
|
||||
@@ -1036,58 +1051,8 @@ class Gemma3nTextModel(nn.Module, SupportsQuant):
|
||||
return self.norm(hidden_states)
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
stacked_params_mapping = [
|
||||
# (param_name, shard_name, shard_id)
|
||||
("qkv_proj", "q_proj", "q"),
|
||||
("qkv_proj", "k_proj", "k"),
|
||||
("qkv_proj", "v_proj", "v"),
|
||||
("gate_up_proj", "gate_proj", 0),
|
||||
("gate_up_proj", "up_proj", 1),
|
||||
]
|
||||
params_dict = dict(self.named_parameters())
|
||||
loaded_params: set[str] = set()
|
||||
for name, loaded_weight in weights:
|
||||
# decoder layer weights, altup_unembed_projections and rmsnorm
|
||||
# are initialized in text model, others are in self decoder
|
||||
if (
|
||||
not name.startswith("layers")
|
||||
and not name.startswith("altup_unembed_projections")
|
||||
and not name.startswith("norm")
|
||||
):
|
||||
name = f"self_decoder.{name}"
|
||||
|
||||
for param_name, shard_name, shard_id in stacked_params_mapping:
|
||||
if shard_name not in name:
|
||||
continue
|
||||
# Avoid spurious match with ".up_proj".
|
||||
if "altup_projections" in name:
|
||||
continue
|
||||
name = name.replace(shard_name, param_name)
|
||||
# Skip loading extra bias for GPTQ models.
|
||||
if name.endswith(".bias") and name not in params_dict:
|
||||
continue
|
||||
if is_pp_missing_parameter(name, self):
|
||||
continue
|
||||
param = params_dict[name]
|
||||
weight_loader = param.weight_loader
|
||||
weight_loader(param, loaded_weight, shard_id)
|
||||
break
|
||||
else:
|
||||
# Skip loading extra bias for GPTQ models.
|
||||
if name.endswith(".bias") and name not in params_dict:
|
||||
continue
|
||||
# Remapping the name of FP8 kv-scale.
|
||||
name = maybe_remap_kv_scale_name(name, params_dict)
|
||||
if name is None:
|
||||
continue
|
||||
if is_pp_missing_parameter(name, self):
|
||||
continue
|
||||
param = params_dict[name]
|
||||
weight_loader = getattr(param, "weight_loader", default_weight_loader)
|
||||
weight_loader(param, loaded_weight)
|
||||
loaded_params.add(name)
|
||||
|
||||
return loaded_params
|
||||
loader = AutoWeightsLoader(self)
|
||||
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
|
||||
|
||||
|
||||
class Gemma3nForCausalLM(nn.Module):
|
||||
|
||||
@@ -303,16 +303,3 @@ class Glm4ForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
|
||||
]
|
||||
loader = AutoWeightsLoader(self, skip_prefixes=skip_prefixes)
|
||||
return loader.load_weights(weights)
|
||||
|
||||
|
||||
def get_spec_layer_idx_from_weight_name(
|
||||
config: Glm4Config, weight_name: str
|
||||
) -> int | None:
|
||||
if hasattr(config, "num_nextn_predict_layers") and (
|
||||
config.num_nextn_predict_layers > 0
|
||||
):
|
||||
layer_idx = config.num_hidden_layers
|
||||
for i in range(config.num_nextn_predict_layers):
|
||||
if f"layers.{layer_idx + i}." in weight_name:
|
||||
return layer_idx + i
|
||||
return None
|
||||
|
||||
@@ -24,8 +24,7 @@
|
||||
"""Inference-only GLM-4.5, GLM-4.6, GLM-4.7 model
|
||||
compatible with HuggingFace weights."""
|
||||
|
||||
import typing
|
||||
from collections.abc import Callable, Iterable
|
||||
from collections.abc import Iterable
|
||||
from itertools import islice
|
||||
|
||||
import torch
|
||||
@@ -43,10 +42,7 @@ from vllm.distributed import (
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.activation import SiluAndMul
|
||||
from vllm.model_executor.layers.attention import Attention
|
||||
from vllm.model_executor.layers.fused_moe import (
|
||||
FusedMoE,
|
||||
fused_moe_make_expert_params_mapping,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe import FusedMoE
|
||||
from vllm.model_executor.layers.layernorm import RMSNorm
|
||||
from vllm.model_executor.layers.linear import (
|
||||
MergedColumnParallelLinear,
|
||||
@@ -60,20 +56,18 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
|
||||
ParallelLMHead,
|
||||
VocabParallelEmbedding,
|
||||
)
|
||||
from vllm.model_executor.model_loader.weight_utils import (
|
||||
default_weight_loader,
|
||||
maybe_remap_kv_scale_name,
|
||||
)
|
||||
from vllm.sequence import IntermediateTensors
|
||||
|
||||
from .interfaces import MixtureOfExperts, SupportsLoRA, SupportsPP
|
||||
from .utils import (
|
||||
AutoWeightsLoader,
|
||||
PPMissingLayer,
|
||||
is_pp_missing_parameter,
|
||||
WeightsMapper,
|
||||
make_empty_intermediate_tensors_factory,
|
||||
make_layers,
|
||||
maybe_fuse_shared_experts,
|
||||
maybe_prefix,
|
||||
skip_spec_layers,
|
||||
)
|
||||
|
||||
logger = init_logger(__name__)
|
||||
@@ -410,6 +404,19 @@ class Glm4MoeDecoderLayer(nn.Module):
|
||||
}
|
||||
)
|
||||
class Glm4MoeModel(nn.Module):
|
||||
hf_to_vllm_mapper = WeightsMapper(
|
||||
orig_to_new_stacked={
|
||||
".q_proj": (".qkv_proj", "q"),
|
||||
".k_proj": (".qkv_proj", "k"),
|
||||
".v_proj": (".qkv_proj", "v"),
|
||||
# .experts.gate_up_proj must be handled by MoERunner.load_weights for EP
|
||||
".mlp.gate_proj": (".mlp.gate_up_proj", 0),
|
||||
".mlp.up_proj": (".mlp.gate_up_proj", 1),
|
||||
".shared_experts.gate_proj": (".shared_experts.gate_up_proj", 0),
|
||||
".shared_experts.up_proj": (".shared_experts.gate_up_proj", 1),
|
||||
}
|
||||
)
|
||||
|
||||
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
|
||||
super().__init__()
|
||||
|
||||
@@ -480,189 +487,14 @@ class Glm4MoeModel(nn.Module):
|
||||
hidden_states, _ = self.norm(hidden_states, residual)
|
||||
return hidden_states
|
||||
|
||||
def get_expert_mapping(self) -> list[tuple[str, str, int, str]]:
|
||||
# Params for weights, fp8 weight scales, fp8 activation scales
|
||||
# (param_name, weight_name, expert_id, shard_id)
|
||||
# FSE widens the mapping by n_shared_experts slots; see deepseek_v2.py.
|
||||
num_experts = self.config.n_routed_experts
|
||||
if (
|
||||
rocm_aiter_ops.is_fusion_moe_shared_experts_enabled()
|
||||
and self.config.n_shared_experts
|
||||
):
|
||||
num_experts += self.config.n_shared_experts
|
||||
return fused_moe_make_expert_params_mapping(
|
||||
self,
|
||||
ckpt_gate_proj_name="gate_proj",
|
||||
ckpt_down_proj_name="down_proj",
|
||||
ckpt_up_proj_name="up_proj",
|
||||
num_experts=num_experts,
|
||||
)
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
rocm_aiter_moe_shared_expert_enabled = (
|
||||
rocm_aiter_ops.is_fusion_moe_shared_experts_enabled()
|
||||
weights = maybe_fuse_shared_experts(
|
||||
skip_spec_layers(weights, self.config),
|
||||
n_routed_experts=self.config.n_routed_experts,
|
||||
n_shared_experts=self.config.n_shared_experts or 1,
|
||||
)
|
||||
stacked_params_mapping = [
|
||||
# (param_name, shard_name, shard_id)
|
||||
("qkv_proj", "q_proj", "q"),
|
||||
("qkv_proj", "k_proj", "k"),
|
||||
("qkv_proj", "v_proj", "v"),
|
||||
("gate_up_proj", "gate_proj", 0),
|
||||
("gate_up_proj", "up_proj", 1),
|
||||
]
|
||||
|
||||
params_dict = dict(self.named_parameters())
|
||||
loaded_params: set[str] = set()
|
||||
expert_params_mapping = self.get_expert_mapping()
|
||||
for name, loaded_weight in weights:
|
||||
spec_layer = get_spec_layer_idx_from_weight_name(self.config, name)
|
||||
if spec_layer is not None:
|
||||
continue
|
||||
|
||||
is_fusion_moe_shared_experts_layer = (
|
||||
rocm_aiter_moe_shared_expert_enabled and ("mlp.shared_experts" in name)
|
||||
)
|
||||
|
||||
for param_name, weight_name, shard_id in stacked_params_mapping:
|
||||
# Skip non-stacked layers and experts (experts handled below).
|
||||
if weight_name not in name:
|
||||
continue
|
||||
# We have mlp.experts[0].gate_proj in the checkpoint.
|
||||
# Since we handle the experts below in expert_params_mapping,
|
||||
# we need to skip here BEFORE we update the name, otherwise
|
||||
# name will be updated to mlp.experts[0].gate_up_proj, which
|
||||
# will then be updated below in expert_params_mapping
|
||||
# for mlp.experts[0].gate_gate_up_proj, which breaks load.
|
||||
if ("mlp.experts." in name) and name not in params_dict:
|
||||
continue
|
||||
if is_fusion_moe_shared_experts_layer:
|
||||
continue
|
||||
|
||||
name = name.replace(weight_name, param_name)
|
||||
# Skip loading extra bias for GPTQ models.
|
||||
if name.endswith(".bias") and name not in params_dict:
|
||||
continue
|
||||
|
||||
name = maybe_remap_kv_scale_name(name, params_dict)
|
||||
if name is None:
|
||||
continue
|
||||
if is_pp_missing_parameter(name, self):
|
||||
continue
|
||||
|
||||
param = params_dict[name]
|
||||
weight_loader = getattr(param, "weight_loader", default_weight_loader)
|
||||
if weight_loader == default_weight_loader:
|
||||
weight_loader(param, loaded_weight)
|
||||
else:
|
||||
weight_loader(param, loaded_weight, shard_id)
|
||||
break
|
||||
else:
|
||||
is_expert_weight = False
|
||||
|
||||
# FSE: split a widened mlp.shared_experts tensor into
|
||||
# n_shared_experts chunks; see deepseek_v2.py for details.
|
||||
num_chunks = 1
|
||||
split_dim = 0
|
||||
chunk_size = 0
|
||||
if is_fusion_moe_shared_experts_layer:
|
||||
num_chunks = getattr(self.config, "n_shared_experts", 1) or 1
|
||||
split_dim = (
|
||||
1
|
||||
if ("down_proj.weight" in name and loaded_weight.ndim > 1)
|
||||
else 0
|
||||
)
|
||||
total = loaded_weight.shape[split_dim]
|
||||
if total % num_chunks != 0:
|
||||
raise ValueError(
|
||||
f"FSE shared-expert weight {name} has dim "
|
||||
f"{total} along axis {split_dim} which is not "
|
||||
f"divisible by n_shared_experts={num_chunks}."
|
||||
)
|
||||
chunk_size = total // num_chunks
|
||||
|
||||
for j in range(num_chunks):
|
||||
chunk_name = name
|
||||
weight_to_load = loaded_weight
|
||||
|
||||
if is_fusion_moe_shared_experts_layer:
|
||||
chunk_slice = slice(j * chunk_size, (j + 1) * chunk_size)
|
||||
if loaded_weight.ndim == 1:
|
||||
weight_to_load = loaded_weight[chunk_slice]
|
||||
elif split_dim == 0:
|
||||
weight_to_load = loaded_weight[chunk_slice, :]
|
||||
else:
|
||||
weight_to_load = loaded_weight[:, chunk_slice]
|
||||
# Synthesize an expert-style name for expert mapping.
|
||||
chunk_name = name.replace(
|
||||
"mlp.shared_experts",
|
||||
f"mlp.experts.{self.config.n_routed_experts + j}",
|
||||
)
|
||||
|
||||
for mapping in expert_params_mapping:
|
||||
param_name, weight_name, expert_id, shard_id = mapping
|
||||
if weight_name not in chunk_name:
|
||||
continue
|
||||
|
||||
# Anyway, this is an expert weight and should not be
|
||||
# attempted to load as other weights later
|
||||
is_expert_weight = True
|
||||
|
||||
# Do not modify `name` since the loop may continue here
|
||||
# Instead, create a new variable
|
||||
name_mapped = chunk_name.replace(weight_name, param_name)
|
||||
|
||||
if is_pp_missing_parameter(name_mapped, self):
|
||||
continue
|
||||
|
||||
param = params_dict[name_mapped]
|
||||
# We should ask the weight loader to return success
|
||||
# or not here since otherwise we may skip experts
|
||||
# with other available replicas.
|
||||
weight_loader = typing.cast(
|
||||
Callable[..., bool], param.weight_loader
|
||||
)
|
||||
success = weight_loader(
|
||||
param,
|
||||
weight_to_load,
|
||||
name_mapped,
|
||||
shard_id=shard_id,
|
||||
expert_id=expert_id,
|
||||
return_success=True,
|
||||
)
|
||||
if success:
|
||||
if not is_fusion_moe_shared_experts_layer:
|
||||
name = name_mapped
|
||||
else:
|
||||
loaded_params.add(name_mapped)
|
||||
break
|
||||
else:
|
||||
if is_expert_weight:
|
||||
# We've checked that this is an expert weight
|
||||
# However it's not mapped locally to this rank
|
||||
# So we simply skip it
|
||||
continue
|
||||
|
||||
# Skip loading extra bias for GPTQ models.
|
||||
if name.endswith(".bias") and name not in params_dict:
|
||||
continue
|
||||
|
||||
# Remapping the name of FP8 kv-scale.
|
||||
name = maybe_remap_kv_scale_name(name, params_dict)
|
||||
if name is None:
|
||||
continue
|
||||
|
||||
if is_pp_missing_parameter(name, self):
|
||||
continue
|
||||
|
||||
param = params_dict[name]
|
||||
weight_loader = getattr(
|
||||
param, "weight_loader", default_weight_loader
|
||||
)
|
||||
weight_loader(param, loaded_weight)
|
||||
if name is not None and not is_fusion_moe_shared_experts_layer:
|
||||
loaded_params.add(name)
|
||||
|
||||
return loaded_params
|
||||
loader = AutoWeightsLoader(self)
|
||||
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
|
||||
|
||||
|
||||
class Glm4MixtureOfExperts(MixtureOfExperts):
|
||||
@@ -776,19 +608,3 @@ class Glm4MoeForCausalLM(nn.Module, SupportsPP, SupportsLoRA, Glm4MixtureOfExper
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
loader = AutoWeightsLoader(self)
|
||||
return loader.load_weights(weights)
|
||||
|
||||
def get_expert_mapping(self) -> list[tuple[str, str, int, str]]:
|
||||
return self.model.get_expert_mapping()
|
||||
|
||||
|
||||
def get_spec_layer_idx_from_weight_name(
|
||||
config: Glm4MoeConfig, weight_name: str
|
||||
) -> int | None:
|
||||
if hasattr(config, "num_nextn_predict_layers") and (
|
||||
config.num_nextn_predict_layers > 0
|
||||
):
|
||||
layer_idx = config.num_hidden_layers
|
||||
for i in range(config.num_nextn_predict_layers):
|
||||
if f"layers.{layer_idx + i}." in weight_name:
|
||||
return layer_idx + i
|
||||
return None
|
||||
|
||||
@@ -70,6 +70,7 @@ from .interfaces import SupportsLoRA, SupportsPP
|
||||
from .utils import (
|
||||
AutoWeightsLoader,
|
||||
PPMissingLayer,
|
||||
get_spec_layer_idx_from_weight_name,
|
||||
is_pp_missing_parameter,
|
||||
make_empty_intermediate_tensors_factory,
|
||||
make_layers,
|
||||
@@ -628,16 +629,3 @@ class Glm4MoeLiteForCausalLM(
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
loader = AutoWeightsLoader(self)
|
||||
return loader.load_weights(weights)
|
||||
|
||||
|
||||
def get_spec_layer_idx_from_weight_name(
|
||||
config: "Glm4MoeLiteConfig", weight_name: str
|
||||
) -> int | None:
|
||||
if hasattr(config, "num_nextn_predict_layers") and (
|
||||
config.num_nextn_predict_layers > 0
|
||||
):
|
||||
layer_idx = config.num_hidden_layers
|
||||
for i in range(config.num_nextn_predict_layers):
|
||||
if f"layers.{layer_idx + i}." in weight_name:
|
||||
return layer_idx + i
|
||||
return None
|
||||
|
||||
@@ -54,10 +54,9 @@ from .glm4_moe_lite import (
|
||||
Glm4MixtureOfExperts,
|
||||
Glm4MoeLite,
|
||||
Glm4MoeLiteDecoderLayer,
|
||||
get_spec_layer_idx_from_weight_name,
|
||||
)
|
||||
from .interfaces import SupportsPP
|
||||
from .utils import maybe_prefix
|
||||
from .utils import get_spec_layer_idx_from_weight_name, maybe_prefix
|
||||
|
||||
|
||||
class SharedHead(nn.Module):
|
||||
|
||||
@@ -51,9 +51,8 @@ from .glm4_moe import (
|
||||
Glm4MixtureOfExperts,
|
||||
Glm4MoE,
|
||||
Glm4MoeDecoderLayer,
|
||||
get_spec_layer_idx_from_weight_name,
|
||||
)
|
||||
from .utils import maybe_prefix
|
||||
from .utils import get_spec_layer_idx_from_weight_name, maybe_prefix
|
||||
|
||||
|
||||
class SharedHead(nn.Module):
|
||||
|
||||
@@ -41,13 +41,14 @@ from vllm.model_executor.model_loader.weight_utils import (
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.sequence import IntermediateTensors
|
||||
|
||||
from .glm4 import Glm4DecoderLayer, get_spec_layer_idx_from_weight_name
|
||||
from .glm4 import Glm4DecoderLayer
|
||||
from .glm4_moe_lite_mtp import (
|
||||
Glm4MoeLiteMultiTokenPredictor,
|
||||
SharedHead,
|
||||
)
|
||||
from .interfaces import SupportsPP
|
||||
from .utils import (
|
||||
get_spec_layer_idx_from_weight_name,
|
||||
is_pp_missing_parameter,
|
||||
maybe_prefix,
|
||||
)
|
||||
|
||||
@@ -65,7 +65,12 @@ from .interfaces import (
|
||||
SupportsPP,
|
||||
SupportsTranscription,
|
||||
)
|
||||
from .utils import AutoWeightsLoader, init_vllm_registered_model, maybe_prefix
|
||||
from .utils import (
|
||||
AutoWeightsLoader,
|
||||
WeightsMapper,
|
||||
init_vllm_registered_model,
|
||||
maybe_prefix,
|
||||
)
|
||||
from .whisper import ISO639_1_SUPPORTED_LANGS, _create_fake_bias_for_k_proj
|
||||
|
||||
|
||||
@@ -395,6 +400,14 @@ class GlmAsrEncoder(nn.Module):
|
||||
"qkv_proj": ["q_proj", "k_proj", "v_proj"],
|
||||
}
|
||||
|
||||
hf_to_vllm_mapper = WeightsMapper(
|
||||
orig_to_new_stacked={
|
||||
".q_proj": (".qkv_proj", "q"),
|
||||
".k_proj": (".qkv_proj", "k"),
|
||||
".v_proj": (".qkv_proj", "v"),
|
||||
}
|
||||
)
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
config,
|
||||
@@ -496,44 +509,9 @@ class GlmAsrEncoder(nn.Module):
|
||||
return _GlmAsrEncoderOutput(last_hidden_state=hidden_states)
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
"""Custom weight loading to handle q_proj/k_proj/v_proj -> qkv_proj mapping."""
|
||||
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
|
||||
|
||||
weights = _create_fake_bias_for_k_proj(weights, ".k_proj.weight")
|
||||
|
||||
stacked_params_mapping = [
|
||||
# (param_name, shard_name, shard_id)
|
||||
("qkv_proj", "q_proj", "q"),
|
||||
("qkv_proj", "k_proj", "k"),
|
||||
("qkv_proj", "v_proj", "v"),
|
||||
]
|
||||
params_dict = dict(self.named_parameters())
|
||||
loaded_params: set[str] = set()
|
||||
|
||||
for name, loaded_weight in weights:
|
||||
for param_name, weight_name, shard_id in stacked_params_mapping:
|
||||
if weight_name not in name:
|
||||
continue
|
||||
name = name.replace(weight_name, param_name)
|
||||
# Skip loading extra bias for GPTQ models.
|
||||
if name.endswith(".bias") and name not in params_dict:
|
||||
continue
|
||||
|
||||
param = params_dict[name]
|
||||
weight_loader = param.weight_loader
|
||||
weight_loader(param, loaded_weight, shard_id)
|
||||
break
|
||||
else:
|
||||
# Default weight loading for non-stacked params
|
||||
if name.endswith(".bias") and name not in params_dict:
|
||||
continue
|
||||
if name not in params_dict:
|
||||
continue
|
||||
param = params_dict[name]
|
||||
weight_loader = getattr(param, "weight_loader", default_weight_loader)
|
||||
weight_loader(param, loaded_weight)
|
||||
loaded_params.add(name)
|
||||
return loaded_params
|
||||
loader = AutoWeightsLoader(self)
|
||||
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
|
||||
|
||||
|
||||
class GlmAsrFeatureInputs(TensorSchema):
|
||||
|
||||
@@ -47,13 +47,11 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
|
||||
ParallelLMHead,
|
||||
VocabParallelEmbedding,
|
||||
)
|
||||
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
|
||||
from vllm.sequence import IntermediateTensors
|
||||
|
||||
from .interfaces import SupportsCrossEncoding, SupportsPP
|
||||
from .utils import (
|
||||
AutoWeightsLoader,
|
||||
is_pp_missing_parameter,
|
||||
make_empty_intermediate_tensors_factory,
|
||||
make_layers,
|
||||
maybe_prefix,
|
||||
@@ -241,32 +239,25 @@ class GPT2Model(nn.Module):
|
||||
hidden_states = self.ln_f(hidden_states)
|
||||
return hidden_states
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
params_dict = dict(self.named_parameters(remove_duplicate=False))
|
||||
loaded_params: set[str] = set()
|
||||
def _transpose_conv1d(
|
||||
self, weights: Iterable[tuple[str, torch.Tensor]]
|
||||
) -> Iterable[tuple[str, torch.Tensor]]:
|
||||
# HF's GPT-2 uses Conv1D instead of Linear, so its 2D weights are
|
||||
# stored transposed relative to what vLLM expects.
|
||||
# Note(zhuohan): the logic below might break quantized models.
|
||||
for name, loaded_weight in weights:
|
||||
if ".attn.bias" in name or ".attn.masked_bias" in name:
|
||||
# Skip attention mask.
|
||||
# NOTE: "c_attn.bias" should not be skipped.
|
||||
continue
|
||||
|
||||
if is_pp_missing_parameter(name, self):
|
||||
continue
|
||||
|
||||
param = params_dict[name]
|
||||
# The HF's GPT-2 implementation uses Conv1D instead of Linear.
|
||||
# Because of this, we need to transpose the weights.
|
||||
# Note(zhuohan): the logic below might break quantized models.
|
||||
for conv1d_weight_name in ["c_attn", "c_proj", "c_fc"]:
|
||||
if conv1d_weight_name not in name:
|
||||
continue
|
||||
if not name.endswith(".weight"):
|
||||
continue
|
||||
if name.endswith(".weight") and any(
|
||||
proj in name for proj in ("c_attn", "c_proj", "c_fc")
|
||||
):
|
||||
loaded_weight = loaded_weight.t()
|
||||
weight_loader = getattr(param, "weight_loader", default_weight_loader)
|
||||
weight_loader(param, loaded_weight)
|
||||
loaded_params.add(name)
|
||||
return loaded_params
|
||||
yield name, loaded_weight
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
# Skip attention mask buffers; NOTE: "c_attn.bias" must not be skipped.
|
||||
loader = AutoWeightsLoader(
|
||||
self, skip_substrs=[".attn.bias", ".attn.masked_bias"]
|
||||
)
|
||||
return loader.load_weights(self._transpose_conv1d(weights))
|
||||
|
||||
|
||||
class GPT2LMHeadModel(nn.Module, SupportsPP):
|
||||
|
||||
@@ -43,13 +43,11 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
|
||||
ParallelLMHead,
|
||||
VocabParallelEmbedding,
|
||||
)
|
||||
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
|
||||
from vllm.sequence import IntermediateTensors
|
||||
|
||||
from .interfaces import SupportsPP
|
||||
from .utils import (
|
||||
AutoWeightsLoader,
|
||||
is_pp_missing_parameter,
|
||||
make_empty_intermediate_tensors_factory,
|
||||
make_layers,
|
||||
maybe_prefix,
|
||||
@@ -249,45 +247,25 @@ class GPTNeoXModel(nn.Module):
|
||||
hidden_states = self.final_layer_norm(hidden_states)
|
||||
return hidden_states
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
params_dict = dict(self.named_parameters())
|
||||
loaded_params: set[str] = set()
|
||||
def _repack_qkv(
|
||||
self, weights: Iterable[tuple[str, torch.Tensor]]
|
||||
) -> Iterable[tuple[str, torch.Tensor]]:
|
||||
# GPT-NeoX's fused QKV is laid out as (num_heads * 3 * head_size) on
|
||||
# its output dim (0), while vLLM expects (3 * num_heads * head_size).
|
||||
num_heads = self.config.num_attention_heads
|
||||
for name, loaded_weight in weights:
|
||||
if (
|
||||
"attention.bias" in name
|
||||
or "attention.masked_bias" in name
|
||||
or "rotary_emb.inv_freq" in name
|
||||
):
|
||||
continue
|
||||
if "rotary_emb.cos_cached" in name or "rotary_emb.sin_cached" in name:
|
||||
# Models trained using OpenRLHF may include
|
||||
# these tensors in the checkpoint. Skip them.
|
||||
continue
|
||||
if is_pp_missing_parameter(name, self):
|
||||
continue
|
||||
param = params_dict[name]
|
||||
|
||||
if "query_key_value" in name:
|
||||
# NOTE: GPT-NeoX's fused QKV's output_dim has the shape of
|
||||
# (num_heads * 3 * head_size), while the
|
||||
# required shape is (3 * num_heads * head_size).
|
||||
# Thus, we need weight conversion.
|
||||
output_dim = getattr(param, "output_dim", None)
|
||||
num_heads = self.config.num_attention_heads
|
||||
if output_dim is not None:
|
||||
loaded_weight_shape = loaded_weight.shape
|
||||
loaded_weight = loaded_weight.view(
|
||||
loaded_weight_shape[:output_dim]
|
||||
+ (num_heads, 3, -1)
|
||||
+ loaded_weight_shape[output_dim + 1 :]
|
||||
)
|
||||
loaded_weight = loaded_weight.transpose(output_dim, output_dim + 1)
|
||||
loaded_weight = loaded_weight.reshape(loaded_weight_shape)
|
||||
shape = loaded_weight.shape
|
||||
loaded_weight = loaded_weight.view((num_heads, 3, -1) + shape[1:])
|
||||
loaded_weight = loaded_weight.transpose(0, 1)
|
||||
loaded_weight = loaded_weight.reshape(shape)
|
||||
yield name, loaded_weight
|
||||
|
||||
weight_loader = getattr(param, "weight_loader", default_weight_loader)
|
||||
weight_loader(param, loaded_weight)
|
||||
loaded_params.add(name)
|
||||
return loaded_params
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
loader = AutoWeightsLoader(
|
||||
self, skip_substrs=["attention.bias", "attention.masked_bias"]
|
||||
)
|
||||
return loader.load_weights(self._repack_qkv(weights))
|
||||
|
||||
|
||||
class GPTNeoXForCausalLM(nn.Module, SupportsPP):
|
||||
|
||||
@@ -73,6 +73,7 @@ from .interfaces import MixtureOfExperts, SupportsLoRA, SupportsPP
|
||||
from .utils import (
|
||||
AutoWeightsLoader,
|
||||
PPMissingLayer,
|
||||
get_spec_layer_idx_from_weight_name,
|
||||
is_pp_missing_parameter,
|
||||
make_empty_intermediate_tensors_factory,
|
||||
make_layers,
|
||||
@@ -650,21 +651,6 @@ class HYV3Model(nn.Module, MixtureOfExperts):
|
||||
return loaded_params
|
||||
|
||||
|
||||
def get_spec_layer_idx_from_weight_name(
|
||||
config: PretrainedConfig, weight_name: str
|
||||
) -> int | None:
|
||||
# HYV3MTP is enabled only when num_nextn_predict_layers is greater than 1
|
||||
if (
|
||||
hasattr(config, "num_nextn_predict_layers")
|
||||
and config.num_nextn_predict_layers > 0
|
||||
):
|
||||
layer_idx = config.num_hidden_layers
|
||||
for i in range(config.num_nextn_predict_layers):
|
||||
if weight_name.startswith(f"model.layers.{layer_idx + i}."):
|
||||
return layer_idx + i
|
||||
return None
|
||||
|
||||
|
||||
class HYV3ForCausalLM(nn.Module, SupportsPP, SupportsLoRA):
|
||||
packed_modules_mapping = {
|
||||
"qkv_proj": ["q_proj", "k_proj", "v_proj"],
|
||||
|
||||
@@ -51,8 +51,12 @@ from vllm.v1.outputs import SamplerOutput
|
||||
from vllm.v1.sample.metadata import SamplingMetadata
|
||||
from vllm.v1.sample.sampler import Sampler
|
||||
|
||||
from .hy_v3 import HYV3DecoderLayer, get_spec_layer_idx_from_weight_name
|
||||
from .utils import is_pp_missing_parameter, maybe_prefix
|
||||
from .hy_v3 import HYV3DecoderLayer
|
||||
from .utils import (
|
||||
get_spec_layer_idx_from_weight_name,
|
||||
is_pp_missing_parameter,
|
||||
maybe_prefix,
|
||||
)
|
||||
|
||||
|
||||
def _is_moe(config: PretrainedConfig) -> bool:
|
||||
|
||||
@@ -36,8 +36,8 @@ from vllm.model_executor.layers.linear import (
|
||||
RowParallelLinear,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization import QuantizationConfig
|
||||
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
|
||||
|
||||
from .utils import AutoWeightsLoader
|
||||
from .vision import is_vit_use_data_parallel, run_dp_sharded_vision_model
|
||||
|
||||
NORM2FN = {
|
||||
@@ -445,11 +445,5 @@ class InternVisionModel(nn.Module):
|
||||
return encoder_outputs
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
params_dict = dict(self.named_parameters())
|
||||
loaded_params: set[str] = set()
|
||||
for name, loaded_weight in weights:
|
||||
param = params_dict[name]
|
||||
weight_loader = getattr(param, "weight_loader", default_weight_loader)
|
||||
weight_loader(param, loaded_weight)
|
||||
loaded_params.add(name)
|
||||
return loaded_params
|
||||
loader = AutoWeightsLoader(self)
|
||||
return loader.load_weights(weights)
|
||||
|
||||
@@ -20,7 +20,8 @@ from vllm.model_executor.layers.conv import Conv2dLayer
|
||||
from vllm.model_executor.layers.layernorm import RMSNorm
|
||||
from vllm.model_executor.layers.linear import ColumnParallelLinear, RowParallelLinear
|
||||
from vllm.model_executor.layers.quantization import QuantizationConfig
|
||||
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
|
||||
|
||||
from .utils import AutoWeightsLoader
|
||||
|
||||
NORM2FN = {
|
||||
"rms_norm": RMSNorm,
|
||||
@@ -433,11 +434,5 @@ class InternS1VisionModel(nn.Module):
|
||||
return encoder_outputs
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
params_dict = dict(self.named_parameters())
|
||||
loaded_params: set[str] = set()
|
||||
for name, loaded_weight in weights:
|
||||
param = params_dict[name]
|
||||
weight_loader = getattr(param, "weight_loader", default_weight_loader)
|
||||
weight_loader(param, loaded_weight)
|
||||
loaded_params.add(name)
|
||||
return loaded_params
|
||||
loader = AutoWeightsLoader(self)
|
||||
return loader.load_weights(weights)
|
||||
|
||||
@@ -14,10 +14,7 @@ from vllm.config import CacheConfig, ModelConfig, VllmConfig
|
||||
from vllm.distributed import get_tensor_model_parallel_world_size
|
||||
from vllm.distributed.parallel_state import get_pp_group
|
||||
from vllm.model_executor.layers.attention import Attention
|
||||
from vllm.model_executor.layers.fused_moe import (
|
||||
FusedMoE,
|
||||
fused_moe_make_expert_params_mapping,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe import FusedMoE
|
||||
from vllm.model_executor.layers.layernorm import RMSNorm
|
||||
from vllm.model_executor.layers.linear import (
|
||||
QKVParallelLinear,
|
||||
@@ -38,7 +35,6 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
|
||||
ParallelLMHead,
|
||||
VocabParallelEmbedding,
|
||||
)
|
||||
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
|
||||
from vllm.model_executor.models.llama import LlamaMLP as JambaMLP
|
||||
from vllm.sequence import IntermediateTensors
|
||||
|
||||
@@ -52,7 +48,6 @@ from .interfaces import (
|
||||
from .utils import (
|
||||
AutoWeightsLoader,
|
||||
WeightsMapper,
|
||||
is_pp_missing_parameter,
|
||||
make_empty_intermediate_tensors_factory,
|
||||
make_layers,
|
||||
maybe_prefix,
|
||||
@@ -378,87 +373,6 @@ class JambaModel(nn.Module):
|
||||
hidden_states, _ = self.final_layernorm(hidden_states, residual)
|
||||
return hidden_states
|
||||
|
||||
def get_expert_mapping(self) -> list[tuple[str, str, int, str]]:
|
||||
# Params for weights, fp8 weight scales, fp8 activation scales
|
||||
# (param_name, weight_name, expert_id, shard_id)
|
||||
return fused_moe_make_expert_params_mapping(
|
||||
self,
|
||||
ckpt_gate_proj_name="gate_proj",
|
||||
ckpt_down_proj_name="down_proj",
|
||||
ckpt_up_proj_name="up_proj",
|
||||
num_experts=self.config.num_experts,
|
||||
)
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
stacked_params_mapping = [
|
||||
# (param_name, shard_name, shard_id)
|
||||
("qkv_proj", "q_proj", "q"),
|
||||
("qkv_proj", "k_proj", "k"),
|
||||
("qkv_proj", "v_proj", "v"),
|
||||
(".gate_up_proj", ".gate_proj", 0),
|
||||
(".gate_up_proj", ".up_proj", 1),
|
||||
]
|
||||
|
||||
params_dict = dict(self.named_parameters())
|
||||
loaded_params: set[str] = set()
|
||||
expert_params_mapping = self.get_expert_mapping()
|
||||
for name, loaded_weight in weights:
|
||||
if "rotary_emb.inv_freq" in name:
|
||||
continue
|
||||
for param_name, weight_name, shard_id in stacked_params_mapping:
|
||||
if weight_name not in name:
|
||||
continue
|
||||
if "experts" in name:
|
||||
continue
|
||||
name = name.replace(weight_name, param_name)
|
||||
# Skip loading extra bias for GPTQ models.
|
||||
if name.endswith(".bias") and name not in params_dict:
|
||||
continue
|
||||
# Skip layers on other devices.
|
||||
if is_pp_missing_parameter(name, self):
|
||||
continue
|
||||
param = params_dict[name]
|
||||
weight_loader = param.weight_loader
|
||||
weight_loader(param, loaded_weight, shard_id)
|
||||
break
|
||||
else:
|
||||
for (
|
||||
param_name,
|
||||
weight_name,
|
||||
expert_id,
|
||||
shard_id,
|
||||
) in expert_params_mapping:
|
||||
if weight_name not in name:
|
||||
continue
|
||||
|
||||
if is_pp_missing_parameter(name, self):
|
||||
continue
|
||||
name = name.replace(weight_name, param_name)
|
||||
param = params_dict[name]
|
||||
weight_loader = param.weight_loader
|
||||
weight_loader(
|
||||
param,
|
||||
loaded_weight,
|
||||
name,
|
||||
shard_id=shard_id,
|
||||
expert_id=expert_id,
|
||||
)
|
||||
break
|
||||
else:
|
||||
# Skip loading extra bias for GPTQ models.
|
||||
if name.endswith(".bias") and name not in params_dict:
|
||||
continue
|
||||
if is_pp_missing_parameter(name, self):
|
||||
continue
|
||||
|
||||
param = params_dict[name]
|
||||
weight_loader = getattr(
|
||||
param, "weight_loader", default_weight_loader
|
||||
)
|
||||
weight_loader(param, loaded_weight)
|
||||
loaded_params.add(name)
|
||||
return loaded_params
|
||||
|
||||
|
||||
class JambaForCausalLM(
|
||||
nn.Module,
|
||||
@@ -470,6 +384,14 @@ class JambaForCausalLM(
|
||||
):
|
||||
hf_to_vllm_mapper = WeightsMapper(
|
||||
orig_to_new_substr={".self_attn.": ".", ".A_log": ".A"},
|
||||
orig_to_new_stacked={
|
||||
".q_proj": (".qkv_proj", "q"),
|
||||
".k_proj": (".qkv_proj", "k"),
|
||||
".v_proj": (".qkv_proj", "v"),
|
||||
# .experts.gate_up_proj must be handled by MoERunner.load_weights for EP
|
||||
".feed_forward.gate_proj": (".feed_forward.gate_up_proj", 0),
|
||||
".feed_forward.up_proj": (".feed_forward.gate_up_proj", 1),
|
||||
},
|
||||
)
|
||||
packed_modules_mapping = {
|
||||
"qkv_proj": [
|
||||
@@ -577,9 +499,6 @@ class JambaForCausalLM(
|
||||
loader = AutoWeightsLoader(self)
|
||||
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
|
||||
|
||||
def get_expert_mapping(self) -> list[tuple[str, str, int, str]]:
|
||||
return self.model.get_expert_mapping()
|
||||
|
||||
|
||||
class JambaForSequenceClassification(JambaForCausalLM):
|
||||
is_pooling_model = True
|
||||
|
||||
@@ -52,6 +52,7 @@ from .interfaces import HasInnerState, IsHybrid, MixtureOfExperts, SupportsPP
|
||||
from .utils import (
|
||||
AutoWeightsLoader,
|
||||
PPMissingLayer,
|
||||
get_spec_layer_idx_from_weight_name,
|
||||
is_pp_missing_parameter,
|
||||
make_layers,
|
||||
maybe_prefix,
|
||||
@@ -643,16 +644,3 @@ class KimiLinearForCausalLM(
|
||||
skip_prefixes=(["lm_head."] if self.config.tie_word_embeddings else None),
|
||||
)
|
||||
return loader.load_weights(weights)
|
||||
|
||||
|
||||
def get_spec_layer_idx_from_weight_name(
|
||||
config: KimiLinearConfig, weight_name: str
|
||||
) -> int | None:
|
||||
if hasattr(config, "num_nextn_predict_layers") and (
|
||||
config.num_nextn_predict_layers > 0
|
||||
):
|
||||
layer_idx = config.num_hidden_layers
|
||||
for i in range(config.num_nextn_predict_layers):
|
||||
if weight_name.startswith(f"model.layers.{layer_idx + i}."):
|
||||
return layer_idx + i
|
||||
return None
|
||||
|
||||
@@ -2,8 +2,7 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Inference-only Laguna model compatible with HuggingFace weights."""
|
||||
|
||||
import typing
|
||||
from collections.abc import Callable, Iterable
|
||||
from collections.abc import Iterable
|
||||
from itertools import islice
|
||||
|
||||
import torch
|
||||
@@ -20,10 +19,7 @@ from vllm.distributed import (
|
||||
)
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.attention import Attention
|
||||
from vllm.model_executor.layers.fused_moe import (
|
||||
FusedMoE,
|
||||
fused_moe_make_expert_params_mapping,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe import FusedMoE
|
||||
from vllm.model_executor.layers.layernorm import RMSNorm
|
||||
from vllm.model_executor.layers.linear import (
|
||||
ColumnParallelLinear,
|
||||
@@ -38,10 +34,6 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
|
||||
ParallelLMHead,
|
||||
VocabParallelEmbedding,
|
||||
)
|
||||
from vllm.model_executor.model_loader.weight_utils import (
|
||||
default_weight_loader,
|
||||
maybe_remap_kv_scale_name,
|
||||
)
|
||||
from vllm.model_executor.models.interfaces import (
|
||||
EagleModelMixin,
|
||||
SupportsEagle3,
|
||||
@@ -51,8 +43,8 @@ from vllm.model_executor.models.interfaces import (
|
||||
from vllm.model_executor.models.utils import (
|
||||
AutoWeightsLoader,
|
||||
PPMissingLayer,
|
||||
WeightsMapper,
|
||||
extract_layer_index,
|
||||
is_pp_missing_parameter,
|
||||
make_empty_intermediate_tensors_factory,
|
||||
make_layers,
|
||||
maybe_prefix,
|
||||
@@ -575,6 +567,14 @@ class LagunaDecoderLayer(nn.Module):
|
||||
|
||||
@support_torch_compile
|
||||
class LagunaModel(nn.Module, EagleModelMixin):
|
||||
hf_to_vllm_mapper = WeightsMapper(
|
||||
orig_to_new_stacked={
|
||||
".q_proj": (".qkv_proj", "q"),
|
||||
".k_proj": (".qkv_proj", "k"),
|
||||
".v_proj": (".qkv_proj", "v"),
|
||||
}
|
||||
)
|
||||
|
||||
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
|
||||
super().__init__()
|
||||
|
||||
@@ -675,167 +675,41 @@ class LagunaModel(nn.Module, EagleModelMixin):
|
||||
return hidden_states, aux_hidden_states
|
||||
return hidden_states
|
||||
|
||||
def get_expert_mapping(self) -> list[tuple[str, str, int, str]]:
|
||||
"""Get expert parameter mapping for weight loading.
|
||||
|
||||
Returns mapping tuples of (param_name, weight_name, expert_id, shard_id)
|
||||
that handle both weights and quantization scales.
|
||||
"""
|
||||
return fused_moe_make_expert_params_mapping(
|
||||
self,
|
||||
ckpt_gate_proj_name="gate_proj",
|
||||
ckpt_down_proj_name="down_proj",
|
||||
ckpt_up_proj_name="up_proj",
|
||||
num_experts=self.config.num_experts,
|
||||
num_redundant_experts=self.num_redundant_experts,
|
||||
)
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
stacked_params_mapping = [
|
||||
# (param_name, shard_name, shard_id)
|
||||
("qkv_proj", "q_proj", "q"),
|
||||
("qkv_proj", "k_proj", "k"),
|
||||
("qkv_proj", "v_proj", "v"),
|
||||
# gate_proj and up_proj are loaded as separate Linears (see
|
||||
# LagunaMLP) so no merge entry is needed here.
|
||||
]
|
||||
|
||||
# Suffixes to skip for GPTQ/modelopt models if param doesn't exist
|
||||
ignore_suffixes = (
|
||||
".bias",
|
||||
"_bias",
|
||||
".k_scale",
|
||||
"_k_scale",
|
||||
".v_scale",
|
||||
"_v_scale",
|
||||
".weight_scale",
|
||||
"_weight_scale",
|
||||
".input_scale",
|
||||
"_input_scale",
|
||||
)
|
||||
|
||||
params_dict = dict(self.named_parameters())
|
||||
loaded_params: set[str] = set()
|
||||
expert_params_mapping = self.get_expert_mapping()
|
||||
|
||||
def _slice_sink(
|
||||
self, weights: Iterable[tuple[str, torch.Tensor]]
|
||||
) -> Iterable[tuple[str, torch.Tensor]]:
|
||||
tp_size = get_tensor_model_parallel_world_size()
|
||||
tp_rank = get_tensor_model_parallel_rank()
|
||||
|
||||
for name, loaded_weight in weights:
|
||||
# Handle attention sinks (distributed across ranks). Derive the
|
||||
# per-rank slice from the parameter's own shape so per-layer
|
||||
# variations in head count are handled correctly.
|
||||
if "sink" in name:
|
||||
param = params_dict.get(name)
|
||||
if param is not None:
|
||||
layer_heads_per_rank = param.shape[0]
|
||||
layer_head_start = tp_rank * layer_heads_per_rank
|
||||
narrow_weight = loaded_weight.narrow(
|
||||
0, layer_head_start, layer_heads_per_rank
|
||||
)
|
||||
param.data.copy_(narrow_weight)
|
||||
loaded_params.add(name)
|
||||
continue
|
||||
heads_per_rank = loaded_weight.shape[0] // tp_size
|
||||
loaded_weight = loaded_weight.narrow(
|
||||
0, tp_rank * heads_per_rank, heads_per_rank
|
||||
)
|
||||
yield name, loaded_weight
|
||||
|
||||
# Handle stacked params (QKV, gate_up for
|
||||
# non-expert layers and shared_expert)
|
||||
for param_name, weight_name, shard_id in stacked_params_mapping:
|
||||
if weight_name not in name:
|
||||
continue
|
||||
# Skip expert weights - handled below via expert_params_mapping
|
||||
if "mlp.experts" in name and "shared_expert" not in name:
|
||||
continue
|
||||
name = name.replace(weight_name, param_name)
|
||||
|
||||
if name.endswith(ignore_suffixes) and name not in params_dict:
|
||||
continue
|
||||
if is_pp_missing_parameter(name, self):
|
||||
continue
|
||||
# Remap FP8 kv_scale names for backwards compatibility
|
||||
if name.endswith("scale"):
|
||||
name = maybe_remap_kv_scale_name(name, params_dict)
|
||||
if name is None:
|
||||
continue
|
||||
if name not in params_dict:
|
||||
continue
|
||||
|
||||
param = params_dict[name]
|
||||
weight_loader = getattr(param, "weight_loader", default_weight_loader)
|
||||
if weight_loader == default_weight_loader:
|
||||
weight_loader(param, loaded_weight)
|
||||
else:
|
||||
weight_loader(param, loaded_weight, shard_id)
|
||||
loaded_params.add(name)
|
||||
break
|
||||
else:
|
||||
# Try expert params mapping (handles weights + quantization scales)
|
||||
is_expert_weight = False
|
||||
for mapping in expert_params_mapping:
|
||||
param_name, weight_name, expert_id, shard_id = mapping
|
||||
if weight_name not in name:
|
||||
continue
|
||||
|
||||
# Mark as expert weight so we skip regular loading below
|
||||
is_expert_weight = True
|
||||
|
||||
# Create mapped name without modifying original
|
||||
name_mapped = name.replace(weight_name, param_name)
|
||||
|
||||
if is_pp_missing_parameter(name_mapped, self):
|
||||
continue
|
||||
if (
|
||||
name_mapped.endswith(ignore_suffixes)
|
||||
and name_mapped not in params_dict
|
||||
):
|
||||
continue
|
||||
if name_mapped not in params_dict:
|
||||
continue
|
||||
|
||||
param = params_dict[name_mapped]
|
||||
# Use return_success to handle expert parallelism correctly
|
||||
weight_loader = typing.cast(
|
||||
Callable[..., bool], param.weight_loader
|
||||
)
|
||||
success = weight_loader(
|
||||
param,
|
||||
loaded_weight,
|
||||
name_mapped,
|
||||
shard_id=shard_id,
|
||||
expert_id=expert_id,
|
||||
return_success=True,
|
||||
)
|
||||
if success:
|
||||
loaded_params.add(name_mapped)
|
||||
break
|
||||
else:
|
||||
# Expert weight not mapped to this rank - skip
|
||||
if is_expert_weight:
|
||||
continue
|
||||
|
||||
# Remap kv_scale names before the ignore_suffixes filter:
|
||||
# the suffix list includes .k_scale/.v_scale, so filtering
|
||||
# first drops the checkpoint key before remap can rewrite
|
||||
# it to the .attn.* name that exists in params_dict.
|
||||
name = maybe_remap_kv_scale_name(name, params_dict)
|
||||
if name is None:
|
||||
continue
|
||||
|
||||
if name.endswith(ignore_suffixes) and name not in params_dict:
|
||||
continue
|
||||
|
||||
if is_pp_missing_parameter(name, self):
|
||||
continue
|
||||
|
||||
if name not in params_dict:
|
||||
continue
|
||||
|
||||
param = params_dict[name]
|
||||
weight_loader = getattr(
|
||||
param, "weight_loader", default_weight_loader
|
||||
)
|
||||
weight_loader(param, loaded_weight)
|
||||
loaded_params.add(name)
|
||||
|
||||
return loaded_params
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
loader = AutoWeightsLoader(
|
||||
self,
|
||||
ignore_unexpected_suffixes=[
|
||||
".bias",
|
||||
"_bias",
|
||||
".k_scale",
|
||||
"_k_scale",
|
||||
".v_scale",
|
||||
"_v_scale",
|
||||
".weight_scale",
|
||||
"_weight_scale",
|
||||
".input_scale",
|
||||
"_input_scale",
|
||||
],
|
||||
)
|
||||
return loader.load_weights(
|
||||
self._slice_sink(weights), mapper=self.hf_to_vllm_mapper
|
||||
)
|
||||
|
||||
|
||||
class LagunaForCausalLM(nn.Module, SupportsPP, SupportsLoRA, SupportsEagle3):
|
||||
@@ -892,9 +766,6 @@ class LagunaForCausalLM(nn.Module, SupportsPP, SupportsLoRA, SupportsEagle3):
|
||||
logits = self.logits_processor(self.lm_head, hidden_states)
|
||||
return logits
|
||||
|
||||
def get_expert_mapping(self) -> list[tuple[str, str, int, str]]:
|
||||
return self.model.get_expert_mapping()
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
loader = AutoWeightsLoader(
|
||||
self,
|
||||
|
||||
@@ -15,10 +15,7 @@ from vllm.distributed import (
|
||||
)
|
||||
from vllm.model_executor.layers.activation import SiluAndMul
|
||||
from vllm.model_executor.layers.attention import Attention
|
||||
from vllm.model_executor.layers.fused_moe import (
|
||||
FusedMoE,
|
||||
fused_moe_make_expert_params_mapping,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe import FusedMoE
|
||||
from vllm.model_executor.layers.layernorm import RMSNorm
|
||||
from vllm.model_executor.layers.linear import (
|
||||
MergedColumnParallelLinear,
|
||||
@@ -40,10 +37,6 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
|
||||
ParallelLMHead,
|
||||
VocabParallelEmbedding,
|
||||
)
|
||||
from vllm.model_executor.model_loader.weight_utils import (
|
||||
default_weight_loader,
|
||||
maybe_remap_moe_expert_param_name,
|
||||
)
|
||||
from vllm.sequence import IntermediateTensors
|
||||
from vllm.transformers_utils.configs.lfm2_moe import Lfm2MoeConfig
|
||||
|
||||
@@ -60,7 +53,6 @@ from .utils import (
|
||||
PPMissingLayer,
|
||||
WeightsMapper,
|
||||
extract_layer_index,
|
||||
is_pp_missing_parameter,
|
||||
make_empty_intermediate_tensors_factory,
|
||||
make_layers,
|
||||
maybe_prefix,
|
||||
@@ -167,6 +159,7 @@ class Lfm2MoeSparseMoeBlock(nn.Module):
|
||||
scoring_func="sigmoid",
|
||||
e_score_correction_bias=self.gate.e_score_correction_bias,
|
||||
routed_scaling_factor=self.routed_scaling_factor,
|
||||
ckpt_names=("w1", "w2", "w3"),
|
||||
)
|
||||
|
||||
def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
|
||||
@@ -402,6 +395,22 @@ class Lfm2MoeShortConvDecoderLayer(nn.Module):
|
||||
|
||||
@support_torch_compile
|
||||
class Lfm2MoeModel(nn.Module):
|
||||
hf_to_vllm_mapper = WeightsMapper(
|
||||
orig_to_new_substr={
|
||||
".conv.": ".short_conv.",
|
||||
"expert_bias": "gate.e_score_correction_bias",
|
||||
},
|
||||
orig_to_new_stacked={
|
||||
".q_proj": (".qkv_proj", "q"),
|
||||
".k_proj": (".qkv_proj", "k"),
|
||||
".v_proj": (".qkv_proj", "v"),
|
||||
# Scoped (with trailing dots) to the dense MLP so routed experts.*.w1/w3
|
||||
# (loaded by FusedMoE) are left untouched and .w1 does not match inside .w13
|
||||
".feed_forward.w1.": (".feed_forward.w13.", 0),
|
||||
".feed_forward.w3.": (".feed_forward.w13.", 1),
|
||||
},
|
||||
)
|
||||
|
||||
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
|
||||
super().__init__()
|
||||
|
||||
@@ -487,102 +496,9 @@ class Lfm2MoeModel(nn.Module):
|
||||
hidden_states, _ = self.embedding_norm(hidden_states, residual)
|
||||
return hidden_states
|
||||
|
||||
def get_expert_mapping(self) -> list[tuple[str, str, int, str]]:
|
||||
return fused_moe_make_expert_params_mapping(
|
||||
self,
|
||||
ckpt_gate_proj_name="w1",
|
||||
ckpt_down_proj_name="w2",
|
||||
ckpt_up_proj_name="w3",
|
||||
num_experts=self.config.num_experts,
|
||||
num_redundant_experts=self.num_redundant_experts,
|
||||
)
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
stacked_params_mapping = [
|
||||
(".qkv_proj", ".q_proj", "q"),
|
||||
(".qkv_proj", ".k_proj", "k"),
|
||||
(".qkv_proj", ".v_proj", "v"),
|
||||
(".w13", ".w1", 0),
|
||||
(".w13", ".w3", 1),
|
||||
]
|
||||
params_dict = dict(self.named_parameters())
|
||||
loaded_params: set[str] = set()
|
||||
expert_params_mapping = self.get_expert_mapping()
|
||||
for name, loaded_weight in weights:
|
||||
if "expert_bias" in name:
|
||||
name = name.replace("expert_bias", "gate.e_score_correction_bias")
|
||||
|
||||
if ".conv." in name:
|
||||
name = name.replace(".conv.", ".short_conv.", 1)
|
||||
|
||||
for param_name, weight_name, shard_id in stacked_params_mapping:
|
||||
# Skip non-stacked layers and experts (experts handled below).
|
||||
# Use segment-boundary matching (trailing dot) to prevent
|
||||
# e.g. ".w1" from matching inside ".w13" in pre-fused keys.
|
||||
if weight_name + "." not in name:
|
||||
continue
|
||||
|
||||
if ("feed_forward.experts." in name) and name not in params_dict:
|
||||
continue
|
||||
name = name.replace(weight_name + ".", param_name + ".")
|
||||
# Skip loading extra bias for GPTQ models.
|
||||
if (
|
||||
name.endswith(".bias") or name.endswith("_bias")
|
||||
) and name not in params_dict:
|
||||
continue
|
||||
# Skip layers on other devices.
|
||||
if is_pp_missing_parameter(name, self):
|
||||
continue
|
||||
|
||||
param = params_dict[name]
|
||||
weight_loader = param.weight_loader
|
||||
weight_loader(param, loaded_weight, shard_id)
|
||||
break
|
||||
else:
|
||||
for mapping in expert_params_mapping:
|
||||
param_name, weight_name, expert_id, shard_id = mapping
|
||||
|
||||
if weight_name not in name:
|
||||
continue
|
||||
|
||||
name = name.replace(weight_name, param_name)
|
||||
# Skip layers on other devices.
|
||||
if is_pp_missing_parameter(name, self):
|
||||
continue
|
||||
|
||||
# Skip loading extra bias for GPTQ models.
|
||||
if (
|
||||
name.endswith(".bias") or name.endswith("_bias")
|
||||
) and name not in params_dict:
|
||||
continue
|
||||
param = params_dict[name]
|
||||
|
||||
weight_loader = param.weight_loader
|
||||
weight_loader(
|
||||
param,
|
||||
loaded_weight,
|
||||
name,
|
||||
shard_id=shard_id,
|
||||
expert_id=expert_id,
|
||||
)
|
||||
break
|
||||
else:
|
||||
# Skip loading extra bias for GPTQ models.
|
||||
if (
|
||||
name.endswith(".bias") or name.endswith("_bias")
|
||||
) and name not in params_dict:
|
||||
continue
|
||||
# Skip layers on other devices.
|
||||
if is_pp_missing_parameter(name, self):
|
||||
continue
|
||||
name = maybe_remap_moe_expert_param_name(name, params_dict)
|
||||
param = params_dict[name]
|
||||
weight_loader = getattr(
|
||||
param, "weight_loader", default_weight_loader
|
||||
)
|
||||
weight_loader(param, loaded_weight)
|
||||
loaded_params.add(name)
|
||||
return loaded_params
|
||||
loader = AutoWeightsLoader(self, ignore_unexpected_suffixes=[".bias", "_bias"])
|
||||
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
|
||||
|
||||
|
||||
class Lfm2MoeForCausalLM(
|
||||
@@ -763,6 +679,3 @@ class Lfm2MoeForCausalLM(
|
||||
skip_prefixes=(["lm_head."] if self.config.tie_word_embeddings else None),
|
||||
)
|
||||
return loader.load_weights(weights)
|
||||
|
||||
def get_expert_mapping(self) -> list[tuple[str, str, int, str]]:
|
||||
return self.model.get_expert_mapping()
|
||||
|
||||
@@ -46,7 +46,6 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
|
||||
ParallelLMHead,
|
||||
VocabParallelEmbedding,
|
||||
)
|
||||
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
|
||||
from vllm.model_executor.models.module_mapping import MultiModelKeys
|
||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
from vllm.multimodal.inputs import (
|
||||
@@ -76,7 +75,6 @@ from .interfaces import (
|
||||
from .utils import (
|
||||
AutoWeightsLoader,
|
||||
WeightsMapper,
|
||||
is_pp_missing_parameter,
|
||||
make_empty_intermediate_tensors_factory,
|
||||
make_layers,
|
||||
maybe_prefix,
|
||||
@@ -888,20 +886,7 @@ class MolmoModel(nn.Module, SupportsQuant):
|
||||
return hidden_states
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
params_dict = dict(self.named_parameters())
|
||||
loaded_params: set[str] = set()
|
||||
|
||||
for name, loaded_weight in weights:
|
||||
if name.endswith(".bias") and name not in params_dict:
|
||||
continue
|
||||
if is_pp_missing_parameter(name, self):
|
||||
continue
|
||||
|
||||
param = params_dict[name]
|
||||
weight_loader = getattr(param, "weight_loader", default_weight_loader)
|
||||
weight_loader(param, loaded_weight)
|
||||
loaded_params.add(name)
|
||||
return loaded_params
|
||||
return AutoWeightsLoader(self).load_weights(weights)
|
||||
|
||||
|
||||
def _lowest_multiple(x: int, k: int) -> int:
|
||||
|
||||
@@ -51,7 +51,6 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
|
||||
ParallelLMHead,
|
||||
VocabParallelEmbedding,
|
||||
)
|
||||
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
|
||||
from vllm.model_executor.models.module_mapping import MultiModelKeys
|
||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
from vllm.multimodal.inputs import (
|
||||
@@ -89,7 +88,6 @@ from .utils import (
|
||||
WeightsMapper,
|
||||
_merge_multimodal_embeddings,
|
||||
extract_layer_index,
|
||||
is_pp_missing_parameter,
|
||||
make_empty_intermediate_tensors_factory,
|
||||
make_layers,
|
||||
maybe_prefix,
|
||||
@@ -1221,20 +1219,7 @@ class Molmo2TextModel(nn.Module, SupportsQuant):
|
||||
return hidden_states
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
params_dict = dict(self.named_parameters())
|
||||
loaded_params: set[str] = set()
|
||||
|
||||
for name, loaded_weight in weights:
|
||||
if name.endswith(".bias") and name not in params_dict:
|
||||
continue
|
||||
if is_pp_missing_parameter(name, self):
|
||||
continue
|
||||
|
||||
param = params_dict[name]
|
||||
weight_loader = getattr(param, "weight_loader", default_weight_loader)
|
||||
weight_loader(param, loaded_weight)
|
||||
loaded_params.add(name)
|
||||
return loaded_params
|
||||
return AutoWeightsLoader(self).load_weights(weights)
|
||||
|
||||
|
||||
def get_patches_grid_size(
|
||||
|
||||
@@ -18,8 +18,7 @@
|
||||
# limitations under the License.
|
||||
"""Inference-only NemotronH model."""
|
||||
|
||||
import typing
|
||||
from collections.abc import Callable, Iterable, Mapping
|
||||
from collections.abc import Iterable, Mapping
|
||||
from itertools import islice
|
||||
|
||||
import torch
|
||||
@@ -59,10 +58,6 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
|
||||
ParallelLMHead,
|
||||
VocabParallelEmbedding,
|
||||
)
|
||||
from vllm.model_executor.model_loader.weight_utils import (
|
||||
default_weight_loader,
|
||||
maybe_remap_kv_scale_name,
|
||||
)
|
||||
from vllm.model_executor.models.interfaces import (
|
||||
EagleModelMixin,
|
||||
HasInnerState,
|
||||
@@ -78,7 +73,6 @@ from vllm.model_executor.models.interfaces import (
|
||||
from vllm.model_executor.models.utils import (
|
||||
AutoWeightsLoader,
|
||||
WeightsMapper,
|
||||
is_pp_missing_parameter,
|
||||
make_empty_intermediate_tensors_factory,
|
||||
make_layers,
|
||||
maybe_prefix,
|
||||
@@ -222,6 +216,7 @@ class NemotronHMoE(nn.Module):
|
||||
intermediate_size=config.moe_intermediate_size,
|
||||
renormalize=config.norm_topk_prob,
|
||||
quant_config=quant_config,
|
||||
ckpt_names=("up_proj", "down_proj", ""),
|
||||
use_grouped_topk=True,
|
||||
num_expert_group=config.n_group,
|
||||
topk_group=config.topk_group,
|
||||
@@ -680,9 +675,12 @@ class NemotronHModel(nn.Module, EagleModelMixin):
|
||||
return max_experts
|
||||
|
||||
def get_expert_mapping(self) -> list[tuple[str, str, int, str]]:
|
||||
# Consumed by `get_moe_expert_mapping` (bitsandbytes / LoRA); the main
|
||||
# weight load is self-served by each `RoutedExperts` layer. Sized to the
|
||||
# MAX expert count so heterogeneous puzzle models load every expert.
|
||||
if self.has_moe:
|
||||
# (param_name, weight_name, expert_id, shard_id)
|
||||
expert_params_mapping = fused_moe_make_expert_params_mapping(
|
||||
return fused_moe_make_expert_params_mapping(
|
||||
# - FusedMoe.w1 (aka gate_proj) should be up_proj since that's
|
||||
# what the activation is applied to
|
||||
# - FusedMoe.w3 (aka up_proj) should be ignored since we're
|
||||
@@ -694,102 +692,9 @@ class NemotronHModel(nn.Module, EagleModelMixin):
|
||||
num_experts=self._get_max_n_routed_experts(),
|
||||
num_redundant_experts=getattr(self, "num_redundant_experts", 0),
|
||||
)
|
||||
return expert_params_mapping
|
||||
|
||||
return []
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
stacked_params_mapping = [
|
||||
# (param_name, shard_name, shard_id)
|
||||
("qkv_proj", "q_proj", "q"),
|
||||
("qkv_proj", "k_proj", "k"),
|
||||
("qkv_proj", "v_proj", "v"),
|
||||
]
|
||||
|
||||
expert_params_mapping = self.get_expert_mapping()
|
||||
|
||||
params_dict = dict(self.named_parameters())
|
||||
loaded_params: set[str] = set()
|
||||
for name, loaded_weight in weights:
|
||||
if "scale" in name or "zero_point" in name:
|
||||
# Remapping the name of FP8 kv-scale.
|
||||
name = maybe_remap_kv_scale_name(name, params_dict)
|
||||
if name is None:
|
||||
continue
|
||||
|
||||
# Skip MTP/spec decode layers early (before stacked params mapping)
|
||||
if name.startswith("mtp."):
|
||||
continue
|
||||
|
||||
# load stacked params
|
||||
for param_name, weight_name, shard_id in stacked_params_mapping:
|
||||
if weight_name not in name:
|
||||
continue
|
||||
name = name.replace(weight_name, param_name)
|
||||
# Skip loading extra bias for GPTQ models.
|
||||
if name.endswith(".bias") and name not in params_dict:
|
||||
continue
|
||||
|
||||
if is_pp_missing_parameter(name, self):
|
||||
continue
|
||||
|
||||
param = params_dict[name]
|
||||
weight_loader = param.weight_loader
|
||||
weight_loader(param, loaded_weight, shard_id)
|
||||
break
|
||||
|
||||
# load other params
|
||||
else:
|
||||
is_expert_weight = False
|
||||
for mapping in expert_params_mapping:
|
||||
param_name, weight_name, expert_id, shard_id = mapping
|
||||
if weight_name not in name:
|
||||
continue
|
||||
|
||||
# Anyway, this is an expert weight and should not be
|
||||
# attempted to load as other weights later
|
||||
is_expert_weight = True
|
||||
|
||||
# Do not modify `name` since the loop may continue here
|
||||
# Instead, create a new variable
|
||||
name_mapped = name.replace(weight_name, param_name)
|
||||
|
||||
if is_pp_missing_parameter(name_mapped, self):
|
||||
continue
|
||||
param = params_dict[name_mapped]
|
||||
# We should ask the weight loader to return success or not
|
||||
# here since otherwise we may skip experts with other
|
||||
# available replicas.
|
||||
weight_loader = typing.cast(
|
||||
Callable[..., bool], param.weight_loader
|
||||
)
|
||||
success = weight_loader(
|
||||
param,
|
||||
loaded_weight,
|
||||
name_mapped,
|
||||
shard_id=shard_id,
|
||||
expert_id=expert_id,
|
||||
return_success=True,
|
||||
)
|
||||
if success:
|
||||
name = name_mapped
|
||||
break
|
||||
else:
|
||||
if is_expert_weight:
|
||||
continue
|
||||
|
||||
if is_pp_missing_parameter(name, self):
|
||||
continue
|
||||
|
||||
param = params_dict[name]
|
||||
weight_loader = getattr(
|
||||
param, "weight_loader", default_weight_loader
|
||||
)
|
||||
weight_loader(param, loaded_weight)
|
||||
|
||||
loaded_params.add(name)
|
||||
return loaded_params
|
||||
|
||||
|
||||
class NemotronHForCausalLM(
|
||||
nn.Module,
|
||||
@@ -809,6 +714,11 @@ class NemotronHForCausalLM(
|
||||
hf_to_vllm_mapper = WeightsMapper(
|
||||
orig_to_new_prefix={"backbone": "model"},
|
||||
orig_to_new_substr={"A_log": "A", "embeddings": "embed_tokens"},
|
||||
orig_to_new_stacked={
|
||||
".q_proj": (".qkv_proj", "q"),
|
||||
".k_proj": (".qkv_proj", "k"),
|
||||
".v_proj": (".qkv_proj", "v"),
|
||||
},
|
||||
)
|
||||
|
||||
packed_modules_mapping = {
|
||||
|
||||
@@ -63,16 +63,13 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
|
||||
ParallelLMHead,
|
||||
VocabParallelEmbedding,
|
||||
)
|
||||
from vllm.model_executor.model_loader.weight_utils import (
|
||||
default_weight_loader,
|
||||
)
|
||||
from vllm.sequence import IntermediateTensors
|
||||
|
||||
from .interfaces import HasInnerState, IsHybrid, SupportsLoRA, SupportsPP
|
||||
from .utils import (
|
||||
AutoWeightsLoader,
|
||||
WeightsMapper,
|
||||
extract_layer_index,
|
||||
is_pp_missing_parameter,
|
||||
make_empty_intermediate_tensors_factory,
|
||||
make_layers,
|
||||
maybe_prefix,
|
||||
@@ -302,6 +299,23 @@ class OlmoHybridDecoderLayer(nn.Module):
|
||||
|
||||
@support_torch_compile
|
||||
class OlmoHybridModel(nn.Module):
|
||||
hf_to_vllm_mapper = WeightsMapper(
|
||||
orig_to_new_stacked={
|
||||
".self_attn.q_proj": (".self_attn.qkv_proj", "q"),
|
||||
".self_attn.k_proj": (".self_attn.qkv_proj", "k"),
|
||||
".self_attn.v_proj": (".self_attn.qkv_proj", "v"),
|
||||
".linear_attn.q_proj": (".linear_attn.in_proj_qkvg", 0),
|
||||
".linear_attn.k_proj": (".linear_attn.in_proj_qkvg", 1),
|
||||
".linear_attn.v_proj": (".linear_attn.in_proj_qkvg", 2),
|
||||
".linear_attn.g_proj": (".linear_attn.in_proj_qkvg", 3),
|
||||
".q_conv1d": (".conv1d", 0),
|
||||
".k_conv1d": (".conv1d", 1),
|
||||
".v_conv1d": (".conv1d", 2),
|
||||
".gate_proj": (".gate_up_proj", 0),
|
||||
".up_proj": (".gate_up_proj", 1),
|
||||
}
|
||||
)
|
||||
|
||||
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
|
||||
super().__init__()
|
||||
self.config = vllm_config.model_config.hf_config
|
||||
@@ -359,79 +373,8 @@ class OlmoHybridModel(nn.Module):
|
||||
return hidden_states
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
stacked_params_mapping = [
|
||||
("qkv_proj", "q_proj", "q"),
|
||||
("qkv_proj", "k_proj", "k"),
|
||||
("qkv_proj", "v_proj", "v"),
|
||||
("gate_up_proj", "gate_proj", 0),
|
||||
("gate_up_proj", "up_proj", 1),
|
||||
]
|
||||
|
||||
linear_attn_stacked_params_mapping = [
|
||||
("in_proj_qkvg", "q_proj", 0),
|
||||
("in_proj_qkvg", "k_proj", 1),
|
||||
("in_proj_qkvg", "v_proj", 2),
|
||||
("in_proj_qkvg", "g_proj", 3),
|
||||
("conv1d", "q_conv1d", 0),
|
||||
("conv1d", "k_conv1d", 1),
|
||||
("conv1d", "v_conv1d", 2),
|
||||
]
|
||||
|
||||
params_dict = dict(self.named_parameters(remove_duplicate=False))
|
||||
loaded_params: set[str] = set()
|
||||
|
||||
for name, loaded_weight in weights:
|
||||
if is_pp_missing_parameter(name, self):
|
||||
continue
|
||||
|
||||
handled = False
|
||||
|
||||
if "linear_attn" in name:
|
||||
for (
|
||||
param_name,
|
||||
weight_name,
|
||||
shard_id,
|
||||
) in linear_attn_stacked_params_mapping:
|
||||
if weight_name not in name:
|
||||
continue
|
||||
mapped_name = name.replace(weight_name, param_name)
|
||||
if mapped_name.endswith(".bias") and (
|
||||
mapped_name not in params_dict
|
||||
):
|
||||
continue
|
||||
if mapped_name not in params_dict:
|
||||
continue
|
||||
param = params_dict[mapped_name]
|
||||
weight_loader = param.weight_loader
|
||||
weight_loader(param, loaded_weight, shard_id)
|
||||
name = mapped_name
|
||||
handled = True
|
||||
break
|
||||
else:
|
||||
for param_name, weight_name, shard_id in stacked_params_mapping:
|
||||
if weight_name not in name:
|
||||
continue
|
||||
name = name.replace(weight_name, param_name)
|
||||
if name.endswith(".bias") and name not in params_dict:
|
||||
continue
|
||||
if name not in params_dict:
|
||||
continue
|
||||
param = params_dict[name]
|
||||
weight_loader = param.weight_loader
|
||||
weight_loader(param, loaded_weight, shard_id)
|
||||
handled = True
|
||||
break
|
||||
|
||||
if not handled:
|
||||
if name.endswith(".bias") and name not in params_dict:
|
||||
continue
|
||||
if name not in params_dict:
|
||||
continue
|
||||
param = params_dict[name]
|
||||
weight_loader = getattr(param, "weight_loader", default_weight_loader)
|
||||
weight_loader(param, loaded_weight)
|
||||
loaded_params.add(name)
|
||||
return loaded_params
|
||||
loader = AutoWeightsLoader(self)
|
||||
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
|
||||
|
||||
|
||||
class OlmoHybridForCausalLM(
|
||||
|
||||
@@ -21,8 +21,7 @@
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import typing
|
||||
from collections.abc import Callable, Iterable
|
||||
from collections.abc import Iterable
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
@@ -44,10 +43,7 @@ from vllm.model_executor.layers.attention import (
|
||||
Attention,
|
||||
StaticSinkAttention,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe import (
|
||||
FusedMoE,
|
||||
fused_moe_make_expert_params_mapping,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe import FusedMoE
|
||||
from vllm.model_executor.layers.layernorm import RMSNorm
|
||||
from vllm.model_executor.layers.linear import (
|
||||
ColumnParallelLinear,
|
||||
@@ -64,10 +60,6 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
|
||||
ParallelLMHead,
|
||||
VocabParallelEmbedding,
|
||||
)
|
||||
from vllm.model_executor.model_loader.weight_utils import (
|
||||
default_weight_loader,
|
||||
maybe_remap_kv_scale_name,
|
||||
)
|
||||
from vllm.model_executor.models.interfaces import (
|
||||
MixtureOfExperts,
|
||||
SupportsLoRA,
|
||||
@@ -76,8 +68,8 @@ from vllm.model_executor.models.interfaces import (
|
||||
from vllm.model_executor.models.utils import (
|
||||
AutoWeightsLoader,
|
||||
PPMissingLayer,
|
||||
WeightsMapper,
|
||||
extract_layer_index,
|
||||
is_pp_missing_parameter,
|
||||
make_empty_intermediate_tensors_factory,
|
||||
make_layers,
|
||||
maybe_prefix,
|
||||
@@ -1056,153 +1048,48 @@ class OpenPanguModel(nn.Module):
|
||||
hidden_states, _ = self.norm(hidden_states, residual)
|
||||
return hidden_states
|
||||
|
||||
def load_attn_mlp_weight(
|
||||
self,
|
||||
attn_mlp_replace_mapping: list[tuple[str, str, int]],
|
||||
params_dict: dict[str, Any],
|
||||
weight_name: str,
|
||||
loaded_weight: torch.Tensor,
|
||||
loaded_params: set[str],
|
||||
) -> bool:
|
||||
for param_name, origin_name, shard_id in attn_mlp_replace_mapping:
|
||||
if origin_name not in weight_name or (
|
||||
("mlp.experts." in weight_name) and weight_name not in params_dict
|
||||
):
|
||||
continue
|
||||
weight_name_mapped = weight_name.replace(origin_name, param_name)
|
||||
if (
|
||||
param_name == "fused_qkv_a_proj"
|
||||
and weight_name_mapped not in params_dict
|
||||
):
|
||||
continue
|
||||
else:
|
||||
weight_name = weight_name_mapped
|
||||
if weight_name.endswith(".bias") and weight_name not in params_dict:
|
||||
continue
|
||||
if is_pp_missing_parameter(weight_name, self):
|
||||
continue
|
||||
|
||||
param = params_dict[weight_name]
|
||||
weight_loader = param.weight_loader
|
||||
weight_loader(param, loaded_weight, shard_id)
|
||||
loaded_params.add(weight_name)
|
||||
return True
|
||||
return False
|
||||
|
||||
def load_expert_weight(
|
||||
self,
|
||||
expert_merge_mapping: list[tuple[str, str, int, str]],
|
||||
params_dict: dict[str, Any],
|
||||
weight_name: str,
|
||||
loaded_weight: torch.Tensor,
|
||||
loaded_params: set[str],
|
||||
flag_dict: dict[str, bool],
|
||||
) -> bool:
|
||||
for mapping in expert_merge_mapping:
|
||||
param_name, origin_name, expert_id, shard_id = mapping
|
||||
if origin_name not in weight_name:
|
||||
continue
|
||||
flag_dict["is_expert_weight"] = True
|
||||
weight_name_mapped = weight_name.replace(origin_name, param_name)
|
||||
if is_pp_missing_parameter(weight_name_mapped, self):
|
||||
continue
|
||||
param = params_dict[weight_name_mapped]
|
||||
weight_loader = typing.cast(Callable[..., bool], param.weight_loader)
|
||||
success = weight_loader(
|
||||
param,
|
||||
loaded_weight,
|
||||
weight_name_mapped,
|
||||
shard_id=shard_id,
|
||||
expert_id=expert_id,
|
||||
return_success=True,
|
||||
)
|
||||
if success:
|
||||
weight_name = weight_name_mapped
|
||||
loaded_params.add(weight_name_mapped)
|
||||
return True
|
||||
return False
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
attn_mlp_replace_mapping = [
|
||||
(".qkv_proj", ".q_proj", "q"),
|
||||
(".qkv_proj", ".k_proj", "k"),
|
||||
(".qkv_proj", ".v_proj", "v"),
|
||||
(".fused_qkv_a_proj", ".q_a_proj", 0),
|
||||
(".fused_qkv_a_proj", ".kv_a_proj_with_mqa", 1),
|
||||
(".gate_up_proj", ".gate_proj", 0),
|
||||
(".gate_up_proj", ".up_proj", 1),
|
||||
]
|
||||
has_experts = hasattr(self.config, "n_routed_experts")
|
||||
if has_experts:
|
||||
expert_merge_mapping = fused_moe_make_expert_params_mapping(
|
||||
self,
|
||||
ckpt_gate_proj_name="gate_proj",
|
||||
ckpt_down_proj_name="down_proj",
|
||||
ckpt_up_proj_name="up_proj",
|
||||
num_experts=self.config.n_routed_experts,
|
||||
num_redundant_experts=self.num_redundant_experts,
|
||||
)
|
||||
|
||||
params_dict = dict(self.named_parameters())
|
||||
loaded_params: set[str] = set()
|
||||
def _filter_spec_layers(
|
||||
self, weights: Iterable[tuple[str, torch.Tensor]]
|
||||
) -> Iterable[tuple[str, torch.Tensor]]:
|
||||
# Skip the MTP (spec-decode) layers that follow the main model's layers.
|
||||
num_mtp = getattr(self.config, "num_nextn_predict_layers", 0)
|
||||
for name, loaded_weight in weights:
|
||||
if "rotary_emb.inv_freq" in name:
|
||||
continue
|
||||
if self.config.tie_word_embeddings and "lm_head.weight" in name:
|
||||
continue
|
||||
|
||||
if (
|
||||
"layers" in name
|
||||
and hasattr(self.config, "num_nextn_predict_layers")
|
||||
and (self.config.num_nextn_predict_layers > 0)
|
||||
):
|
||||
if "layers" in name and num_mtp > 0:
|
||||
layer_idx = int(name.split("layers.")[-1].split(".")[0])
|
||||
mtp_idx = layer_idx - self.config.num_hidden_layers
|
||||
if mtp_idx >= 0 and mtp_idx < self.config.num_nextn_predict_layers:
|
||||
continue # skip spec decode layers for main model
|
||||
|
||||
flag_dict = {"is_expert_weight": False}
|
||||
if (
|
||||
self.load_attn_mlp_weight(
|
||||
attn_mlp_replace_mapping,
|
||||
params_dict,
|
||||
name,
|
||||
loaded_weight,
|
||||
loaded_params,
|
||||
)
|
||||
or has_experts
|
||||
and self.load_expert_weight(
|
||||
expert_merge_mapping,
|
||||
params_dict,
|
||||
name,
|
||||
loaded_weight,
|
||||
loaded_params,
|
||||
flag_dict,
|
||||
)
|
||||
):
|
||||
continue
|
||||
else:
|
||||
if flag_dict["is_expert_weight"]:
|
||||
if 0 <= mtp_idx < num_mtp:
|
||||
continue
|
||||
if name.endswith(".bias") and name not in params_dict:
|
||||
continue
|
||||
name = maybe_remap_kv_scale_name(name, params_dict)
|
||||
if name.endswith("e_score_correction_bias"):
|
||||
name = name.replace(
|
||||
"e_score_correction_bias", "gate.e_score_correction_bias"
|
||||
)
|
||||
if name is None:
|
||||
continue
|
||||
if is_pp_missing_parameter(name, self):
|
||||
continue
|
||||
|
||||
param = params_dict[name]
|
||||
weight_loader = getattr(param, "weight_loader", default_weight_loader)
|
||||
weight_loader(param, loaded_weight)
|
||||
loaded_params.add(name)
|
||||
yield name, loaded_weight
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
# .experts.gate_up_proj must be handled by MoERunner.load_weights for EP
|
||||
stacked: dict[str, tuple[str, int | str]] = {
|
||||
".mlp.gate_proj": (".mlp.gate_up_proj", 0),
|
||||
".mlp.up_proj": (".mlp.gate_up_proj", 1),
|
||||
".shared_experts.gate_proj": (".shared_experts.gate_up_proj", 0),
|
||||
".shared_experts.up_proj": (".shared_experts.gate_up_proj", 1),
|
||||
}
|
||||
# The attention layout is config-dependent: MLA with a fused low-rank
|
||||
# projection, MLA without it, or standard qkv. Only add the stacked
|
||||
# entries whose fused target actually exists as a parameter.
|
||||
param_names = list(dict(self.named_parameters()))
|
||||
if any(".fused_qkv_a_proj." in n for n in param_names):
|
||||
stacked[".q_a_proj"] = (".fused_qkv_a_proj", 0)
|
||||
stacked[".kv_a_proj_with_mqa"] = (".fused_qkv_a_proj", 1)
|
||||
if any(".qkv_proj." in n for n in param_names):
|
||||
stacked[".q_proj"] = (".qkv_proj", "q")
|
||||
stacked[".k_proj"] = (".qkv_proj", "k")
|
||||
stacked[".v_proj"] = (".qkv_proj", "v")
|
||||
mapper = WeightsMapper(
|
||||
orig_to_new_substr={
|
||||
"e_score_correction_bias": "gate.e_score_correction_bias",
|
||||
},
|
||||
orig_to_new_stacked=stacked,
|
||||
)
|
||||
loader = AutoWeightsLoader(self)
|
||||
loaded = loader.load_weights(self._filter_spec_layers(weights), mapper=mapper)
|
||||
self.post_weight_load()
|
||||
return loaded_params
|
||||
return loaded
|
||||
|
||||
def post_weight_load(self) -> None:
|
||||
for name, module in self.named_modules():
|
||||
|
||||
@@ -32,10 +32,7 @@ from vllm.distributed import (
|
||||
)
|
||||
from vllm.model_executor.layers.activation import SiluAndMul
|
||||
from vllm.model_executor.layers.attention import Attention
|
||||
from vllm.model_executor.layers.fused_moe import (
|
||||
FusedMoE,
|
||||
fused_moe_make_expert_params_mapping,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe import FusedMoE
|
||||
from vllm.model_executor.layers.layernorm import RMSNorm
|
||||
from vllm.model_executor.layers.linear import (
|
||||
MergedColumnParallelLinear,
|
||||
@@ -49,14 +46,13 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
|
||||
ParallelLMHead,
|
||||
VocabParallelEmbedding,
|
||||
)
|
||||
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
|
||||
from vllm.sequence import IntermediateTensors
|
||||
|
||||
from .interfaces import MixtureOfExperts, SupportsLoRA, SupportsPP
|
||||
from .utils import (
|
||||
AutoWeightsLoader,
|
||||
PPMissingLayer,
|
||||
is_pp_missing_parameter,
|
||||
WeightsMapper,
|
||||
make_empty_intermediate_tensors_factory,
|
||||
make_layers,
|
||||
maybe_prefix,
|
||||
@@ -74,45 +70,17 @@ def _zero_mean_tensor(t: torch.Tensor) -> torch.Tensor:
|
||||
return t - t.mean()
|
||||
|
||||
|
||||
def _rename_and_normalize_weights(
|
||||
def _normalize_expert_bias(
|
||||
weights: Iterable[tuple[str, torch.Tensor]],
|
||||
) -> Iterator[tuple[str, torch.Tensor]]:
|
||||
"""
|
||||
Translate HuggingFace Param2MoE weight names to vLLM internal names
|
||||
and zero-mean the expert-bias tensor so the router stays balanced.
|
||||
"""Zero-mean the MoE router's per-expert score bias for load balance.
|
||||
|
||||
Mapping table (HF → vLLM):
|
||||
model.word_embeddings.* → model.embed_tokens.*
|
||||
*.attention.query_key_value.* → *.self_attn.qkv_proj.*
|
||||
*.attention.dense.* → *.self_attn.o_proj.*
|
||||
*.attention.query_layernorm.* → *.self_attn.q_layernorm.*
|
||||
*.attention.key_layernorm.* → *.self_attn.k_layernorm.*
|
||||
*.mlp.gate.expert_bias → *.mlp.gate.e_score_correction_bias
|
||||
(also zero-meant for load balance)
|
||||
The rename to ``e_score_correction_bias`` is done by the mapper; only the
|
||||
tensor adjustment lives here, since a WeightsMapper cannot transform data.
|
||||
"""
|
||||
for name, w in weights:
|
||||
# Embedding table
|
||||
name = name.replace("model.word_embeddings.", "model.embed_tokens.")
|
||||
# Fused QKV projection (HF: query_key_value → vLLM: qkv_proj)
|
||||
name = name.replace(".attention.query_key_value.", ".self_attn.qkv_proj.")
|
||||
# Output projection (HF: dense → vLLM: o_proj)
|
||||
name = name.replace(".attention.dense.", ".self_attn.o_proj.")
|
||||
# Per-head query norm
|
||||
name = name.replace(".attention.query_layernorm.", ".self_attn.q_layernorm.")
|
||||
# Per-head key norm
|
||||
name = name.replace(".attention.key_layernorm.", ".self_attn.k_layernorm.")
|
||||
# Catch any remaining .attention. → .self_attn. prefixes
|
||||
# (e.g. future bias params on the projection layers)
|
||||
name = name.replace(".attention.", ".self_attn.")
|
||||
|
||||
# Expert-score bias: rename + zero-mean
|
||||
if name.endswith(".mlp.gate.expert_bias"):
|
||||
name = name.replace(
|
||||
".mlp.gate.expert_bias",
|
||||
".mlp.gate.e_score_correction_bias",
|
||||
)
|
||||
if _is_expert_bias_name(name):
|
||||
w = _zero_mean_tensor(w)
|
||||
|
||||
yield name, w
|
||||
|
||||
|
||||
@@ -123,7 +91,8 @@ class Param2MoEAttention(nn.Module):
|
||||
Notable differences from a vanilla GQA layer:
|
||||
* The checkpoint fuses Q, K, V into a single ``query_key_value`` weight.
|
||||
vLLM receives it already renamed to ``qkv_proj`` by the weight-name
|
||||
translator and splits it during ``load_weights``.
|
||||
translator and loads it directly; ``QKVParallelLinear`` splits the
|
||||
fused ``[Q|K|V]`` tensor internally.
|
||||
* Optional per-head RMS norms on Q and K (``use_qk_norm=True``).
|
||||
"""
|
||||
|
||||
@@ -472,6 +441,28 @@ class Param2MoEDecoderLayer(nn.Module):
|
||||
|
||||
|
||||
class Param2MoEModel(nn.Module):
|
||||
hf_to_vllm_mapper = WeightsMapper(
|
||||
orig_to_new_substr={
|
||||
"word_embeddings.": "embed_tokens.",
|
||||
# Fused query_key_value is renamed to qkv_proj and loaded directly;
|
||||
# QKVParallelLinear splits the fused [Q|K|V] tensor internally.
|
||||
".attention.query_key_value.": ".self_attn.qkv_proj.",
|
||||
".attention.dense.": ".self_attn.o_proj.",
|
||||
".attention.query_layernorm.": ".self_attn.q_layernorm.",
|
||||
".attention.key_layernorm.": ".self_attn.k_layernorm.",
|
||||
# Catch-all for any remaining .attention. names (must come last).
|
||||
".attention.": ".self_attn.",
|
||||
".mlp.gate.expert_bias": ".mlp.gate.e_score_correction_bias",
|
||||
},
|
||||
orig_to_new_stacked={
|
||||
# .experts.gate_up_proj must be handled by MoERunner.load_weights for EP
|
||||
".mlp.gate_proj": (".mlp.gate_up_proj", 0),
|
||||
".mlp.up_proj": (".mlp.gate_up_proj", 1),
|
||||
".shared_experts.gate_proj": (".shared_experts.gate_up_proj", 0),
|
||||
".shared_experts.up_proj": (".shared_experts.gate_up_proj", 1),
|
||||
},
|
||||
)
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
@@ -558,147 +549,9 @@ class Param2MoEModel(nn.Module):
|
||||
self,
|
||||
weights: Iterable[tuple[str, torch.Tensor]],
|
||||
) -> set[str]:
|
||||
"""
|
||||
Custom weight loader for the inner Param2MoEModel.
|
||||
|
||||
Receives weights that have already been renamed/normalised by the
|
||||
outer model and whose ``model.`` prefix has been stripped by
|
||||
``AutoWeightsLoader``. Handles:
|
||||
1. Fused QKV split (query_key_value → qkv_proj q/k/v shards).
|
||||
2. gate_proj + up_proj → gate_up_proj stacking (dense + shared-exp).
|
||||
3. Routed-expert weights via the fused-MoE mapping.
|
||||
4. All remaining weights via their default loader.
|
||||
"""
|
||||
config = self.config
|
||||
num_heads: int = config.num_attention_heads
|
||||
num_kv_heads: int = config.num_key_value_heads
|
||||
head_dim: int = config.head_dim or (config.hidden_size // num_heads)
|
||||
q_split = num_heads * head_dim
|
||||
kv_split = num_kv_heads * head_dim
|
||||
|
||||
stacked_params_mapping = [
|
||||
# (vllm_param_name, ckpt_weight_name, shard_id)
|
||||
("gate_up_proj", "gate_proj", 0),
|
||||
("gate_up_proj", "up_proj", 1),
|
||||
]
|
||||
|
||||
params_dict = dict(self.named_parameters(remove_duplicate=False))
|
||||
loaded_params: set[str] = set()
|
||||
expert_params_mapping = self.get_expert_mapping()
|
||||
|
||||
for name, loaded_weight in weights:
|
||||
# ------------------------------------------------------------------
|
||||
# 1. Fused QKV: split into q / k / v shards for QKVParallelLinear
|
||||
# ------------------------------------------------------------------
|
||||
if name.endswith(".self_attn.qkv_proj.weight"):
|
||||
if name not in params_dict:
|
||||
continue
|
||||
if is_pp_missing_parameter(name, self):
|
||||
continue
|
||||
|
||||
param = params_dict[name]
|
||||
weight_loader = getattr(param, "weight_loader", default_weight_loader)
|
||||
q_w = loaded_weight[:q_split, :]
|
||||
k_w = loaded_weight[q_split : q_split + kv_split, :]
|
||||
v_w = loaded_weight[q_split + kv_split :, :]
|
||||
weight_loader(param, q_w, "q")
|
||||
weight_loader(param, k_w, "k")
|
||||
weight_loader(param, v_w, "v")
|
||||
loaded_params.add(name)
|
||||
continue
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# 2. gate_proj / up_proj → gate_up_proj (dense MLP + shared-exp.)
|
||||
# ------------------------------------------------------------------
|
||||
matched_stacked = False
|
||||
for param_name, weight_name, shard_id in stacked_params_mapping:
|
||||
if weight_name not in name:
|
||||
continue
|
||||
if "mlp.experts" in name: # routed experts handled below
|
||||
continue
|
||||
new_name = name.replace(weight_name, param_name)
|
||||
if new_name.endswith(".bias") and new_name not in params_dict:
|
||||
continue
|
||||
if new_name not in params_dict:
|
||||
continue
|
||||
if is_pp_missing_parameter(new_name, self):
|
||||
continue
|
||||
|
||||
param = params_dict[new_name]
|
||||
weight_loader = getattr(param, "weight_loader", default_weight_loader)
|
||||
weight_loader(param, loaded_weight, shard_id)
|
||||
loaded_params.add(new_name)
|
||||
matched_stacked = True
|
||||
break
|
||||
|
||||
if matched_stacked:
|
||||
continue
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# 3. Routed expert weights → fused-MoE kernel layout
|
||||
# ------------------------------------------------------------------
|
||||
matched_expert = False
|
||||
for (
|
||||
param_name,
|
||||
weight_name,
|
||||
expert_id,
|
||||
shard_id,
|
||||
) in expert_params_mapping:
|
||||
if weight_name not in name:
|
||||
continue
|
||||
new_name = name.replace(weight_name, param_name)
|
||||
if is_pp_missing_parameter(new_name, self):
|
||||
continue
|
||||
if new_name not in params_dict:
|
||||
continue
|
||||
|
||||
param = params_dict[new_name]
|
||||
weight_loader = getattr(param, "weight_loader", default_weight_loader)
|
||||
weight_loader(
|
||||
param,
|
||||
loaded_weight,
|
||||
name,
|
||||
shard_id=shard_id,
|
||||
expert_id=expert_id,
|
||||
)
|
||||
loaded_params.add(new_name)
|
||||
matched_expert = True
|
||||
break
|
||||
|
||||
if matched_expert:
|
||||
continue
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# 4. All other weights: direct load (layernorms, embed_tokens, …)
|
||||
# ------------------------------------------------------------------
|
||||
if name.endswith(".bias") and name not in params_dict:
|
||||
continue
|
||||
if name not in params_dict:
|
||||
continue
|
||||
if is_pp_missing_parameter(name, self):
|
||||
continue
|
||||
|
||||
param = params_dict[name]
|
||||
weight_loader = getattr(param, "weight_loader", default_weight_loader)
|
||||
try:
|
||||
weight_loader(param, loaded_weight)
|
||||
except Exception as e:
|
||||
raise RuntimeError(
|
||||
f"[param2moe] Failed to load weight '{name}' "
|
||||
f"with shape {tuple(loaded_weight.shape)} "
|
||||
f"into param type {type(param).__name__}: {e}"
|
||||
) from e
|
||||
loaded_params.add(name)
|
||||
|
||||
return loaded_params
|
||||
|
||||
def get_expert_mapping(self) -> list[tuple[str, str, int, str]]:
|
||||
return fused_moe_make_expert_params_mapping(
|
||||
self,
|
||||
ckpt_gate_proj_name="gate_proj",
|
||||
ckpt_down_proj_name="down_proj",
|
||||
ckpt_up_proj_name="up_proj",
|
||||
num_experts=self.config.num_experts,
|
||||
loader = AutoWeightsLoader(self)
|
||||
return loader.load_weights(
|
||||
_normalize_expert_bias(weights), mapper=self.hf_to_vllm_mapper
|
||||
)
|
||||
|
||||
|
||||
@@ -859,4 +712,4 @@ class Param2MoEForCausalLM(
|
||||
weights: Iterable[tuple[str, torch.Tensor]],
|
||||
) -> set[str]:
|
||||
loader = AutoWeightsLoader(self)
|
||||
return loader.load_weights(_rename_and_normalize_weights(weights))
|
||||
return loader.load_weights(weights)
|
||||
|
||||
@@ -93,6 +93,7 @@ from .utils import (
|
||||
extract_layer_index,
|
||||
make_empty_intermediate_tensors_factory,
|
||||
make_layers,
|
||||
maybe_fuse_shared_experts,
|
||||
maybe_prefix,
|
||||
)
|
||||
|
||||
@@ -261,20 +262,20 @@ class Qwen3_5Model(Qwen3NextModel):
|
||||
self.aux_hidden_state_layers: tuple[int, ...] = ()
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
mapper = self.hf_to_vllm_mapper
|
||||
# FSE must match construction (Qwen3NextSparseMoeBlock): reroute the
|
||||
# shared expert into the extra fused slot only when AITER FSE is both
|
||||
# requested and compatible with the quant spec.
|
||||
is_fse = rocm_aiter_ops.is_fusion_moe_shared_experts_enabled() and (
|
||||
_is_shared_expert_fse_compatible(self.quant_config)
|
||||
)
|
||||
if is_fse:
|
||||
num_routed = self.config.num_experts
|
||||
mapper = mapper | WeightsMapper(
|
||||
orig_to_new_substr={"mlp.shared_expert.": f"mlp.experts.{num_routed}."}
|
||||
if "moe" in self.config.model_type:
|
||||
weights = maybe_fuse_shared_experts(
|
||||
weights,
|
||||
enabled=rocm_aiter_ops.is_fusion_moe_shared_experts_enabled()
|
||||
and _is_shared_expert_fse_compatible(self.quant_config),
|
||||
n_routed_experts=self.config.num_experts,
|
||||
n_shared_experts=1,
|
||||
ckpt_prefix="mlp.shared_expert",
|
||||
)
|
||||
loader = AutoWeightsLoader(self)
|
||||
return loader.load_weights(weights, mapper=mapper)
|
||||
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
|
||||
|
||||
|
||||
class Qwen3_5ForCausalLMBase(
|
||||
|
||||
@@ -41,9 +41,9 @@ from .interfaces import (
|
||||
from .utils import (
|
||||
AutoWeightsLoader,
|
||||
PPMissingLayer,
|
||||
WeightsMapper,
|
||||
_merge_multimodal_embeddings,
|
||||
make_empty_intermediate_tensors_factory,
|
||||
maybe_fuse_shared_experts,
|
||||
maybe_prefix,
|
||||
)
|
||||
|
||||
@@ -174,17 +174,18 @@ class Qwen3_5MultiTokenPredictor(nn.Module):
|
||||
return hidden_states
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
mapper = self.hf_to_vllm_mapper
|
||||
is_fse = rocm_aiter_ops.is_fusion_moe_shared_experts_enabled() and (
|
||||
_is_shared_expert_fse_compatible(get_current_vllm_config().quant_config)
|
||||
weights = maybe_fuse_shared_experts(
|
||||
weights,
|
||||
enabled=rocm_aiter_ops.is_fusion_moe_shared_experts_enabled()
|
||||
and _is_shared_expert_fse_compatible(
|
||||
get_current_vllm_config().quant_config
|
||||
),
|
||||
n_routed_experts=getattr(self.config, "num_experts", 0),
|
||||
n_shared_experts=1,
|
||||
ckpt_prefix="mlp.shared_expert",
|
||||
)
|
||||
if is_fse:
|
||||
num_routed = getattr(self.config, "num_experts", 0)
|
||||
mapper = mapper | WeightsMapper(
|
||||
orig_to_new_substr={"mlp.shared_expert.": f"mlp.experts.{num_routed}."}
|
||||
)
|
||||
loader = AutoWeightsLoader(self)
|
||||
return loader.load_weights(weights, mapper=mapper)
|
||||
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
|
||||
|
||||
|
||||
@support_torch_compile(
|
||||
|
||||
@@ -31,10 +31,6 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
|
||||
ParallelLMHead,
|
||||
VocabParallelEmbedding,
|
||||
)
|
||||
from vllm.model_executor.model_loader.weight_utils import (
|
||||
default_weight_loader,
|
||||
maybe_remap_kv_scale_name,
|
||||
)
|
||||
from vllm.multimodal.inputs import NestedTensors
|
||||
from vllm.transformers_utils.config import set_default_rope_theta
|
||||
from vllm.transformers_utils.repo_utils import get_hf_file_bytes
|
||||
@@ -44,6 +40,7 @@ from .qwen2 import Qwen2MLP as Qwen3MLP
|
||||
from .qwen3 import Qwen3ForCausalLM
|
||||
from .utils import (
|
||||
AutoWeightsLoader,
|
||||
WeightsMapper,
|
||||
get_draft_quant_config,
|
||||
maybe_prefix,
|
||||
process_eagle_weight,
|
||||
@@ -333,6 +330,17 @@ class DFlashQwen3DecoderLayer(nn.Module):
|
||||
|
||||
@support_torch_compile
|
||||
class DFlashQwen3Model(nn.Module):
|
||||
hf_to_vllm_mapper = WeightsMapper(
|
||||
orig_to_new_substr={"midlayer.": "layers.0."},
|
||||
orig_to_new_stacked={
|
||||
".q_proj": (".qkv_proj", "q"),
|
||||
".k_proj": (".qkv_proj", "k"),
|
||||
".v_proj": (".qkv_proj", "v"),
|
||||
".gate_proj": (".gate_up_proj", 0),
|
||||
".up_proj": (".gate_up_proj", 1),
|
||||
},
|
||||
)
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
@@ -624,51 +632,26 @@ class DFlashQwen3Model(nn.Module):
|
||||
hidden_states, _ = self.norm(hidden_states, residual)
|
||||
return hidden_states
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
stacked_params_mapping = [
|
||||
(".qkv_proj", ".q_proj", "q"),
|
||||
(".qkv_proj", ".k_proj", "k"),
|
||||
(".qkv_proj", ".v_proj", "v"),
|
||||
(".gate_up_proj", ".gate_proj", 0),
|
||||
(".gate_up_proj", ".up_proj", 1),
|
||||
]
|
||||
params_dict = dict(self.named_parameters())
|
||||
loaded_params: set[str] = set()
|
||||
tp_rank = get_tensor_model_parallel_rank()
|
||||
def _preprocess(
|
||||
self, weights: Iterable[tuple[str, torch.Tensor]]
|
||||
) -> Iterable[tuple[str, torch.Tensor]]:
|
||||
tp_size = get_tensor_model_parallel_world_size()
|
||||
tp_rank = get_tensor_model_parallel_rank()
|
||||
for name, loaded_weight in weights:
|
||||
if "midlayer." in name:
|
||||
name = name.replace("midlayer.", "layers.0.")
|
||||
if "scale" in name:
|
||||
name = maybe_remap_kv_scale_name(name, params_dict)
|
||||
if name is None:
|
||||
continue
|
||||
if "attention_sink_bias" in name:
|
||||
if name not in params_dict:
|
||||
continue
|
||||
# Sink bias is per-head; shard it across TP ranks like the
|
||||
# attention heads themselves.
|
||||
param = params_dict[name]
|
||||
heads_per_rank = loaded_weight.shape[0] // tp_size
|
||||
head_start = tp_rank * heads_per_rank
|
||||
narrow_weight = loaded_weight.narrow(0, head_start, heads_per_rank)
|
||||
param.data.copy_(narrow_weight)
|
||||
loaded_params.add(name)
|
||||
continue
|
||||
for param_name, weight_name, shard_id in stacked_params_mapping:
|
||||
if weight_name not in name:
|
||||
continue
|
||||
name = name.replace(weight_name, param_name)
|
||||
param = params_dict[name]
|
||||
weight_loader = param.weight_loader
|
||||
weight_loader(param, loaded_weight, shard_id)
|
||||
break
|
||||
else:
|
||||
param = params_dict[name]
|
||||
weight_loader = getattr(param, "weight_loader", default_weight_loader)
|
||||
weight_loader(param, loaded_weight)
|
||||
loaded_params.add(name)
|
||||
return loaded_params
|
||||
loaded_weight = loaded_weight.narrow(
|
||||
0, tp_rank * heads_per_rank, heads_per_rank
|
||||
)
|
||||
yield name, loaded_weight
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
loader = AutoWeightsLoader(self)
|
||||
return loader.load_weights(
|
||||
self._preprocess(weights), mapper=self.hf_to_vllm_mapper
|
||||
)
|
||||
|
||||
|
||||
class DFlashQwen3ForCausalLM(Qwen3ForCausalLM):
|
||||
|
||||
@@ -71,6 +71,7 @@ from .utils import (
|
||||
extract_layer_index,
|
||||
make_empty_intermediate_tensors_factory,
|
||||
make_layers,
|
||||
maybe_fuse_shared_experts,
|
||||
maybe_prefix,
|
||||
)
|
||||
|
||||
@@ -713,17 +714,14 @@ class Qwen3NextModel(nn.Module, EagleModelMixin):
|
||||
return hidden_states
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
mapper = self.hf_to_vllm_mapper
|
||||
if rocm_aiter_ops.is_fusion_moe_shared_experts_enabled():
|
||||
# AITER fused-shared-experts: route the shared_expert checkpoint
|
||||
# weights into the extra fused expert slot. Merge (not mutate) so the
|
||||
# shared class mapper isn't permanently altered.
|
||||
num_routed = getattr(self.config, "num_experts", 0)
|
||||
mapper = mapper | WeightsMapper(
|
||||
orig_to_new_substr={"mlp.shared_expert.": f"mlp.experts.{num_routed}."}
|
||||
)
|
||||
weights = maybe_fuse_shared_experts(
|
||||
weights,
|
||||
n_routed_experts=getattr(self.config, "num_experts", 0),
|
||||
n_shared_experts=1,
|
||||
ckpt_prefix="mlp.shared_expert",
|
||||
)
|
||||
loader = AutoWeightsLoader(self)
|
||||
return loader.load_weights(weights, mapper=mapper)
|
||||
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
|
||||
|
||||
|
||||
class QwenNextMixtureOfExperts(MixtureOfExperts):
|
||||
|
||||
@@ -7,7 +7,6 @@ from collections.abc import Iterable
|
||||
import torch
|
||||
from torch import nn
|
||||
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
from vllm.compilation.decorators import support_torch_compile
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.distributed.parallel_state import get_pp_group
|
||||
@@ -30,8 +29,8 @@ from vllm.transformers_utils.configs.qwen3_next import Qwen3NextConfig
|
||||
|
||||
from .utils import (
|
||||
AutoWeightsLoader,
|
||||
WeightsMapper,
|
||||
make_empty_intermediate_tensors_factory,
|
||||
maybe_fuse_shared_experts,
|
||||
maybe_prefix,
|
||||
)
|
||||
|
||||
@@ -154,16 +153,14 @@ class Qwen3NextMultiTokenPredictor(nn.Module):
|
||||
return hidden_states
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
mapper = self.hf_to_vllm_mapper
|
||||
if rocm_aiter_ops.is_fusion_moe_shared_experts_enabled():
|
||||
# AITER fused-shared-experts: route the shared_expert checkpoint
|
||||
# weights into the extra fused expert slot.
|
||||
num_routed = self.config.num_experts
|
||||
mapper = mapper | WeightsMapper(
|
||||
orig_to_new_substr={"mlp.shared_expert.": f"mlp.experts.{num_routed}."}
|
||||
)
|
||||
weights = maybe_fuse_shared_experts(
|
||||
weights,
|
||||
n_routed_experts=self.config.num_experts,
|
||||
n_shared_experts=1,
|
||||
ckpt_prefix="mlp.shared_expert",
|
||||
)
|
||||
loader = AutoWeightsLoader(self)
|
||||
return loader.load_weights(weights, mapper=mapper)
|
||||
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
|
||||
|
||||
|
||||
@support_torch_compile
|
||||
|
||||
@@ -35,11 +35,7 @@ from vllm.distributed import (
|
||||
get_tensor_model_parallel_world_size,
|
||||
)
|
||||
from vllm.model_executor.layers.activation import SiluAndMul
|
||||
from vllm.model_executor.layers.fused_moe import (
|
||||
FusedMoE,
|
||||
MoERunner,
|
||||
fused_moe_make_expert_params_mapping,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe import FusedMoE, MoERunner
|
||||
from vllm.model_executor.layers.layernorm import RMSNorm
|
||||
from vllm.model_executor.layers.linear import (
|
||||
ColumnParallelLinear,
|
||||
@@ -55,7 +51,6 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
|
||||
ParallelLMHead,
|
||||
VocabParallelEmbedding,
|
||||
)
|
||||
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
|
||||
from vllm.sequence import IntermediateTensors
|
||||
|
||||
from .bailing_moe import BailingMoeForCausalLM
|
||||
@@ -63,7 +58,7 @@ from .interfaces import MixtureOfExperts, SupportsLoRA, SupportsPP
|
||||
from .utils import (
|
||||
AutoWeightsLoader,
|
||||
PPMissingLayer,
|
||||
is_pp_missing_parameter,
|
||||
WeightsMapper,
|
||||
make_empty_intermediate_tensors_factory,
|
||||
make_layers,
|
||||
maybe_prefix,
|
||||
@@ -448,6 +443,16 @@ class SarvamMLABlock(nn.Module):
|
||||
|
||||
|
||||
class SarvamMLAModel(nn.Module):
|
||||
hf_to_vllm_mapper = WeightsMapper(
|
||||
orig_to_new_stacked={
|
||||
# .experts.gate_up_proj must be handled by MoERunner.load_weights for EP
|
||||
".mlp.gate_proj": (".mlp.gate_up_proj", 0),
|
||||
".mlp.up_proj": (".mlp.gate_up_proj", 1),
|
||||
".shared_experts.gate_proj": (".shared_experts.gate_up_proj", 0),
|
||||
".shared_experts.up_proj": (".shared_experts.gate_up_proj", 1),
|
||||
}
|
||||
)
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
@@ -532,97 +537,14 @@ class SarvamMLAModel(nn.Module):
|
||||
hidden_states, _ = self.norm(hidden_states, residual)
|
||||
return hidden_states
|
||||
|
||||
def get_expert_mapping(self) -> list[tuple[str, str, int, str]]:
|
||||
return fused_moe_make_expert_params_mapping(
|
||||
self,
|
||||
ckpt_gate_proj_name="gate_proj",
|
||||
ckpt_down_proj_name="down_proj",
|
||||
ckpt_up_proj_name="up_proj",
|
||||
num_experts=self.config.num_experts,
|
||||
)
|
||||
|
||||
def load_weights(
|
||||
self,
|
||||
weights: Iterable[tuple[str, torch.Tensor]],
|
||||
) -> set[str]:
|
||||
"""Load weights with stacked gate+up and MoE expert remapping."""
|
||||
weights = _normalized_weights(weights)
|
||||
stacked_params_mapping = [
|
||||
("gate_up_proj", "gate_proj", 0),
|
||||
("gate_up_proj", "up_proj", 1),
|
||||
]
|
||||
|
||||
params_dict = dict(self.named_parameters(remove_duplicate=False))
|
||||
loaded_params: set[str] = set()
|
||||
expert_params_mapping = self.get_expert_mapping()
|
||||
|
||||
for name, loaded_weight in weights:
|
||||
for param_name, weight_name, shard_id in stacked_params_mapping:
|
||||
if weight_name not in name:
|
||||
continue
|
||||
if "mlp.experts" in name:
|
||||
continue
|
||||
new_name = name.replace(weight_name, param_name)
|
||||
if new_name.endswith(".bias") and new_name not in params_dict:
|
||||
continue
|
||||
if new_name not in params_dict:
|
||||
continue
|
||||
if is_pp_missing_parameter(new_name, self):
|
||||
continue
|
||||
|
||||
param = params_dict[new_name]
|
||||
weight_loader = getattr(param, "weight_loader", default_weight_loader)
|
||||
weight_loader(param, loaded_weight, shard_id)
|
||||
loaded_params.add(new_name)
|
||||
break
|
||||
else:
|
||||
mapped = False
|
||||
for (
|
||||
param_name,
|
||||
weight_name,
|
||||
expert_id,
|
||||
shard_id,
|
||||
) in expert_params_mapping:
|
||||
if weight_name not in name:
|
||||
continue
|
||||
|
||||
new_name = name.replace(weight_name, param_name)
|
||||
if is_pp_missing_parameter(new_name, self):
|
||||
continue
|
||||
if new_name not in params_dict:
|
||||
continue
|
||||
|
||||
param = params_dict[new_name]
|
||||
weight_loader = getattr(
|
||||
param, "weight_loader", default_weight_loader
|
||||
)
|
||||
weight_loader(
|
||||
param,
|
||||
loaded_weight,
|
||||
name,
|
||||
shard_id=shard_id,
|
||||
expert_id=expert_id,
|
||||
)
|
||||
loaded_params.add(new_name)
|
||||
mapped = True
|
||||
break
|
||||
|
||||
if mapped:
|
||||
continue
|
||||
|
||||
if name.endswith(".bias") and name not in params_dict:
|
||||
continue
|
||||
if name not in params_dict:
|
||||
continue
|
||||
if is_pp_missing_parameter(name, self):
|
||||
continue
|
||||
|
||||
param = params_dict[name]
|
||||
weight_loader = getattr(param, "weight_loader", default_weight_loader)
|
||||
weight_loader(param, loaded_weight)
|
||||
loaded_params.add(name)
|
||||
|
||||
return loaded_params
|
||||
loader = AutoWeightsLoader(self)
|
||||
return loader.load_weights(
|
||||
_normalized_weights(weights), mapper=self.hf_to_vllm_mapper
|
||||
)
|
||||
|
||||
|
||||
class SarvamMixtureOfExperts(MixtureOfExperts):
|
||||
@@ -764,9 +686,6 @@ class SarvamMLAForCausalLM(nn.Module, SupportsPP, SupportsLoRA, SarvamMixtureOfE
|
||||
)
|
||||
return loader.load_weights(weights)
|
||||
|
||||
def get_expert_mapping(self) -> list[tuple[str, str, int, str]]:
|
||||
return self.model.get_expert_mapping()
|
||||
|
||||
|
||||
class SarvamMoEForCausalLM(BailingMoeForCausalLM):
|
||||
"""Same as BailingMoeForCausalLM, but normalizes gate expert_bias pre-load."""
|
||||
|
||||
@@ -53,6 +53,7 @@ from .utils import (
|
||||
PPMissingLayer,
|
||||
WeightsMapper,
|
||||
extract_layer_index,
|
||||
get_spec_layer_idx_from_weight_name,
|
||||
is_pp_missing_parameter,
|
||||
make_empty_intermediate_tensors_factory,
|
||||
make_layers,
|
||||
@@ -978,18 +979,3 @@ class Step3p5ForCausalLM(nn.Module, SupportsPP, MixtureOfExperts):
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
loader = AutoWeightsLoader(self)
|
||||
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
|
||||
|
||||
|
||||
def get_spec_layer_idx_from_weight_name(
|
||||
config: ModelConfig, weight_name: str
|
||||
) -> int | None:
|
||||
if hasattr(config, "num_nextn_predict_layers") and (
|
||||
config.num_nextn_predict_layers > 0
|
||||
):
|
||||
layer_idx = config.num_hidden_layers
|
||||
for i in range(config.num_nextn_predict_layers):
|
||||
if weight_name.startswith(
|
||||
f"layers.{layer_idx + i}." # Step3p5Model
|
||||
) or weight_name.startswith(f"model.layers.{layer_idx + i}."): # Step3p5MTP
|
||||
return layer_idx + i
|
||||
return None
|
||||
|
||||
@@ -18,8 +18,8 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
|
||||
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
|
||||
from vllm.sequence import IntermediateTensors
|
||||
|
||||
from .step3p5 import Step3p5DecoderLayer, get_spec_layer_idx_from_weight_name
|
||||
from .utils import maybe_prefix
|
||||
from .step3p5 import Step3p5DecoderLayer
|
||||
from .utils import get_spec_layer_idx_from_weight_name, maybe_prefix
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
@@ -26,19 +26,23 @@ import torch
|
||||
import torch.nn as nn
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
|
||||
from vllm.model_executor.models.llama import LlamaForCausalLM, LlamaModel
|
||||
|
||||
from .llama import LlamaDecoderLayer
|
||||
from .utils import (
|
||||
AutoWeightsLoader,
|
||||
PPMissingLayer,
|
||||
WeightsMapper,
|
||||
is_pp_missing_parameter,
|
||||
)
|
||||
from .utils import AutoWeightsLoader, PPMissingLayer, WeightsMapper
|
||||
|
||||
|
||||
class TeleChat2Model(LlamaModel):
|
||||
hf_to_vllm_mapper = WeightsMapper(
|
||||
orig_to_new_stacked={
|
||||
".query": (".qkv_proj", "q"),
|
||||
".key": (".qkv_proj", "k"),
|
||||
".value": (".qkv_proj", "v"),
|
||||
".gate_proj": (".gate_up_proj", 0),
|
||||
".up_proj": (".gate_up_proj", 1),
|
||||
}
|
||||
)
|
||||
|
||||
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
|
||||
hf_config = vllm_config.model_config.hf_config
|
||||
|
||||
@@ -65,62 +69,34 @@ class TeleChat2Model(LlamaModel):
|
||||
layer.mlp.gate_up_proj.bias = None
|
||||
layer.mlp.gate_up_proj.skip_bias_add = True
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
stacked_params_mapping = [
|
||||
("gate_up_proj", "gate_proj", 0),
|
||||
("gate_up_proj", "up_proj", 1),
|
||||
]
|
||||
params_dict = dict(self.named_parameters())
|
||||
loaded_params: set[str] = set()
|
||||
def _split_key_value(
|
||||
self, weights: Iterable[tuple[str, torch.Tensor]]
|
||||
) -> Iterable[tuple[str, torch.Tensor]]:
|
||||
# TeleChat2 stores k/v as a single per-head-interleaved `key_value`
|
||||
# tensor. De-interleave it into separate k/v so the qkv_proj mapper
|
||||
# can stack them.
|
||||
total_num_heads = self.config.n_head
|
||||
head_dim = self.config.hidden_size // total_num_heads
|
||||
for name, loaded_weight in weights:
|
||||
if "self_attn.key_value" in name:
|
||||
k_weight = []
|
||||
v_weight = []
|
||||
for i in range(total_num_heads):
|
||||
start = i * head_dim * 2
|
||||
k_weight.append(loaded_weight[start : start + head_dim, :])
|
||||
v_weight.append(
|
||||
loaded_weight[start + head_dim : start + 2 * head_dim :]
|
||||
)
|
||||
k_weight = torch.cat(k_weight, dim=0)
|
||||
v_weight = torch.cat(v_weight, dim=0)
|
||||
name = name.replace("key_value", "qkv_proj")
|
||||
if is_pp_missing_parameter(name, self):
|
||||
continue
|
||||
param = params_dict[name]
|
||||
weight_loader = param.weight_loader
|
||||
weight_loader(param, k_weight, "k")
|
||||
weight_loader(param, v_weight, "v")
|
||||
elif "query" in name:
|
||||
name = name.replace("query", "qkv_proj")
|
||||
if is_pp_missing_parameter(name, self):
|
||||
continue
|
||||
param = params_dict[name]
|
||||
weight_loader = param.weight_loader
|
||||
weight_loader(param, loaded_weight, "q")
|
||||
starts = [i * head_dim * 2 for i in range(total_num_heads)]
|
||||
k_weight = torch.cat(
|
||||
[loaded_weight[s : s + head_dim, :] for s in starts], dim=0
|
||||
)
|
||||
v_weight = torch.cat(
|
||||
[loaded_weight[s + head_dim : s + 2 * head_dim, :] for s in starts],
|
||||
dim=0,
|
||||
)
|
||||
yield name.replace("key_value", "key"), k_weight
|
||||
yield name.replace("key_value", "value"), v_weight
|
||||
else:
|
||||
for param_name, weight_name, shard_id in stacked_params_mapping:
|
||||
if weight_name not in name:
|
||||
continue
|
||||
name = name.replace(weight_name, param_name)
|
||||
if is_pp_missing_parameter(name, self):
|
||||
continue
|
||||
param = params_dict[name]
|
||||
weight_loader = param.weight_loader
|
||||
weight_loader(param, loaded_weight, shard_id)
|
||||
break
|
||||
else:
|
||||
if is_pp_missing_parameter(name, self):
|
||||
continue
|
||||
param = params_dict[name]
|
||||
weight_loader = getattr(
|
||||
param, "weight_loader", default_weight_loader
|
||||
)
|
||||
weight_loader(param, loaded_weight)
|
||||
loaded_params.add(name)
|
||||
return loaded_params
|
||||
yield name, loaded_weight
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
loader = AutoWeightsLoader(self)
|
||||
return loader.load_weights(
|
||||
self._split_key_value(weights), mapper=self.hf_to_vllm_mapper
|
||||
)
|
||||
|
||||
|
||||
class TeleChat2ForCausalLM(LlamaForCausalLM):
|
||||
|
||||
@@ -35,6 +35,7 @@ if TYPE_CHECKING:
|
||||
from transformers import PretrainedConfig
|
||||
from transformers.conversion_mapping import WeightRenaming
|
||||
|
||||
from vllm.config.model import ModelConfig
|
||||
from vllm.model_executor.layers.quantization import QuantizationConfig
|
||||
|
||||
logger = init_logger(__name__)
|
||||
@@ -424,6 +425,116 @@ class AutoWeightsLoader:
|
||||
return autoloaded_weights
|
||||
|
||||
|
||||
def maybe_fuse_shared_experts(
|
||||
weights: Iterable[tuple[str, torch.Tensor]],
|
||||
*,
|
||||
n_routed_experts: int,
|
||||
n_shared_experts: int,
|
||||
ckpt_prefix: str = "mlp.shared_experts",
|
||||
enabled: bool | None = None,
|
||||
) -> Iterable[tuple[str, torch.Tensor]]:
|
||||
"""Route AITER fused-shared-expert checkpoint weights into fused slots.
|
||||
|
||||
When AITER fused-shared-experts is active, shared experts are packed into
|
||||
the routed expert tensor. The checkpoint stores them under `ckpt_prefix`
|
||||
as a single (possibly widened) tensor; this splits it into
|
||||
`n_shared_experts` chunks named `mlp.experts.{n_routed_experts + j}` so
|
||||
the `RoutedExperts` loader treats them as extra experts. Yields the input
|
||||
unchanged when the fusion is inactive, so callers can wrap unconditionally.
|
||||
|
||||
Args:
|
||||
weights: Iterable of `(name, tensor)` checkpoint pairs.
|
||||
n_routed_experts: Number of routed experts; offsets the fused slots.
|
||||
n_shared_experts: Number of shared experts packed into the tensor.
|
||||
ckpt_prefix: Checkpoint module name of the shared experts.
|
||||
enabled: Whether AITER fused-shared-experts is active. Defaults to
|
||||
`rocm_aiter_ops.is_fusion_moe_shared_experts_enabled()`; pass an
|
||||
explicit value only when the model gates on something more (e.g.
|
||||
quant-spec compatibility) and it must match its construction-time
|
||||
decision.
|
||||
|
||||
Yields:
|
||||
`(name, tensor)` pairs with shared experts routed to fused slots.
|
||||
"""
|
||||
if enabled is None:
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
|
||||
enabled = rocm_aiter_ops.is_fusion_moe_shared_experts_enabled()
|
||||
if not enabled:
|
||||
yield from weights
|
||||
return
|
||||
|
||||
# Match on the dotted boundary so e.g. "mlp.shared_expert." does not also
|
||||
# catch a sibling "mlp.shared_expert_gate".
|
||||
prefix = f"{ckpt_prefix}."
|
||||
for name, loaded_weight in weights:
|
||||
if prefix not in name:
|
||||
yield name, loaded_weight
|
||||
continue
|
||||
# gate/up split on the output dim; down_proj on the input dim.
|
||||
split_dim = 1 if ("down_proj.weight" in name and loaded_weight.ndim > 1) else 0
|
||||
total = loaded_weight.shape[split_dim]
|
||||
if total % n_shared_experts != 0:
|
||||
raise ValueError(
|
||||
f"FSE shared-expert weight {name!r} has size {total} along axis "
|
||||
f"{split_dim}, not divisible by n_shared_experts={n_shared_experts}."
|
||||
)
|
||||
chunk = total // n_shared_experts
|
||||
for j in range(n_shared_experts):
|
||||
sl = slice(j * chunk, (j + 1) * chunk)
|
||||
if loaded_weight.ndim == 1:
|
||||
chunk_weight = loaded_weight[sl]
|
||||
elif split_dim == 0:
|
||||
chunk_weight = loaded_weight[sl, :]
|
||||
else:
|
||||
chunk_weight = loaded_weight[:, sl]
|
||||
yield (
|
||||
name.replace(prefix, f"mlp.experts.{n_routed_experts + j}."),
|
||||
chunk_weight,
|
||||
)
|
||||
|
||||
|
||||
def get_spec_layer_idx_from_weight_name(
|
||||
config: "ModelConfig", weight_name: str
|
||||
) -> int | None:
|
||||
"""Return the MTP layer index a weight belongs to, or None.
|
||||
|
||||
Args:
|
||||
config: The model config; must expose `num_hidden_layers` and, for MTP
|
||||
checkpoints, `num_nextn_predict_layers`.
|
||||
weight_name: Checkpoint weight name to classify.
|
||||
|
||||
Returns:
|
||||
The absolute layer index for an MTP-layer weight, else None.
|
||||
"""
|
||||
if not (n := getattr(config, "num_nextn_predict_layers", 0)):
|
||||
return None
|
||||
base = config.num_hidden_layers
|
||||
for i in range(n):
|
||||
if weight_name.startswith((f"model.layers.{base + i}.", f"layers.{base + i}.")):
|
||||
return base + i
|
||||
return None
|
||||
|
||||
|
||||
def skip_spec_layers(
|
||||
weights: Iterable[tuple[str, torch.Tensor]], config: "ModelConfig"
|
||||
) -> Iterable[tuple[str, torch.Tensor]]:
|
||||
"""Drop MTP spec-layer weights (loaded by the MTP head, not the base model).
|
||||
|
||||
Args:
|
||||
weights: Iterable of `(name, tensor)` checkpoint pairs.
|
||||
config: The model config, passed to `get_spec_layer_idx_from_weight_name`.
|
||||
|
||||
Yields:
|
||||
`(name, tensor)` pairs whose weight is not an MTP-layer weight.
|
||||
"""
|
||||
return (
|
||||
(name, w)
|
||||
for name, w in weights
|
||||
if get_spec_layer_idx_from_weight_name(config, name) is None
|
||||
)
|
||||
|
||||
|
||||
def init_vllm_registered_model(
|
||||
vllm_config: VllmConfig,
|
||||
*,
|
||||
|
||||
Reference in New Issue
Block a user