[Model] Use AutoWeightsLoader for Voyage (#42972)

Signed-off-by: Furkan Fidan <dev@yufufi.com>
This commit is contained in:
Furkan F
2026-05-22 05:28:23 +00:00
committed by GitHub
parent 1fe3303983
commit e746a2eebf
+27 -90
View File
@@ -2,23 +2,20 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from __future__ import annotations from __future__ import annotations
from collections import defaultdict
from collections.abc import Iterable from collections.abc import Iterable
import regex as re
import torch import torch
import torch.nn as nn import torch.nn as nn
from vllm.model_executor.model_loader.weight_utils import default_weight_loader from vllm.config import VllmConfig
from vllm.model_executor.models.qwen3 import Qwen3Model from vllm.model_executor.models.qwen3 import Qwen3Model
from vllm.model_executor.models.utils import WeightsMapper from vllm.model_executor.models.utils import AutoWeightsLoader, maybe_prefix
from vllm.sequence import IntermediateTensors
WeightItem = tuple[str, torch.Tensor] WeightItem = tuple[str, torch.Tensor]
_LAYER_RE = re.compile(r"^layers\.(\d+)\.(.+)$")
class VoyageQwen3BidirectionalEmbedModel(nn.Module):
class VoyageQwen3BidirectionalEmbedModel(Qwen3Model):
""" """
Qwen3Model + Voyage embedding head + bidirectional attention. Qwen3Model + Voyage embedding head + bidirectional attention.
@@ -32,16 +29,14 @@ class VoyageQwen3BidirectionalEmbedModel(Qwen3Model):
- mlp.gate_up_proj (fused) - mlp.gate_up_proj (fused)
- self_attn.qkv_proj (fused) - self_attn.qkv_proj (fused)
- No "model." prefix - No "model." prefix
We remap/fuse weights using generator pipeline and load directly
(bypassing parent's stacked_params_mapping which would cause
double-transformation like qkv_proj -> qkqkv_proj).
""" """
hf_to_vllm_mapper = WeightsMapper(orig_to_new_prefix={"model.": ""}) def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
super().__init__()
def __init__(self, *args, **kwargs): self.config = vllm_config.model_config.hf_config
super().__init__(*args, **kwargs) self.model = Qwen3Model(
vllm_config=vllm_config, prefix=maybe_prefix(prefix, "model")
)
# Embedding head (hidden_size -> num_labels, bias=False) # Embedding head (hidden_size -> num_labels, bias=False)
self.linear = nn.Linear( self.linear = nn.Linear(
@@ -50,81 +45,23 @@ class VoyageQwen3BidirectionalEmbedModel(Qwen3Model):
bias=False, bias=False,
) )
def forward(self, *args, **kwargs): self.make_empty_intermediate_tensors = (
out = super().forward(*args, **kwargs) self.model.make_empty_intermediate_tensors
)
def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
return self.model.embed_input_ids(input_ids)
def forward(
self,
input_ids: torch.Tensor | None,
positions: torch.Tensor,
intermediate_tensors: IntermediateTensors | None = None,
inputs_embeds: torch.Tensor | None = None,
) -> torch.Tensor:
out = self.model(input_ids, positions, intermediate_tensors, inputs_embeds)
return self.linear(out) return self.linear(out)
def _fuse_qkv_proj(self, weights: Iterable[WeightItem]) -> Iterable[WeightItem]:
"""Fuse q_proj, k_proj, v_proj into qkv_proj."""
qkv_buf: dict[int, dict[str, torch.Tensor]] = defaultdict(dict)
qkv_suffixes = {
"self_attn.q_proj.weight": "q",
"self_attn.k_proj.weight": "k",
"self_attn.v_proj.weight": "v",
}
for name, tensor in weights:
m = _LAYER_RE.match(name)
if m and m.group(2) in qkv_suffixes:
layer_idx = int(m.group(1))
qkv_buf[layer_idx][qkv_suffixes[m.group(2)]] = tensor
else:
yield name, tensor
# Yield fused QKV weights
for layer_idx in sorted(qkv_buf.keys()):
parts = qkv_buf[layer_idx]
if all(p in parts for p in ("q", "k", "v")):
fused = torch.cat([parts["q"], parts["k"], parts["v"]], dim=0)
yield f"layers.{layer_idx}.self_attn.qkv_proj.weight", fused
elif parts:
missing = [p for p in ("q", "k", "v") if p not in parts]
raise ValueError(f"Layer {layer_idx} missing QKV parts: {missing}")
def _fuse_gate_up_proj(self, weights: Iterable[WeightItem]) -> Iterable[WeightItem]:
"""Fuse gate_proj and up_proj into gate_up_proj."""
mlp_buf: dict[int, dict[str, torch.Tensor]] = defaultdict(dict)
mlp_suffixes = {
"mlp.gate_proj.weight": "gate",
"mlp.up_proj.weight": "up",
}
for name, tensor in weights:
m = _LAYER_RE.match(name)
if m and m.group(2) in mlp_suffixes:
layer_idx = int(m.group(1))
mlp_buf[layer_idx][mlp_suffixes[m.group(2)]] = tensor
else:
yield name, tensor
# Yield fused gate_up weights
for layer_idx in sorted(mlp_buf.keys()):
parts = mlp_buf[layer_idx]
if all(p in parts for p in ("gate", "up")):
fused = torch.cat([parts["gate"], parts["up"]], dim=0)
yield f"layers.{layer_idx}.mlp.gate_up_proj.weight", fused
elif parts:
missing = [p for p in ("gate", "up") if p not in parts]
raise ValueError(f"Layer {layer_idx} missing MLP parts: {missing}")
def load_weights(self, weights: Iterable[WeightItem]) -> set[str]: def load_weights(self, weights: Iterable[WeightItem]) -> set[str]:
"""Remap, fuse, and load weights using generator pipeline.""" loader = AutoWeightsLoader(self)
# Chain weight transformations return loader.load_weights(weights)
weights = self.hf_to_vllm_mapper.apply(weights)
weights = self._fuse_qkv_proj(weights)
weights = self._fuse_gate_up_proj(weights)
# Load weights directly into model parameters
# (bypass parent's stacked_params_mapping)
params_dict = dict(self.named_parameters())
loaded_params: set[str] = set()
for name, loaded_weight in weights:
if name not in params_dict:
continue
param = params_dict[name]
weight_loader = getattr(param, "weight_loader", default_weight_loader)
weight_loader(param, loaded_weight)
loaded_params.add(name)
return loaded_params