Remove unnecessary load_weights methods (#44589)

Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
This commit is contained in:
Harry Mellor
2026-06-29 01:52:23 -07:00
committed by GitHub
parent db28ae2d07
commit 5051698e41
54 changed files with 821 additions and 1975 deletions
+85 -37
View File
@@ -3,9 +3,12 @@
import itertools
from abc import abstractmethod
from collections.abc import Iterable
from typing import Any
import torch
from torch.nn.parameter import Parameter
from typing_extensions import TypeIs
import vllm.envs as envs
from vllm.distributed import (
@@ -632,31 +635,31 @@ class MergedColumnParallelLinear(ColumnParallelLinear):
disable_tp=disable_tp,
)
def validate_shard_id(self, loaded_shard_id: int | tuple[int, ...] | None):
if loaded_shard_id is None:
return
if isinstance(loaded_shard_id, tuple):
for idx in loaded_shard_id:
def validate_shard_id(self, shard_id: Any) -> TypeIs[int | tuple[int, ...] | None]:
if isinstance(shard_id, int):
if shard_id < 0 or shard_id >= len(self.output_sizes):
raise ValueError(
f"Shard id should be between 0 and {len(self.output_sizes) - 1}. "
f"Got shard id {shard_id}."
)
return True
if shard_id is None:
return True
if isinstance(shard_id, tuple):
for idx in shard_id:
if not (0 <= idx < len(self.output_sizes)):
raise ValueError(
f"Shard id index {idx} should be between 0 and "
f"{len(self.output_sizes) - 1}. Got shard id {loaded_shard_id}."
f"{len(self.output_sizes) - 1}. Got shard id {shard_id}."
)
if len(loaded_shard_id) > 1 and any(
b - a != 1 for a, b in zip(loaded_shard_id[:-1], loaded_shard_id[1:])
if len(shard_id) > 1 and any(
b - a != 1 for a, b in zip(shard_id[:-1], shard_id[1:])
):
raise ValueError(
"Shard id with multiple indices should be consecutive. "
f"Got shard id {loaded_shard_id}."
f"Got shard id {shard_id}."
)
return
elif isinstance(loaded_shard_id, int):
if loaded_shard_id < 0 or loaded_shard_id >= len(self.output_sizes):
raise ValueError(
f"Shard id should be between 0 and {len(self.output_sizes) - 1}. "
f"Got shard id {loaded_shard_id}."
)
return
return True
raise ValueError("This line should not be reached")
def weight_loader(
@@ -910,6 +913,31 @@ class MergedColumnParallelLinear(ColumnParallelLinear):
tp_rank=self.tp_rank,
)
def load_weights(
self, weights: Iterable[tuple[str, torch.Tensor]]
) -> Iterable[str]:
for name, loaded_weight in weights:
shard_id = getattr(loaded_weight, "shard_id", None)
self.validate_shard_id(shard_id)
# Load into self if name is not an attr of self or its submodules
param: Parameter
if "." in name:
submodule, _, attr = name.rpartition(".")
param = getattr(self.get_submodule(submodule), attr, self)
else:
param = getattr(self, name, self)
if param is None and name == "bias":
continue
param.weight_loader(param, loaded_weight, shard_id)
logger.debug(
"Loaded shard %s with shape %s into %s.%s",
shard_id,
loaded_weight.shape,
self.prefix,
name,
)
yield name
class QKVParallelLinear(ColumnParallelLinear):
"""Linear layers for the attention's QKV transformation.
@@ -996,17 +1024,13 @@ class QKVParallelLinear(ColumnParallelLinear):
disable_tp=disable_tp,
)
def validate_shard_id(self, loaded_shard_id: str | None):
if loaded_shard_id is None:
return
if isinstance(loaded_shard_id, str):
if loaded_shard_id not in ["q", "k", "v"]:
raise ValueError(
"Shard id for QKVParallelLinear should be 'q', 'k', or 'v', "
f"got shard id {loaded_shard_id}."
)
return
raise ValueError("This line should not be reached")
def validate_shard_id(self, shard_id: Any) -> TypeIs[str | None]:
if shard_id in {"q", "k", "v"} or shard_id is None:
return True
raise ValueError(
"Shard id for QKVParallelLinear should be 'q', 'k', or 'v', "
f"got shard id {shard_id}."
)
def _get_shard_offset_mapping(self, loaded_shard_id: str):
shard_offset_mapping = {
@@ -1302,6 +1326,31 @@ class QKVParallelLinear(ColumnParallelLinear):
assert param_data.shape == loaded_weight.shape
param_data.copy_(loaded_weight)
def load_weights(
self, weights: Iterable[tuple[str, torch.Tensor]]
) -> Iterable[str]:
for name, loaded_weight in weights:
shard_id = getattr(loaded_weight, "shard_id", None)
self.validate_shard_id(shard_id)
# Load into self if name is not an attr of self or its submodules
param: Parameter
if "." in name:
submodule, _, attr = name.rpartition(".")
param = getattr(self.get_submodule(submodule), attr, self)
else:
param = getattr(self, name, self)
if param is None and name == "bias":
continue
param.weight_loader(param, loaded_weight, shard_id)
logger.debug(
"Loaded shard %s with shape %s into %s.%s",
shard_id,
loaded_weight.shape,
self.prefix,
name,
)
yield name
class MinimaxM3QKVParallelLinearWithIndexer(QKVParallelLinear):
"""QKV projection fused with a lightning-indexer's index_q/index_k.
@@ -1387,15 +1436,14 @@ class MinimaxM3QKVParallelLinearWithIndexer(QKVParallelLinear):
prefix=prefix,
)
def validate_shard_id(self, loaded_shard_id: str | None) -> None:
if loaded_shard_id is None:
return
if loaded_shard_id not in ("q", "k", "v", "index_q", "index_k"):
raise ValueError(
"Shard id for MinimaxM3QKVParallelLinearWithIndexer must be one of "
"'q', 'k', 'v', 'index_q', 'index_k'; got "
f"{loaded_shard_id}."
)
def validate_shard_id(self, shard_id: Any) -> TypeIs[str | None]:
if shard_id in {"q", "k", "v", "index_q", "index_k"} or shard_id is None:
return True
raise ValueError(
"Shard id for MinimaxM3QKVParallelLinearWithIndexer must be one of "
"'q', 'k', 'v', 'index_q', 'index_k'; got "
f"{shard_id}."
)
def _get_shard_offset_mapping(self, loaded_shard_id: str) -> int | None:
h = self.head_size
@@ -5,6 +5,7 @@ import inspect
from abc import ABC, abstractmethod
from typing import TYPE_CHECKING, Any
import regex as re
import torch
from torch import nn
from transformers import PretrainedConfig
@@ -19,10 +20,12 @@ else:
class QuantizeMethodBase(ABC):
"""Base class for different quantized methods."""
# Whether this method creates weights on meta device for online quantization.
# When True, weights are created on meta device and quantized layer-wise
# in process_weights_after_loading, reducing peak memory during loading.
uses_meta_device: bool = False
"""
Whether this method creates weights on meta device for online quantization.
When True, weights are created on meta device and quantized layer-wise
in process_weights_after_loading, reducing peak memory during loading.
"""
@abstractmethod
def create_weights(
@@ -84,6 +87,18 @@ def method_has_implemented_embedding(method_class: type[QuantizeMethodBase]) ->
class QuantizationConfig(ABC):
"""Base class for quantization configs."""
_ignore_unexpected_suffixes = (
".q_scale",
".k_scale",
".v_scale",
".q_zero_point",
".k_zero_point",
".v_zero_point",
)
"""Suffixes of quantization parameters that may be present in the checkpoint but
not in the model, and should be ignored if unexpected during loading. These are used
after remapping, so should be in vLLM format (e.g. .q_scale, not .q.scale)."""
def __init__(self):
super().__init__()
# mapping is updated by models as they initialize
@@ -176,14 +191,40 @@ class QuantizationConfig(ABC):
"""
raise NotImplementedError
def get_cache_scale_mapper(self) -> "WeightsMapper | None":
@staticmethod
def get_cache_scale_mapper() -> "WeightsMapper":
"""Mapping from checkpoint KV-cache scale names to vLLM scale names.
Returning a mapper here causes `AutoWeightsLoader` to apply it to the
weight stream automatically; individual model `load_weights` methods
do not need to know about KV-cache scales.
"""
return None
from vllm.model_executor.models.utils import WeightsMapper
orig_to_new_regex = {
# Deprecated fused kv_scale -> attn.k_scale
re.compile(r"\.kv_scale$"): r".attn.k_scale",
# ModelOpt: .self_attn.{k,v}_proj.{k,v}_scale -> .self_attn.attn.*
re.compile(r"\.self_attn\.[kv]_proj\.([kv])_scale$"): (
r".self_attn.attn.\1_scale"
),
# Fused QKV / qkqkv proj: .self_attn.qk(qk)v_proj.{k,v}_scale -> attn
re.compile(r"\.self_attn\.qk(?:qk)?v_proj\.([kv])_scale$"): (
r".self_attn.attn.\1_scale"
),
# NemotronH: .mixer.{k,v}_proj.{k,v}_scale -> .mixer.attn.*
re.compile(r"\.mixer\.[kv]_proj\.([kv])_scale$"): r".mixer.attn.\1_scale",
# HYV3: .self_attn.q.scale -> .self_attn.attn.q_scale
re.compile(r"\.self_attn\.q\.scale$"): r".self_attn.attn.q_scale",
# HYV3: .self_attn.{k,v}_cache.scale -> .self_attn.attn.{k,v}_scale
re.compile(r"\.self_attn\.([kv])_cache\.scale$"): (
r".self_attn.attn.\1_scale"
),
# Default: .{q,k,v}_scale -> .attn.{q,k,v}_scale (unless already .attn)
re.compile(r"(?<!\.attn)\.([qkv])_scale$"): r".attn.\1_scale",
re.compile(r"(?<!\.attn)\.([qkv])_zero_point$"): r".attn.\1_zero_point",
}
return WeightsMapper(orig_to_new_regex=orig_to_new_regex)
def apply_vllm_mapper( # noqa: B027
self, hf_to_vllm_mapper: "WeightsMapper"
+10 -9
View File
@@ -219,18 +219,19 @@ class Fp8Config(QuantizationConfig):
return Fp8KVCacheMethod(self)
return None
def get_cache_scale_mapper(self) -> "WeightsMapper":
@staticmethod
def get_cache_scale_mapper() -> "WeightsMapper":
"""Map compressed-tensors KV-cache scale names to vLLM names."""
from vllm.model_executor.models.utils import WeightsMapper
return WeightsMapper(
orig_to_new_suffix={
".k_proj.output_scale": ".attn.k_scale",
".v_proj.output_scale": ".attn.v_scale",
".q_proj.output_scale": ".attn.q_scale",
".self_attn.prob_output_scale": ".self_attn.attn.prob_scale",
}
)
orig_to_new_suffix = {
".k_proj.output_scale": ".attn.k_scale",
".v_proj.output_scale": ".attn.v_scale",
".q_proj.output_scale": ".attn.q_scale",
".self_attn.prob_output_scale": ".self_attn.attn.prob_scale",
}
cache_scale_mapper = WeightsMapper(orig_to_new_suffix=orig_to_new_suffix)
return cache_scale_mapper | QuantizationConfig.get_cache_scale_mapper()
class CopyNumelCounter(TorchDispatchMode):
@@ -679,16 +679,17 @@ class QuarkConfig(QuantizationConfig):
return scheme
def get_cache_scale_mapper(self) -> "WeightsMapper":
@staticmethod
def get_cache_scale_mapper() -> "WeightsMapper":
"""Map Quark KV-cache scale names to vLLM names."""
return WeightsMapper(
orig_to_new_suffix={
".k_proj.output_scale": ".attn.k_scale",
".v_proj.output_scale": ".attn.v_scale",
".q_proj.output_scale": ".attn.q_scale",
".self_attn.prob_output_scale": ".self_attn.attn.prob_scale",
}
)
orig_to_new_suffix = {
".k_proj.output_scale": ".attn.k_scale",
".v_proj.output_scale": ".attn.v_scale",
".q_proj.output_scale": ".attn.q_scale",
".self_attn.prob_output_scale": ".self_attn.attn.prob_scale",
}
cache_scale_mapper = WeightsMapper(orig_to_new_suffix=orig_to_new_suffix)
return cache_scale_mapper | QuantizationConfig.get_cache_scale_mapper()
class QuarkLinearMethod(LinearMethodBase):