Remove unnecessary load_weights methods (#44589)

Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
This commit is contained in:
Harry Mellor
2026-06-29 01:52:23 -07:00
committed by GitHub
parent db28ae2d07
commit 5051698e41
54 changed files with 821 additions and 1975 deletions
@@ -5,6 +5,7 @@ import inspect
from abc import ABC, abstractmethod
from typing import TYPE_CHECKING, Any
import regex as re
import torch
from torch import nn
from transformers import PretrainedConfig
@@ -19,10 +20,12 @@ else:
class QuantizeMethodBase(ABC):
"""Base class for different quantized methods."""
# Whether this method creates weights on meta device for online quantization.
# When True, weights are created on meta device and quantized layer-wise
# in process_weights_after_loading, reducing peak memory during loading.
uses_meta_device: bool = False
"""
Whether this method creates weights on meta device for online quantization.
When True, weights are created on meta device and quantized layer-wise
in process_weights_after_loading, reducing peak memory during loading.
"""
@abstractmethod
def create_weights(
@@ -84,6 +87,18 @@ def method_has_implemented_embedding(method_class: type[QuantizeMethodBase]) ->
class QuantizationConfig(ABC):
"""Base class for quantization configs."""
_ignore_unexpected_suffixes = (
".q_scale",
".k_scale",
".v_scale",
".q_zero_point",
".k_zero_point",
".v_zero_point",
)
"""Suffixes of quantization parameters that may be present in the checkpoint but
not in the model, and should be ignored if unexpected during loading. These are used
after remapping, so should be in vLLM format (e.g. .q_scale, not .q.scale)."""
def __init__(self):
super().__init__()
# mapping is updated by models as they initialize
@@ -176,14 +191,40 @@ class QuantizationConfig(ABC):
"""
raise NotImplementedError
def get_cache_scale_mapper(self) -> "WeightsMapper | None":
@staticmethod
def get_cache_scale_mapper() -> "WeightsMapper":
"""Mapping from checkpoint KV-cache scale names to vLLM scale names.
Returning a mapper here causes `AutoWeightsLoader` to apply it to the
weight stream automatically; individual model `load_weights` methods
do not need to know about KV-cache scales.
"""
return None
from vllm.model_executor.models.utils import WeightsMapper
orig_to_new_regex = {
# Deprecated fused kv_scale -> attn.k_scale
re.compile(r"\.kv_scale$"): r".attn.k_scale",
# ModelOpt: .self_attn.{k,v}_proj.{k,v}_scale -> .self_attn.attn.*
re.compile(r"\.self_attn\.[kv]_proj\.([kv])_scale$"): (
r".self_attn.attn.\1_scale"
),
# Fused QKV / qkqkv proj: .self_attn.qk(qk)v_proj.{k,v}_scale -> attn
re.compile(r"\.self_attn\.qk(?:qk)?v_proj\.([kv])_scale$"): (
r".self_attn.attn.\1_scale"
),
# NemotronH: .mixer.{k,v}_proj.{k,v}_scale -> .mixer.attn.*
re.compile(r"\.mixer\.[kv]_proj\.([kv])_scale$"): r".mixer.attn.\1_scale",
# HYV3: .self_attn.q.scale -> .self_attn.attn.q_scale
re.compile(r"\.self_attn\.q\.scale$"): r".self_attn.attn.q_scale",
# HYV3: .self_attn.{k,v}_cache.scale -> .self_attn.attn.{k,v}_scale
re.compile(r"\.self_attn\.([kv])_cache\.scale$"): (
r".self_attn.attn.\1_scale"
),
# Default: .{q,k,v}_scale -> .attn.{q,k,v}_scale (unless already .attn)
re.compile(r"(?<!\.attn)\.([qkv])_scale$"): r".attn.\1_scale",
re.compile(r"(?<!\.attn)\.([qkv])_zero_point$"): r".attn.\1_zero_point",
}
return WeightsMapper(orig_to_new_regex=orig_to_new_regex)
def apply_vllm_mapper( # noqa: B027
self, hf_to_vllm_mapper: "WeightsMapper"
+10 -9
View File
@@ -219,18 +219,19 @@ class Fp8Config(QuantizationConfig):
return Fp8KVCacheMethod(self)
return None
def get_cache_scale_mapper(self) -> "WeightsMapper":
@staticmethod
def get_cache_scale_mapper() -> "WeightsMapper":
"""Map compressed-tensors KV-cache scale names to vLLM names."""
from vllm.model_executor.models.utils import WeightsMapper
return WeightsMapper(
orig_to_new_suffix={
".k_proj.output_scale": ".attn.k_scale",
".v_proj.output_scale": ".attn.v_scale",
".q_proj.output_scale": ".attn.q_scale",
".self_attn.prob_output_scale": ".self_attn.attn.prob_scale",
}
)
orig_to_new_suffix = {
".k_proj.output_scale": ".attn.k_scale",
".v_proj.output_scale": ".attn.v_scale",
".q_proj.output_scale": ".attn.q_scale",
".self_attn.prob_output_scale": ".self_attn.attn.prob_scale",
}
cache_scale_mapper = WeightsMapper(orig_to_new_suffix=orig_to_new_suffix)
return cache_scale_mapper | QuantizationConfig.get_cache_scale_mapper()
class CopyNumelCounter(TorchDispatchMode):
@@ -679,16 +679,17 @@ class QuarkConfig(QuantizationConfig):
return scheme
def get_cache_scale_mapper(self) -> "WeightsMapper":
@staticmethod
def get_cache_scale_mapper() -> "WeightsMapper":
"""Map Quark KV-cache scale names to vLLM names."""
return WeightsMapper(
orig_to_new_suffix={
".k_proj.output_scale": ".attn.k_scale",
".v_proj.output_scale": ".attn.v_scale",
".q_proj.output_scale": ".attn.q_scale",
".self_attn.prob_output_scale": ".self_attn.attn.prob_scale",
}
)
orig_to_new_suffix = {
".k_proj.output_scale": ".attn.k_scale",
".v_proj.output_scale": ".attn.v_scale",
".q_proj.output_scale": ".attn.q_scale",
".self_attn.prob_output_scale": ".self_attn.attn.prob_scale",
}
cache_scale_mapper = WeightsMapper(orig_to_new_suffix=orig_to_new_suffix)
return cache_scale_mapper | QuantizationConfig.get_cache_scale_mapper()
class QuarkLinearMethod(LinearMethodBase):