Remove unnecessary load_weights methods (#44589)
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
This commit is contained in:
@@ -5,6 +5,7 @@ import inspect
|
||||
from abc import ABC, abstractmethod
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
import regex as re
|
||||
import torch
|
||||
from torch import nn
|
||||
from transformers import PretrainedConfig
|
||||
@@ -19,10 +20,12 @@ else:
|
||||
class QuantizeMethodBase(ABC):
|
||||
"""Base class for different quantized methods."""
|
||||
|
||||
# Whether this method creates weights on meta device for online quantization.
|
||||
# When True, weights are created on meta device and quantized layer-wise
|
||||
# in process_weights_after_loading, reducing peak memory during loading.
|
||||
uses_meta_device: bool = False
|
||||
"""
|
||||
Whether this method creates weights on meta device for online quantization.
|
||||
When True, weights are created on meta device and quantized layer-wise
|
||||
in process_weights_after_loading, reducing peak memory during loading.
|
||||
"""
|
||||
|
||||
@abstractmethod
|
||||
def create_weights(
|
||||
@@ -84,6 +87,18 @@ def method_has_implemented_embedding(method_class: type[QuantizeMethodBase]) ->
|
||||
class QuantizationConfig(ABC):
|
||||
"""Base class for quantization configs."""
|
||||
|
||||
_ignore_unexpected_suffixes = (
|
||||
".q_scale",
|
||||
".k_scale",
|
||||
".v_scale",
|
||||
".q_zero_point",
|
||||
".k_zero_point",
|
||||
".v_zero_point",
|
||||
)
|
||||
"""Suffixes of quantization parameters that may be present in the checkpoint but
|
||||
not in the model, and should be ignored if unexpected during loading. These are used
|
||||
after remapping, so should be in vLLM format (e.g. .q_scale, not .q.scale)."""
|
||||
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
# mapping is updated by models as they initialize
|
||||
@@ -176,14 +191,40 @@ class QuantizationConfig(ABC):
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
def get_cache_scale_mapper(self) -> "WeightsMapper | None":
|
||||
@staticmethod
|
||||
def get_cache_scale_mapper() -> "WeightsMapper":
|
||||
"""Mapping from checkpoint KV-cache scale names to vLLM scale names.
|
||||
|
||||
Returning a mapper here causes `AutoWeightsLoader` to apply it to the
|
||||
weight stream automatically; individual model `load_weights` methods
|
||||
do not need to know about KV-cache scales.
|
||||
"""
|
||||
return None
|
||||
from vllm.model_executor.models.utils import WeightsMapper
|
||||
|
||||
orig_to_new_regex = {
|
||||
# Deprecated fused kv_scale -> attn.k_scale
|
||||
re.compile(r"\.kv_scale$"): r".attn.k_scale",
|
||||
# ModelOpt: .self_attn.{k,v}_proj.{k,v}_scale -> .self_attn.attn.*
|
||||
re.compile(r"\.self_attn\.[kv]_proj\.([kv])_scale$"): (
|
||||
r".self_attn.attn.\1_scale"
|
||||
),
|
||||
# Fused QKV / qkqkv proj: .self_attn.qk(qk)v_proj.{k,v}_scale -> attn
|
||||
re.compile(r"\.self_attn\.qk(?:qk)?v_proj\.([kv])_scale$"): (
|
||||
r".self_attn.attn.\1_scale"
|
||||
),
|
||||
# NemotronH: .mixer.{k,v}_proj.{k,v}_scale -> .mixer.attn.*
|
||||
re.compile(r"\.mixer\.[kv]_proj\.([kv])_scale$"): r".mixer.attn.\1_scale",
|
||||
# HYV3: .self_attn.q.scale -> .self_attn.attn.q_scale
|
||||
re.compile(r"\.self_attn\.q\.scale$"): r".self_attn.attn.q_scale",
|
||||
# HYV3: .self_attn.{k,v}_cache.scale -> .self_attn.attn.{k,v}_scale
|
||||
re.compile(r"\.self_attn\.([kv])_cache\.scale$"): (
|
||||
r".self_attn.attn.\1_scale"
|
||||
),
|
||||
# Default: .{q,k,v}_scale -> .attn.{q,k,v}_scale (unless already .attn)
|
||||
re.compile(r"(?<!\.attn)\.([qkv])_scale$"): r".attn.\1_scale",
|
||||
re.compile(r"(?<!\.attn)\.([qkv])_zero_point$"): r".attn.\1_zero_point",
|
||||
}
|
||||
return WeightsMapper(orig_to_new_regex=orig_to_new_regex)
|
||||
|
||||
def apply_vllm_mapper( # noqa: B027
|
||||
self, hf_to_vllm_mapper: "WeightsMapper"
|
||||
|
||||
@@ -219,18 +219,19 @@ class Fp8Config(QuantizationConfig):
|
||||
return Fp8KVCacheMethod(self)
|
||||
return None
|
||||
|
||||
def get_cache_scale_mapper(self) -> "WeightsMapper":
|
||||
@staticmethod
|
||||
def get_cache_scale_mapper() -> "WeightsMapper":
|
||||
"""Map compressed-tensors KV-cache scale names to vLLM names."""
|
||||
from vllm.model_executor.models.utils import WeightsMapper
|
||||
|
||||
return WeightsMapper(
|
||||
orig_to_new_suffix={
|
||||
".k_proj.output_scale": ".attn.k_scale",
|
||||
".v_proj.output_scale": ".attn.v_scale",
|
||||
".q_proj.output_scale": ".attn.q_scale",
|
||||
".self_attn.prob_output_scale": ".self_attn.attn.prob_scale",
|
||||
}
|
||||
)
|
||||
orig_to_new_suffix = {
|
||||
".k_proj.output_scale": ".attn.k_scale",
|
||||
".v_proj.output_scale": ".attn.v_scale",
|
||||
".q_proj.output_scale": ".attn.q_scale",
|
||||
".self_attn.prob_output_scale": ".self_attn.attn.prob_scale",
|
||||
}
|
||||
cache_scale_mapper = WeightsMapper(orig_to_new_suffix=orig_to_new_suffix)
|
||||
return cache_scale_mapper | QuantizationConfig.get_cache_scale_mapper()
|
||||
|
||||
|
||||
class CopyNumelCounter(TorchDispatchMode):
|
||||
|
||||
@@ -679,16 +679,17 @@ class QuarkConfig(QuantizationConfig):
|
||||
|
||||
return scheme
|
||||
|
||||
def get_cache_scale_mapper(self) -> "WeightsMapper":
|
||||
@staticmethod
|
||||
def get_cache_scale_mapper() -> "WeightsMapper":
|
||||
"""Map Quark KV-cache scale names to vLLM names."""
|
||||
return WeightsMapper(
|
||||
orig_to_new_suffix={
|
||||
".k_proj.output_scale": ".attn.k_scale",
|
||||
".v_proj.output_scale": ".attn.v_scale",
|
||||
".q_proj.output_scale": ".attn.q_scale",
|
||||
".self_attn.prob_output_scale": ".self_attn.attn.prob_scale",
|
||||
}
|
||||
)
|
||||
orig_to_new_suffix = {
|
||||
".k_proj.output_scale": ".attn.k_scale",
|
||||
".v_proj.output_scale": ".attn.v_scale",
|
||||
".q_proj.output_scale": ".attn.q_scale",
|
||||
".self_attn.prob_output_scale": ".self_attn.attn.prob_scale",
|
||||
}
|
||||
cache_scale_mapper = WeightsMapper(orig_to_new_suffix=orig_to_new_suffix)
|
||||
return cache_scale_mapper | QuantizationConfig.get_cache_scale_mapper()
|
||||
|
||||
|
||||
class QuarkLinearMethod(LinearMethodBase):
|
||||
|
||||
Reference in New Issue
Block a user