Remove unnecessary load_weights methods (#44589)
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
This commit is contained in:
@@ -3,9 +3,12 @@
|
||||
|
||||
import itertools
|
||||
from abc import abstractmethod
|
||||
from collections.abc import Iterable
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
from torch.nn.parameter import Parameter
|
||||
from typing_extensions import TypeIs
|
||||
|
||||
import vllm.envs as envs
|
||||
from vllm.distributed import (
|
||||
@@ -632,31 +635,31 @@ class MergedColumnParallelLinear(ColumnParallelLinear):
|
||||
disable_tp=disable_tp,
|
||||
)
|
||||
|
||||
def validate_shard_id(self, loaded_shard_id: int | tuple[int, ...] | None):
|
||||
if loaded_shard_id is None:
|
||||
return
|
||||
if isinstance(loaded_shard_id, tuple):
|
||||
for idx in loaded_shard_id:
|
||||
def validate_shard_id(self, shard_id: Any) -> TypeIs[int | tuple[int, ...] | None]:
|
||||
if isinstance(shard_id, int):
|
||||
if shard_id < 0 or shard_id >= len(self.output_sizes):
|
||||
raise ValueError(
|
||||
f"Shard id should be between 0 and {len(self.output_sizes) - 1}. "
|
||||
f"Got shard id {shard_id}."
|
||||
)
|
||||
return True
|
||||
if shard_id is None:
|
||||
return True
|
||||
if isinstance(shard_id, tuple):
|
||||
for idx in shard_id:
|
||||
if not (0 <= idx < len(self.output_sizes)):
|
||||
raise ValueError(
|
||||
f"Shard id index {idx} should be between 0 and "
|
||||
f"{len(self.output_sizes) - 1}. Got shard id {loaded_shard_id}."
|
||||
f"{len(self.output_sizes) - 1}. Got shard id {shard_id}."
|
||||
)
|
||||
if len(loaded_shard_id) > 1 and any(
|
||||
b - a != 1 for a, b in zip(loaded_shard_id[:-1], loaded_shard_id[1:])
|
||||
if len(shard_id) > 1 and any(
|
||||
b - a != 1 for a, b in zip(shard_id[:-1], shard_id[1:])
|
||||
):
|
||||
raise ValueError(
|
||||
"Shard id with multiple indices should be consecutive. "
|
||||
f"Got shard id {loaded_shard_id}."
|
||||
f"Got shard id {shard_id}."
|
||||
)
|
||||
return
|
||||
elif isinstance(loaded_shard_id, int):
|
||||
if loaded_shard_id < 0 or loaded_shard_id >= len(self.output_sizes):
|
||||
raise ValueError(
|
||||
f"Shard id should be between 0 and {len(self.output_sizes) - 1}. "
|
||||
f"Got shard id {loaded_shard_id}."
|
||||
)
|
||||
return
|
||||
return True
|
||||
raise ValueError("This line should not be reached")
|
||||
|
||||
def weight_loader(
|
||||
@@ -910,6 +913,31 @@ class MergedColumnParallelLinear(ColumnParallelLinear):
|
||||
tp_rank=self.tp_rank,
|
||||
)
|
||||
|
||||
def load_weights(
|
||||
self, weights: Iterable[tuple[str, torch.Tensor]]
|
||||
) -> Iterable[str]:
|
||||
for name, loaded_weight in weights:
|
||||
shard_id = getattr(loaded_weight, "shard_id", None)
|
||||
self.validate_shard_id(shard_id)
|
||||
# Load into self if name is not an attr of self or its submodules
|
||||
param: Parameter
|
||||
if "." in name:
|
||||
submodule, _, attr = name.rpartition(".")
|
||||
param = getattr(self.get_submodule(submodule), attr, self)
|
||||
else:
|
||||
param = getattr(self, name, self)
|
||||
if param is None and name == "bias":
|
||||
continue
|
||||
param.weight_loader(param, loaded_weight, shard_id)
|
||||
logger.debug(
|
||||
"Loaded shard %s with shape %s into %s.%s",
|
||||
shard_id,
|
||||
loaded_weight.shape,
|
||||
self.prefix,
|
||||
name,
|
||||
)
|
||||
yield name
|
||||
|
||||
|
||||
class QKVParallelLinear(ColumnParallelLinear):
|
||||
"""Linear layers for the attention's QKV transformation.
|
||||
@@ -996,17 +1024,13 @@ class QKVParallelLinear(ColumnParallelLinear):
|
||||
disable_tp=disable_tp,
|
||||
)
|
||||
|
||||
def validate_shard_id(self, loaded_shard_id: str | None):
|
||||
if loaded_shard_id is None:
|
||||
return
|
||||
if isinstance(loaded_shard_id, str):
|
||||
if loaded_shard_id not in ["q", "k", "v"]:
|
||||
raise ValueError(
|
||||
"Shard id for QKVParallelLinear should be 'q', 'k', or 'v', "
|
||||
f"got shard id {loaded_shard_id}."
|
||||
)
|
||||
return
|
||||
raise ValueError("This line should not be reached")
|
||||
def validate_shard_id(self, shard_id: Any) -> TypeIs[str | None]:
|
||||
if shard_id in {"q", "k", "v"} or shard_id is None:
|
||||
return True
|
||||
raise ValueError(
|
||||
"Shard id for QKVParallelLinear should be 'q', 'k', or 'v', "
|
||||
f"got shard id {shard_id}."
|
||||
)
|
||||
|
||||
def _get_shard_offset_mapping(self, loaded_shard_id: str):
|
||||
shard_offset_mapping = {
|
||||
@@ -1302,6 +1326,31 @@ class QKVParallelLinear(ColumnParallelLinear):
|
||||
assert param_data.shape == loaded_weight.shape
|
||||
param_data.copy_(loaded_weight)
|
||||
|
||||
def load_weights(
|
||||
self, weights: Iterable[tuple[str, torch.Tensor]]
|
||||
) -> Iterable[str]:
|
||||
for name, loaded_weight in weights:
|
||||
shard_id = getattr(loaded_weight, "shard_id", None)
|
||||
self.validate_shard_id(shard_id)
|
||||
# Load into self if name is not an attr of self or its submodules
|
||||
param: Parameter
|
||||
if "." in name:
|
||||
submodule, _, attr = name.rpartition(".")
|
||||
param = getattr(self.get_submodule(submodule), attr, self)
|
||||
else:
|
||||
param = getattr(self, name, self)
|
||||
if param is None and name == "bias":
|
||||
continue
|
||||
param.weight_loader(param, loaded_weight, shard_id)
|
||||
logger.debug(
|
||||
"Loaded shard %s with shape %s into %s.%s",
|
||||
shard_id,
|
||||
loaded_weight.shape,
|
||||
self.prefix,
|
||||
name,
|
||||
)
|
||||
yield name
|
||||
|
||||
|
||||
class MinimaxM3QKVParallelLinearWithIndexer(QKVParallelLinear):
|
||||
"""QKV projection fused with a lightning-indexer's index_q/index_k.
|
||||
@@ -1387,15 +1436,14 @@ class MinimaxM3QKVParallelLinearWithIndexer(QKVParallelLinear):
|
||||
prefix=prefix,
|
||||
)
|
||||
|
||||
def validate_shard_id(self, loaded_shard_id: str | None) -> None:
|
||||
if loaded_shard_id is None:
|
||||
return
|
||||
if loaded_shard_id not in ("q", "k", "v", "index_q", "index_k"):
|
||||
raise ValueError(
|
||||
"Shard id for MinimaxM3QKVParallelLinearWithIndexer must be one of "
|
||||
"'q', 'k', 'v', 'index_q', 'index_k'; got "
|
||||
f"{loaded_shard_id}."
|
||||
)
|
||||
def validate_shard_id(self, shard_id: Any) -> TypeIs[str | None]:
|
||||
if shard_id in {"q", "k", "v", "index_q", "index_k"} or shard_id is None:
|
||||
return True
|
||||
raise ValueError(
|
||||
"Shard id for MinimaxM3QKVParallelLinearWithIndexer must be one of "
|
||||
"'q', 'k', 'v', 'index_q', 'index_k'; got "
|
||||
f"{shard_id}."
|
||||
)
|
||||
|
||||
def _get_shard_offset_mapping(self, loaded_shard_id: str) -> int | None:
|
||||
h = self.head_size
|
||||
|
||||
@@ -5,6 +5,7 @@ import inspect
|
||||
from abc import ABC, abstractmethod
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
import regex as re
|
||||
import torch
|
||||
from torch import nn
|
||||
from transformers import PretrainedConfig
|
||||
@@ -19,10 +20,12 @@ else:
|
||||
class QuantizeMethodBase(ABC):
|
||||
"""Base class for different quantized methods."""
|
||||
|
||||
# Whether this method creates weights on meta device for online quantization.
|
||||
# When True, weights are created on meta device and quantized layer-wise
|
||||
# in process_weights_after_loading, reducing peak memory during loading.
|
||||
uses_meta_device: bool = False
|
||||
"""
|
||||
Whether this method creates weights on meta device for online quantization.
|
||||
When True, weights are created on meta device and quantized layer-wise
|
||||
in process_weights_after_loading, reducing peak memory during loading.
|
||||
"""
|
||||
|
||||
@abstractmethod
|
||||
def create_weights(
|
||||
@@ -84,6 +87,18 @@ def method_has_implemented_embedding(method_class: type[QuantizeMethodBase]) ->
|
||||
class QuantizationConfig(ABC):
|
||||
"""Base class for quantization configs."""
|
||||
|
||||
_ignore_unexpected_suffixes = (
|
||||
".q_scale",
|
||||
".k_scale",
|
||||
".v_scale",
|
||||
".q_zero_point",
|
||||
".k_zero_point",
|
||||
".v_zero_point",
|
||||
)
|
||||
"""Suffixes of quantization parameters that may be present in the checkpoint but
|
||||
not in the model, and should be ignored if unexpected during loading. These are used
|
||||
after remapping, so should be in vLLM format (e.g. .q_scale, not .q.scale)."""
|
||||
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
# mapping is updated by models as they initialize
|
||||
@@ -176,14 +191,40 @@ class QuantizationConfig(ABC):
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
def get_cache_scale_mapper(self) -> "WeightsMapper | None":
|
||||
@staticmethod
|
||||
def get_cache_scale_mapper() -> "WeightsMapper":
|
||||
"""Mapping from checkpoint KV-cache scale names to vLLM scale names.
|
||||
|
||||
Returning a mapper here causes `AutoWeightsLoader` to apply it to the
|
||||
weight stream automatically; individual model `load_weights` methods
|
||||
do not need to know about KV-cache scales.
|
||||
"""
|
||||
return None
|
||||
from vllm.model_executor.models.utils import WeightsMapper
|
||||
|
||||
orig_to_new_regex = {
|
||||
# Deprecated fused kv_scale -> attn.k_scale
|
||||
re.compile(r"\.kv_scale$"): r".attn.k_scale",
|
||||
# ModelOpt: .self_attn.{k,v}_proj.{k,v}_scale -> .self_attn.attn.*
|
||||
re.compile(r"\.self_attn\.[kv]_proj\.([kv])_scale$"): (
|
||||
r".self_attn.attn.\1_scale"
|
||||
),
|
||||
# Fused QKV / qkqkv proj: .self_attn.qk(qk)v_proj.{k,v}_scale -> attn
|
||||
re.compile(r"\.self_attn\.qk(?:qk)?v_proj\.([kv])_scale$"): (
|
||||
r".self_attn.attn.\1_scale"
|
||||
),
|
||||
# NemotronH: .mixer.{k,v}_proj.{k,v}_scale -> .mixer.attn.*
|
||||
re.compile(r"\.mixer\.[kv]_proj\.([kv])_scale$"): r".mixer.attn.\1_scale",
|
||||
# HYV3: .self_attn.q.scale -> .self_attn.attn.q_scale
|
||||
re.compile(r"\.self_attn\.q\.scale$"): r".self_attn.attn.q_scale",
|
||||
# HYV3: .self_attn.{k,v}_cache.scale -> .self_attn.attn.{k,v}_scale
|
||||
re.compile(r"\.self_attn\.([kv])_cache\.scale$"): (
|
||||
r".self_attn.attn.\1_scale"
|
||||
),
|
||||
# Default: .{q,k,v}_scale -> .attn.{q,k,v}_scale (unless already .attn)
|
||||
re.compile(r"(?<!\.attn)\.([qkv])_scale$"): r".attn.\1_scale",
|
||||
re.compile(r"(?<!\.attn)\.([qkv])_zero_point$"): r".attn.\1_zero_point",
|
||||
}
|
||||
return WeightsMapper(orig_to_new_regex=orig_to_new_regex)
|
||||
|
||||
def apply_vllm_mapper( # noqa: B027
|
||||
self, hf_to_vllm_mapper: "WeightsMapper"
|
||||
|
||||
@@ -219,18 +219,19 @@ class Fp8Config(QuantizationConfig):
|
||||
return Fp8KVCacheMethod(self)
|
||||
return None
|
||||
|
||||
def get_cache_scale_mapper(self) -> "WeightsMapper":
|
||||
@staticmethod
|
||||
def get_cache_scale_mapper() -> "WeightsMapper":
|
||||
"""Map compressed-tensors KV-cache scale names to vLLM names."""
|
||||
from vllm.model_executor.models.utils import WeightsMapper
|
||||
|
||||
return WeightsMapper(
|
||||
orig_to_new_suffix={
|
||||
".k_proj.output_scale": ".attn.k_scale",
|
||||
".v_proj.output_scale": ".attn.v_scale",
|
||||
".q_proj.output_scale": ".attn.q_scale",
|
||||
".self_attn.prob_output_scale": ".self_attn.attn.prob_scale",
|
||||
}
|
||||
)
|
||||
orig_to_new_suffix = {
|
||||
".k_proj.output_scale": ".attn.k_scale",
|
||||
".v_proj.output_scale": ".attn.v_scale",
|
||||
".q_proj.output_scale": ".attn.q_scale",
|
||||
".self_attn.prob_output_scale": ".self_attn.attn.prob_scale",
|
||||
}
|
||||
cache_scale_mapper = WeightsMapper(orig_to_new_suffix=orig_to_new_suffix)
|
||||
return cache_scale_mapper | QuantizationConfig.get_cache_scale_mapper()
|
||||
|
||||
|
||||
class CopyNumelCounter(TorchDispatchMode):
|
||||
|
||||
@@ -679,16 +679,17 @@ class QuarkConfig(QuantizationConfig):
|
||||
|
||||
return scheme
|
||||
|
||||
def get_cache_scale_mapper(self) -> "WeightsMapper":
|
||||
@staticmethod
|
||||
def get_cache_scale_mapper() -> "WeightsMapper":
|
||||
"""Map Quark KV-cache scale names to vLLM names."""
|
||||
return WeightsMapper(
|
||||
orig_to_new_suffix={
|
||||
".k_proj.output_scale": ".attn.k_scale",
|
||||
".v_proj.output_scale": ".attn.v_scale",
|
||||
".q_proj.output_scale": ".attn.q_scale",
|
||||
".self_attn.prob_output_scale": ".self_attn.attn.prob_scale",
|
||||
}
|
||||
)
|
||||
orig_to_new_suffix = {
|
||||
".k_proj.output_scale": ".attn.k_scale",
|
||||
".v_proj.output_scale": ".attn.v_scale",
|
||||
".q_proj.output_scale": ".attn.q_scale",
|
||||
".self_attn.prob_output_scale": ".self_attn.attn.prob_scale",
|
||||
}
|
||||
cache_scale_mapper = WeightsMapper(orig_to_new_suffix=orig_to_new_suffix)
|
||||
return cache_scale_mapper | QuantizationConfig.get_cache_scale_mapper()
|
||||
|
||||
|
||||
class QuarkLinearMethod(LinearMethodBase):
|
||||
|
||||
Reference in New Issue
Block a user