[Core] Support weight_loader_v2 for UnquantizedLinearMethod (#23036)
Signed-off-by: Kyle Sayers <kylesayrs@gmail.com>
This commit is contained in:
@@ -22,6 +22,7 @@ from vllm.model_executor.layers.utils import dispatch_unquantized_gemm
|
||||
# yapf: disable
|
||||
from vllm.model_executor.parameter import (BasevLLMParameter,
|
||||
BlockQuantScaleParameter,
|
||||
ModelWeightParameter,
|
||||
PackedColumnParameter,
|
||||
PackedvLLMParameter,
|
||||
PerTensorScaleParameter,
|
||||
@@ -34,6 +35,7 @@ from vllm.utils import GiB_bytes
|
||||
logger = init_logger(__name__)
|
||||
|
||||
WEIGHT_LOADER_V2_SUPPORTED = [
|
||||
"UnquantizedLinearMethod",
|
||||
"CompressedTensorsLinearMethod",
|
||||
"CompressedTensorsLinearTransformMethod",
|
||||
"BitBLASLinearMethod",
|
||||
@@ -196,10 +198,14 @@ class UnquantizedLinearMethod(LinearMethodBase):
|
||||
# The amount of memory allocated for the weights is
|
||||
# sum(output_partition_sizes) * input_size_per_partition.
|
||||
try:
|
||||
weight = Parameter(torch.empty(sum(output_partition_sizes),
|
||||
input_size_per_partition,
|
||||
dtype=params_dtype),
|
||||
requires_grad=False)
|
||||
weight_loader = extra_weight_attrs.pop("weight_loader")
|
||||
weight = ModelWeightParameter(data=torch.empty(
|
||||
sum(output_partition_sizes),
|
||||
input_size_per_partition,
|
||||
dtype=params_dtype),
|
||||
input_dim=1,
|
||||
output_dim=0,
|
||||
weight_loader=weight_loader)
|
||||
except torch.cuda.OutOfMemoryError as e:
|
||||
logger.error("Failed to create unquantized linear weights: %s", e)
|
||||
if torch.cuda.is_available():
|
||||
@@ -212,7 +218,7 @@ class UnquantizedLinearMethod(LinearMethodBase):
|
||||
"Failed to create unquantized linear weights. "
|
||||
"This may be caused by insufficient memory to allocate "
|
||||
"the weight.") from e
|
||||
set_weight_attrs(weight, {"input_dim": 1, "output_dim": 0})
|
||||
|
||||
layer.register_parameter("weight", weight)
|
||||
set_weight_attrs(weight, extra_weight_attrs)
|
||||
|
||||
|
||||
@@ -61,9 +61,24 @@ class BasevLLMParameter(Parameter):
|
||||
self.tp_size = get_tensor_model_parallel_world_size()
|
||||
|
||||
@property
|
||||
def weight_loader(self):
|
||||
def weight_loader(self) -> Callable:
|
||||
# NOTE(@ksayers) some models such as mamba_mixer2 override the
|
||||
# weight loader to support custom loading. In the future, model-specific
|
||||
# weight loading should be implemented via Model.load_weights. In the
|
||||
# meantime, support deleting and overriding `weight_loader`` attribute
|
||||
if self._weight_loader is None:
|
||||
raise AttributeError(f"{self.__class__.__name__} weight_loader "
|
||||
"attribute has been deleted")
|
||||
return self._weight_loader
|
||||
|
||||
@weight_loader.setter
|
||||
def weight_loader(self, value: Callable):
|
||||
self._weight_loader = value
|
||||
|
||||
@weight_loader.deleter
|
||||
def weight_loader(self):
|
||||
self._weight_loader = None # type: ignore[assignment]
|
||||
|
||||
def _is_1d_and_scalar(self, loaded_weight: torch.Tensor):
|
||||
cond1 = self.data.ndim == 1 and self.data.numel() == 1
|
||||
cond2 = loaded_weight.ndim == 0 and loaded_weight.numel() == 1
|
||||
@@ -97,6 +112,12 @@ class BasevLLMParameter(Parameter):
|
||||
assert shard_id in qkv_idxs
|
||||
return qkv_idxs[shard_id]
|
||||
|
||||
@classmethod
|
||||
def __torch_function__(cls, func, types, args=(), kwargs=None):
|
||||
if kwargs is None:
|
||||
kwargs = {}
|
||||
return super().__torch_function__(func, types, args, kwargs)
|
||||
|
||||
|
||||
class _ColumnvLLMParameter(BasevLLMParameter):
|
||||
"""
|
||||
|
||||
Reference in New Issue
Block a user