Remove unnecessary load_weights methods (#44589)
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
This commit is contained in:
@@ -3,9 +3,12 @@
|
||||
|
||||
import itertools
|
||||
from abc import abstractmethod
|
||||
from collections.abc import Iterable
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
from torch.nn.parameter import Parameter
|
||||
from typing_extensions import TypeIs
|
||||
|
||||
import vllm.envs as envs
|
||||
from vllm.distributed import (
|
||||
@@ -632,31 +635,31 @@ class MergedColumnParallelLinear(ColumnParallelLinear):
|
||||
disable_tp=disable_tp,
|
||||
)
|
||||
|
||||
def validate_shard_id(self, loaded_shard_id: int | tuple[int, ...] | None):
|
||||
if loaded_shard_id is None:
|
||||
return
|
||||
if isinstance(loaded_shard_id, tuple):
|
||||
for idx in loaded_shard_id:
|
||||
def validate_shard_id(self, shard_id: Any) -> TypeIs[int | tuple[int, ...] | None]:
|
||||
if isinstance(shard_id, int):
|
||||
if shard_id < 0 or shard_id >= len(self.output_sizes):
|
||||
raise ValueError(
|
||||
f"Shard id should be between 0 and {len(self.output_sizes) - 1}. "
|
||||
f"Got shard id {shard_id}."
|
||||
)
|
||||
return True
|
||||
if shard_id is None:
|
||||
return True
|
||||
if isinstance(shard_id, tuple):
|
||||
for idx in shard_id:
|
||||
if not (0 <= idx < len(self.output_sizes)):
|
||||
raise ValueError(
|
||||
f"Shard id index {idx} should be between 0 and "
|
||||
f"{len(self.output_sizes) - 1}. Got shard id {loaded_shard_id}."
|
||||
f"{len(self.output_sizes) - 1}. Got shard id {shard_id}."
|
||||
)
|
||||
if len(loaded_shard_id) > 1 and any(
|
||||
b - a != 1 for a, b in zip(loaded_shard_id[:-1], loaded_shard_id[1:])
|
||||
if len(shard_id) > 1 and any(
|
||||
b - a != 1 for a, b in zip(shard_id[:-1], shard_id[1:])
|
||||
):
|
||||
raise ValueError(
|
||||
"Shard id with multiple indices should be consecutive. "
|
||||
f"Got shard id {loaded_shard_id}."
|
||||
f"Got shard id {shard_id}."
|
||||
)
|
||||
return
|
||||
elif isinstance(loaded_shard_id, int):
|
||||
if loaded_shard_id < 0 or loaded_shard_id >= len(self.output_sizes):
|
||||
raise ValueError(
|
||||
f"Shard id should be between 0 and {len(self.output_sizes) - 1}. "
|
||||
f"Got shard id {loaded_shard_id}."
|
||||
)
|
||||
return
|
||||
return True
|
||||
raise ValueError("This line should not be reached")
|
||||
|
||||
def weight_loader(
|
||||
@@ -910,6 +913,31 @@ class MergedColumnParallelLinear(ColumnParallelLinear):
|
||||
tp_rank=self.tp_rank,
|
||||
)
|
||||
|
||||
def load_weights(
|
||||
self, weights: Iterable[tuple[str, torch.Tensor]]
|
||||
) -> Iterable[str]:
|
||||
for name, loaded_weight in weights:
|
||||
shard_id = getattr(loaded_weight, "shard_id", None)
|
||||
self.validate_shard_id(shard_id)
|
||||
# Load into self if name is not an attr of self or its submodules
|
||||
param: Parameter
|
||||
if "." in name:
|
||||
submodule, _, attr = name.rpartition(".")
|
||||
param = getattr(self.get_submodule(submodule), attr, self)
|
||||
else:
|
||||
param = getattr(self, name, self)
|
||||
if param is None and name == "bias":
|
||||
continue
|
||||
param.weight_loader(param, loaded_weight, shard_id)
|
||||
logger.debug(
|
||||
"Loaded shard %s with shape %s into %s.%s",
|
||||
shard_id,
|
||||
loaded_weight.shape,
|
||||
self.prefix,
|
||||
name,
|
||||
)
|
||||
yield name
|
||||
|
||||
|
||||
class QKVParallelLinear(ColumnParallelLinear):
|
||||
"""Linear layers for the attention's QKV transformation.
|
||||
@@ -996,17 +1024,13 @@ class QKVParallelLinear(ColumnParallelLinear):
|
||||
disable_tp=disable_tp,
|
||||
)
|
||||
|
||||
def validate_shard_id(self, loaded_shard_id: str | None):
|
||||
if loaded_shard_id is None:
|
||||
return
|
||||
if isinstance(loaded_shard_id, str):
|
||||
if loaded_shard_id not in ["q", "k", "v"]:
|
||||
raise ValueError(
|
||||
"Shard id for QKVParallelLinear should be 'q', 'k', or 'v', "
|
||||
f"got shard id {loaded_shard_id}."
|
||||
)
|
||||
return
|
||||
raise ValueError("This line should not be reached")
|
||||
def validate_shard_id(self, shard_id: Any) -> TypeIs[str | None]:
|
||||
if shard_id in {"q", "k", "v"} or shard_id is None:
|
||||
return True
|
||||
raise ValueError(
|
||||
"Shard id for QKVParallelLinear should be 'q', 'k', or 'v', "
|
||||
f"got shard id {shard_id}."
|
||||
)
|
||||
|
||||
def _get_shard_offset_mapping(self, loaded_shard_id: str):
|
||||
shard_offset_mapping = {
|
||||
@@ -1302,6 +1326,31 @@ class QKVParallelLinear(ColumnParallelLinear):
|
||||
assert param_data.shape == loaded_weight.shape
|
||||
param_data.copy_(loaded_weight)
|
||||
|
||||
def load_weights(
|
||||
self, weights: Iterable[tuple[str, torch.Tensor]]
|
||||
) -> Iterable[str]:
|
||||
for name, loaded_weight in weights:
|
||||
shard_id = getattr(loaded_weight, "shard_id", None)
|
||||
self.validate_shard_id(shard_id)
|
||||
# Load into self if name is not an attr of self or its submodules
|
||||
param: Parameter
|
||||
if "." in name:
|
||||
submodule, _, attr = name.rpartition(".")
|
||||
param = getattr(self.get_submodule(submodule), attr, self)
|
||||
else:
|
||||
param = getattr(self, name, self)
|
||||
if param is None and name == "bias":
|
||||
continue
|
||||
param.weight_loader(param, loaded_weight, shard_id)
|
||||
logger.debug(
|
||||
"Loaded shard %s with shape %s into %s.%s",
|
||||
shard_id,
|
||||
loaded_weight.shape,
|
||||
self.prefix,
|
||||
name,
|
||||
)
|
||||
yield name
|
||||
|
||||
|
||||
class MinimaxM3QKVParallelLinearWithIndexer(QKVParallelLinear):
|
||||
"""QKV projection fused with a lightning-indexer's index_q/index_k.
|
||||
@@ -1387,15 +1436,14 @@ class MinimaxM3QKVParallelLinearWithIndexer(QKVParallelLinear):
|
||||
prefix=prefix,
|
||||
)
|
||||
|
||||
def validate_shard_id(self, loaded_shard_id: str | None) -> None:
|
||||
if loaded_shard_id is None:
|
||||
return
|
||||
if loaded_shard_id not in ("q", "k", "v", "index_q", "index_k"):
|
||||
raise ValueError(
|
||||
"Shard id for MinimaxM3QKVParallelLinearWithIndexer must be one of "
|
||||
"'q', 'k', 'v', 'index_q', 'index_k'; got "
|
||||
f"{loaded_shard_id}."
|
||||
)
|
||||
def validate_shard_id(self, shard_id: Any) -> TypeIs[str | None]:
|
||||
if shard_id in {"q", "k", "v", "index_q", "index_k"} or shard_id is None:
|
||||
return True
|
||||
raise ValueError(
|
||||
"Shard id for MinimaxM3QKVParallelLinearWithIndexer must be one of "
|
||||
"'q', 'k', 'v', 'index_q', 'index_k'; got "
|
||||
f"{shard_id}."
|
||||
)
|
||||
|
||||
def _get_shard_offset_mapping(self, loaded_shard_id: str) -> int | None:
|
||||
h = self.head_size
|
||||
|
||||
Reference in New Issue
Block a user