[Bugfix]Fix transformer backend failed: AttributeError: 'Parameter' object has no attribute 'weight_loader' (#48699)
Signed-off-by: Lai, Yejing <yejing.lai@intel.com> Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> Co-authored-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
This commit is contained in:
co-authored by
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
Harry Mellor
parent
4c6e2e4b30
commit
7b3192523e
@@ -587,22 +587,23 @@ class Base(
|
||||
self.model: "PreTrainedModel" = AutoModel.from_config(...)
|
||||
```
|
||||
"""
|
||||
dtype = dtype or self.model_config.dtype
|
||||
device = self.device_config.device
|
||||
|
||||
def _init_parameters(module: nn.Module, dtype: torch.dtype | None):
|
||||
def _init_parameters(module: nn.Module):
|
||||
for name, param in module.named_parameters(recurse=False):
|
||||
if param.device == torch.device("meta"):
|
||||
new_param = nn.Parameter(
|
||||
torch.empty_like(
|
||||
param.data,
|
||||
dtype=dtype or self.model_config.dtype,
|
||||
device=self.device_config.device,
|
||||
)
|
||||
)
|
||||
setattr(module, name, new_param)
|
||||
# Already on device, nothing to do
|
||||
if param.device != torch.device("meta"):
|
||||
continue
|
||||
# Already a vLLM parameter, nothing to do
|
||||
if hasattr(param, "weight_loader"):
|
||||
continue
|
||||
data = torch.empty_like(param.data, dtype=dtype, device=device)
|
||||
setattr(module, name, nn.Parameter(data=data))
|
||||
for child in module.children():
|
||||
_init_parameters(child, dtype)
|
||||
_init_parameters(child)
|
||||
|
||||
_init_parameters(module, dtype)
|
||||
_init_parameters(module)
|
||||
|
||||
def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
|
||||
return self.model.get_input_embeddings()(input_ids)
|
||||
|
||||
Reference in New Issue
Block a user