[Weight processing] Copy over new_data attributes in replace_parameter (#49601)

Signed-off-by: Felix Marty <Felix.Marty@amd.com>
Co-authored-by: Claude <noreply@anthropic.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
fxmarty-amd
2026-08-06 13:03:37 -04:00
committed by GitHub
co-authored by Claude mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent b38e111d3e
commit 81be2e09ae
10 changed files with 334 additions and 51 deletions
+35 -13
View File
@@ -47,21 +47,29 @@ def set_weight_attrs(
def replace_parameter(
layer: torch.nn.Module,
param_name: str,
new_data: torch.Tensor | None,
new_tensor: torch.Tensor | None,
prefer_copy: bool = False,
):
"""
Replace a parameter of a layer while maintaining the ability to reload the weight.
Called within implementations of the `process_weights_after_loading` method.
Custom attributes set on ``new_tensor`` (e.g. kernel dispatch flags such as
``is_shuffled``) are carried over to the replacement parameter, except
``weight_loader``, which is always taken from the existing parameter.
Attributes of the existing parameter are otherwise dropped when a new
parameter is registered, but kept when ``prefer_copy`` reuses it in place.
This function should not be called on weights which are tied/shared
Args:
layer: Layer containing parameter to replace
param_name: Name of parameter to replace
new_data: New data of the new parameter, or None to set the parameter to None
new_tensor: New data of the new parameter, or None to set the parameter
to None
prefer_copy: If True and the existing parameter is compatible with
``new_data`` (same shape, dtype, and device), copy ``new_data``
``new_tensor`` (same shape, dtype, and device), copy ``new_tensor``
into the existing parameter in place rather than re-registering
a new parameter. This preserves the parameter's storage address
(``data_ptr``), which is required for captured CUDA graphs to
@@ -69,27 +77,41 @@ def replace_parameter(
"""
# should not be used on a tied/shared param
# If new_data is None, set the parameter to None
if new_data is None:
# If new_tensor is None, set the parameter to None
if new_tensor is None:
setattr(layer, param_name, None)
return
if isinstance(new_data, torch.nn.Parameter):
new_data = new_data.data
old_param: torch.nn.Parameter | None = getattr(layer, param_name, None)
new_tensor_attrs = dict(new_tensor.__dict__)
# `weight_loader` is the only attribute not ported over from new_tensor,
# old_param.weight_loader only is supported. `_weight_loader` is
# `BasevLLMParameter`'s backing field for its `weight_loader` property, so
# a loader riding along under that name has to be dropped as well.
new_tensor_attrs.pop("weight_loader", None)
new_tensor_attrs.pop("_weight_loader", None)
if isinstance(new_tensor, torch.nn.Parameter):
new_tensor = new_tensor.data
if (
prefer_copy
and old_param is not None
and old_param.shape == new_data.shape
and old_param.dtype == new_data.dtype
and old_param.device == new_data.device
and old_param.shape == new_tensor.shape
and old_param.dtype == new_tensor.dtype
and old_param.device == new_tensor.device
):
old_param.copy_(new_data)
old_param.copy_(new_tensor)
for attr_name, attr in new_tensor_attrs.items():
setattr(old_param, attr_name, attr)
return
new_param = torch.nn.Parameter(new_data, requires_grad=False)
new_param = torch.nn.Parameter(new_tensor, requires_grad=False)
for attr_name, attr in new_tensor_attrs.items():
setattr(new_param, attr_name, attr)
if old_param is not None and hasattr(old_param, "weight_loader"):
weight_loader = old_param.weight_loader