[Bugfix] Preserve unloaded non-persistent buffers during layerwise reload (#44371)
Signed-off-by: Joan Velja <joan.velja22@gmail.com> Co-authored-by: Dakai An <77474977+andakai@users.noreply.github.com>
This commit is contained in:
@@ -71,6 +71,18 @@ class _ParentAliasedChildBufferLayer(torch.nn.Module):
|
||||
)
|
||||
|
||||
|
||||
class _ChildAliasOnlyBufferLayer(torch.nn.Module):
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self.conv1d = torch.nn.Linear(3, 2, bias=False)
|
||||
self.conv1d.weight.data.copy_(
|
||||
torch.arange(6, dtype=torch.float32).reshape(2, 3)
|
||||
)
|
||||
self.register_buffer(
|
||||
"conv_weights", self.conv1d.weight.detach().view(-1), persistent=False
|
||||
)
|
||||
|
||||
|
||||
class _AliasedBufferWithUninitializedChildLayer(_AliasedBufferLayer):
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
@@ -80,6 +92,13 @@ class _AliasedBufferWithUninitializedChildLayer(_AliasedBufferLayer):
|
||||
)
|
||||
|
||||
|
||||
class _NonPersistentBufferLayer(torch.nn.Module):
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self.weight = torch.nn.Parameter(torch.ones(2, 2))
|
||||
self.register_buffer("scale", torch.tensor(0.25), persistent=False)
|
||||
|
||||
|
||||
def test_move_metatensors():
|
||||
tensor = torch.empty((1, 2, 3))
|
||||
meta_tensor = to_meta_tensor(tensor)
|
||||
@@ -289,6 +308,8 @@ def test_layerwise_reload_skips_non_persistent_parameter_alias_buffers(monkeypat
|
||||
assert layer.weight_view.untyped_storage().data_ptr() == (
|
||||
layer.weight.untyped_storage().data_ptr()
|
||||
)
|
||||
assert "weight_view" in layer._non_persistent_buffers_set
|
||||
assert "0.weight_view" not in model.state_dict()
|
||||
|
||||
|
||||
def test_capture_layer_to_meta_skips_uninitialized_parameter_storage_ptrs():
|
||||
@@ -332,6 +353,109 @@ def test_layerwise_reload_skips_child_parameter_alias_buffers(monkeypatch):
|
||||
assert layer.conv_weights.untyped_storage().data_ptr() == (
|
||||
layer.conv1d.weight.untyped_storage().data_ptr()
|
||||
)
|
||||
assert "conv_weights" in layer._non_persistent_buffers_set
|
||||
assert "0.conv_weights" not in model.state_dict()
|
||||
|
||||
|
||||
def test_layerwise_reload_restores_alias_buffer_on_zero_size_layer(monkeypatch):
|
||||
layer = _ChildAliasOnlyBufferLayer()
|
||||
model = torch.nn.Sequential(layer)
|
||||
loaded_conv = torch.full_like(layer.conv1d.weight, 7.0)
|
||||
|
||||
def materialize_with_sentinel(meta_tensor):
|
||||
tensor = torch.empty_strided(
|
||||
size=tuple(meta_tensor.size()),
|
||||
stride=tuple(meta_tensor.stride()),
|
||||
dtype=meta_tensor.dtype,
|
||||
requires_grad=False,
|
||||
)
|
||||
tensor.fill_(-123.0)
|
||||
tensor.__class__ = meta_tensor.__class__
|
||||
tensor.__dict__ = meta_tensor.__dict__.copy()
|
||||
return tensor
|
||||
|
||||
monkeypatch.setattr(
|
||||
reload_meta, "materialize_meta_tensor", materialize_with_sentinel
|
||||
)
|
||||
|
||||
record_metadata_for_reloading(model)
|
||||
initialize_layerwise_reload(model)
|
||||
layer.conv1d.weight.weight_loader(layer.conv1d.weight, loaded_conv)
|
||||
finalize_layerwise_reload(model, model_config=None)
|
||||
|
||||
assert torch.equal(layer.conv_weights, loaded_conv.view(-1))
|
||||
assert layer.conv_weights.untyped_storage().data_ptr() == (
|
||||
layer.conv1d.weight.untyped_storage().data_ptr()
|
||||
)
|
||||
assert "conv_weights" in layer._non_persistent_buffers_set
|
||||
assert "0.conv_weights" not in model.state_dict()
|
||||
|
||||
|
||||
def test_layerwise_reload_preserves_unloaded_non_persistent_buffers(monkeypatch):
|
||||
layer = _NonPersistentBufferLayer()
|
||||
model = torch.nn.Sequential(layer)
|
||||
loaded_weight = torch.full_like(layer.weight, 7.0)
|
||||
original_scale = layer.scale.clone()
|
||||
|
||||
def materialize_with_sentinel(meta_tensor):
|
||||
tensor = torch.empty_strided(
|
||||
size=tuple(meta_tensor.size()),
|
||||
stride=tuple(meta_tensor.stride()),
|
||||
dtype=meta_tensor.dtype,
|
||||
requires_grad=False,
|
||||
)
|
||||
tensor.fill_(-123.0)
|
||||
tensor.__class__ = meta_tensor.__class__
|
||||
tensor.__dict__ = meta_tensor.__dict__.copy()
|
||||
return tensor
|
||||
|
||||
monkeypatch.setattr(
|
||||
reload_meta, "materialize_meta_tensor", materialize_with_sentinel
|
||||
)
|
||||
|
||||
record_metadata_for_reloading(model)
|
||||
initialize_layerwise_reload(model)
|
||||
layer.weight.weight_loader(layer.weight, loaded_weight)
|
||||
finalize_layerwise_reload(model, model_config=None)
|
||||
|
||||
assert torch.equal(layer.weight, loaded_weight)
|
||||
assert torch.equal(layer.scale, original_scale)
|
||||
assert "scale" in layer._non_persistent_buffers_set
|
||||
assert "0.scale" not in model.state_dict()
|
||||
|
||||
|
||||
def test_layerwise_reload_updates_loaded_non_persistent_buffers(monkeypatch):
|
||||
layer = _NonPersistentBufferLayer()
|
||||
model = torch.nn.Sequential(layer)
|
||||
loaded_weight = torch.full_like(layer.weight, 7.0)
|
||||
loaded_scale = torch.full_like(layer.scale, 0.5)
|
||||
|
||||
def materialize_with_sentinel(meta_tensor):
|
||||
tensor = torch.empty_strided(
|
||||
size=tuple(meta_tensor.size()),
|
||||
stride=tuple(meta_tensor.stride()),
|
||||
dtype=meta_tensor.dtype,
|
||||
requires_grad=False,
|
||||
)
|
||||
tensor.fill_(-123.0)
|
||||
tensor.__class__ = meta_tensor.__class__
|
||||
tensor.__dict__ = meta_tensor.__dict__.copy()
|
||||
return tensor
|
||||
|
||||
monkeypatch.setattr(
|
||||
reload_meta, "materialize_meta_tensor", materialize_with_sentinel
|
||||
)
|
||||
|
||||
record_metadata_for_reloading(model)
|
||||
initialize_layerwise_reload(model)
|
||||
layer.weight.weight_loader(layer.weight, loaded_weight)
|
||||
layer.scale.weight_loader(layer.scale, loaded_scale)
|
||||
finalize_layerwise_reload(model, model_config=None)
|
||||
|
||||
assert torch.equal(layer.weight, loaded_weight)
|
||||
assert torch.equal(layer.scale, loaded_scale)
|
||||
assert "scale" in layer._non_persistent_buffers_set
|
||||
assert "0.scale" not in model.state_dict()
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
|
||||
@@ -109,6 +109,8 @@ def initialize_layerwise_reload(model: torch.nn.Module):
|
||||
|
||||
# Save current tensors for later copying
|
||||
info.kernel_tensors = get_layer_params_buffers(layer)
|
||||
# snapshot now: restore_layer_on_meta drops alias buffers from the live set
|
||||
info.kernel_non_persistent_buffers = set(layer._non_persistent_buffers_set)
|
||||
|
||||
# Restore layer parameters/buffers onto meta device
|
||||
restore_layer_on_meta(layer, info)
|
||||
@@ -259,10 +261,12 @@ def finalize_layerwise_processing(model: torch.nn.Module, model_config: ModelCon
|
||||
_layerwise_process(layer, info)
|
||||
continue
|
||||
|
||||
# reloading: place kernel tensors back as a fallback
|
||||
elif info.load_numel_total > 0: # type: ignore[operator]
|
||||
# reloading: place kernel tensors back as a fallback. Always place, even
|
||||
# when nothing is loadable (load_numel_total == 0), so parameter-alias
|
||||
# buffers on such layers are restored rather than left deleted.
|
||||
if info.load_numel_total > 0: # type: ignore[operator]
|
||||
logger.warning("%s: Failed to load weights", layer.__class__.__name__)
|
||||
_place_kernel_tensors(layer, info)
|
||||
_place_kernel_tensors(layer, info)
|
||||
|
||||
# Process non-attention layers which did not load all elements. This can happen
|
||||
# if the created weight has extra padding elements which are not loaded
|
||||
@@ -388,11 +392,15 @@ def _copy_and_restore_kernel_tensors(layer: torch.nn.Module, info: LayerReloadin
|
||||
kernel tensor references on the layer. Preserves cudagraph references."""
|
||||
assert info.kernel_tensors is not None
|
||||
parameters, buffers = info.kernel_tensors
|
||||
non_persistent = info.kernel_non_persistent_buffers
|
||||
loaded_tensor_names = {name for name, _ in info.loaded_weights}
|
||||
for name, param in parameters.items():
|
||||
param.data.copy_(getattr(layer, name))
|
||||
for name, buffer in buffers.items():
|
||||
if name not in layer._buffers:
|
||||
continue
|
||||
if name in non_persistent and name not in loaded_tensor_names:
|
||||
continue
|
||||
buffer.data.copy_(getattr(layer, name))
|
||||
|
||||
_place_kernel_tensors(layer, info)
|
||||
@@ -404,7 +412,8 @@ def _place_kernel_tensors(layer: torch.nn.Module, info: LayerReloadingInfo):
|
||||
|
||||
assert info.kernel_tensors is not None
|
||||
parameters, buffers = info.kernel_tensors
|
||||
non_persistent = info.kernel_non_persistent_buffers
|
||||
for name, param in parameters.items():
|
||||
layer.register_parameter(name, param)
|
||||
for name, buffer in buffers.items():
|
||||
layer.register_buffer(name, buffer)
|
||||
layer.register_buffer(name, buffer, persistent=name not in non_persistent)
|
||||
|
||||
@@ -117,6 +117,7 @@ def restore_layer_on_meta(layer: torch.nn.Module, info: LayerReloadingInfo):
|
||||
if layer.__class__.__name__ in SKIP_MODULES:
|
||||
return
|
||||
|
||||
non_persistent = set(layer._non_persistent_buffers_set)
|
||||
for name in get_layer_tensors(layer):
|
||||
if name not in SKIP_TENSORS:
|
||||
delattr(layer, name)
|
||||
@@ -130,7 +131,7 @@ def restore_layer_on_meta(layer: torch.nn.Module, info: LayerReloadingInfo):
|
||||
for name, buffer in restore_buffers.items():
|
||||
if name not in SKIP_TENSORS:
|
||||
buffer = restore_layer_refs(buffer, layer)
|
||||
layer.register_buffer(name, buffer)
|
||||
layer.register_buffer(name, buffer, persistent=name not in non_persistent)
|
||||
|
||||
|
||||
def materialize_layer(layer: torch.nn.Module, info: LayerReloadingInfo):
|
||||
|
||||
@@ -29,6 +29,10 @@ class LayerReloadingInfo:
|
||||
# kernel formatted tensors, copied into by `_layerwise_process` when reloading
|
||||
kernel_tensors: LayerTensors | None = None
|
||||
|
||||
# non-persistent buffer names captured with `kernel_tensors`, so buffer
|
||||
# persistence survives `_non_persistent_buffers_set` being mutated during reload
|
||||
kernel_non_persistent_buffers: set[str] = field(default_factory=set)
|
||||
|
||||
def reset(self):
|
||||
self.__init__( # type: ignore[misc]
|
||||
restore_metadata=self.restore_metadata, restore_device=self.restore_device
|
||||
|
||||
Reference in New Issue
Block a user