diff --git a/tests/model_executor/model_loader/test_reload.py b/tests/model_executor/model_loader/test_reload.py index b3ed0c11bb..1480191083 100644 --- a/tests/model_executor/model_loader/test_reload.py +++ b/tests/model_executor/model_loader/test_reload.py @@ -71,6 +71,18 @@ class _ParentAliasedChildBufferLayer(torch.nn.Module): ) +class _ChildAliasOnlyBufferLayer(torch.nn.Module): + def __init__(self): + super().__init__() + self.conv1d = torch.nn.Linear(3, 2, bias=False) + self.conv1d.weight.data.copy_( + torch.arange(6, dtype=torch.float32).reshape(2, 3) + ) + self.register_buffer( + "conv_weights", self.conv1d.weight.detach().view(-1), persistent=False + ) + + class _AliasedBufferWithUninitializedChildLayer(_AliasedBufferLayer): def __init__(self): super().__init__() @@ -80,6 +92,13 @@ class _AliasedBufferWithUninitializedChildLayer(_AliasedBufferLayer): ) +class _NonPersistentBufferLayer(torch.nn.Module): + def __init__(self): + super().__init__() + self.weight = torch.nn.Parameter(torch.ones(2, 2)) + self.register_buffer("scale", torch.tensor(0.25), persistent=False) + + def test_move_metatensors(): tensor = torch.empty((1, 2, 3)) meta_tensor = to_meta_tensor(tensor) @@ -289,6 +308,8 @@ def test_layerwise_reload_skips_non_persistent_parameter_alias_buffers(monkeypat assert layer.weight_view.untyped_storage().data_ptr() == ( layer.weight.untyped_storage().data_ptr() ) + assert "weight_view" in layer._non_persistent_buffers_set + assert "0.weight_view" not in model.state_dict() def test_capture_layer_to_meta_skips_uninitialized_parameter_storage_ptrs(): @@ -332,6 +353,109 @@ def test_layerwise_reload_skips_child_parameter_alias_buffers(monkeypatch): assert layer.conv_weights.untyped_storage().data_ptr() == ( layer.conv1d.weight.untyped_storage().data_ptr() ) + assert "conv_weights" in layer._non_persistent_buffers_set + assert "0.conv_weights" not in model.state_dict() + + +def test_layerwise_reload_restores_alias_buffer_on_zero_size_layer(monkeypatch): + layer = _ChildAliasOnlyBufferLayer() + model = torch.nn.Sequential(layer) + loaded_conv = torch.full_like(layer.conv1d.weight, 7.0) + + def materialize_with_sentinel(meta_tensor): + tensor = torch.empty_strided( + size=tuple(meta_tensor.size()), + stride=tuple(meta_tensor.stride()), + dtype=meta_tensor.dtype, + requires_grad=False, + ) + tensor.fill_(-123.0) + tensor.__class__ = meta_tensor.__class__ + tensor.__dict__ = meta_tensor.__dict__.copy() + return tensor + + monkeypatch.setattr( + reload_meta, "materialize_meta_tensor", materialize_with_sentinel + ) + + record_metadata_for_reloading(model) + initialize_layerwise_reload(model) + layer.conv1d.weight.weight_loader(layer.conv1d.weight, loaded_conv) + finalize_layerwise_reload(model, model_config=None) + + assert torch.equal(layer.conv_weights, loaded_conv.view(-1)) + assert layer.conv_weights.untyped_storage().data_ptr() == ( + layer.conv1d.weight.untyped_storage().data_ptr() + ) + assert "conv_weights" in layer._non_persistent_buffers_set + assert "0.conv_weights" not in model.state_dict() + + +def test_layerwise_reload_preserves_unloaded_non_persistent_buffers(monkeypatch): + layer = _NonPersistentBufferLayer() + model = torch.nn.Sequential(layer) + loaded_weight = torch.full_like(layer.weight, 7.0) + original_scale = layer.scale.clone() + + def materialize_with_sentinel(meta_tensor): + tensor = torch.empty_strided( + size=tuple(meta_tensor.size()), + stride=tuple(meta_tensor.stride()), + dtype=meta_tensor.dtype, + requires_grad=False, + ) + tensor.fill_(-123.0) + tensor.__class__ = meta_tensor.__class__ + tensor.__dict__ = meta_tensor.__dict__.copy() + return tensor + + monkeypatch.setattr( + reload_meta, "materialize_meta_tensor", materialize_with_sentinel + ) + + record_metadata_for_reloading(model) + initialize_layerwise_reload(model) + layer.weight.weight_loader(layer.weight, loaded_weight) + finalize_layerwise_reload(model, model_config=None) + + assert torch.equal(layer.weight, loaded_weight) + assert torch.equal(layer.scale, original_scale) + assert "scale" in layer._non_persistent_buffers_set + assert "0.scale" not in model.state_dict() + + +def test_layerwise_reload_updates_loaded_non_persistent_buffers(monkeypatch): + layer = _NonPersistentBufferLayer() + model = torch.nn.Sequential(layer) + loaded_weight = torch.full_like(layer.weight, 7.0) + loaded_scale = torch.full_like(layer.scale, 0.5) + + def materialize_with_sentinel(meta_tensor): + tensor = torch.empty_strided( + size=tuple(meta_tensor.size()), + stride=tuple(meta_tensor.stride()), + dtype=meta_tensor.dtype, + requires_grad=False, + ) + tensor.fill_(-123.0) + tensor.__class__ = meta_tensor.__class__ + tensor.__dict__ = meta_tensor.__dict__.copy() + return tensor + + monkeypatch.setattr( + reload_meta, "materialize_meta_tensor", materialize_with_sentinel + ) + + record_metadata_for_reloading(model) + initialize_layerwise_reload(model) + layer.weight.weight_loader(layer.weight, loaded_weight) + layer.scale.weight_loader(layer.scale, loaded_scale) + finalize_layerwise_reload(model, model_config=None) + + assert torch.equal(layer.weight, loaded_weight) + assert torch.equal(layer.scale, loaded_scale) + assert "scale" in layer._non_persistent_buffers_set + assert "0.scale" not in model.state_dict() @pytest.mark.parametrize( diff --git a/vllm/model_executor/model_loader/reload/layerwise.py b/vllm/model_executor/model_loader/reload/layerwise.py index d0d26fed3e..eb609d63c5 100644 --- a/vllm/model_executor/model_loader/reload/layerwise.py +++ b/vllm/model_executor/model_loader/reload/layerwise.py @@ -109,6 +109,8 @@ def initialize_layerwise_reload(model: torch.nn.Module): # Save current tensors for later copying info.kernel_tensors = get_layer_params_buffers(layer) + # snapshot now: restore_layer_on_meta drops alias buffers from the live set + info.kernel_non_persistent_buffers = set(layer._non_persistent_buffers_set) # Restore layer parameters/buffers onto meta device restore_layer_on_meta(layer, info) @@ -259,10 +261,12 @@ def finalize_layerwise_processing(model: torch.nn.Module, model_config: ModelCon _layerwise_process(layer, info) continue - # reloading: place kernel tensors back as a fallback - elif info.load_numel_total > 0: # type: ignore[operator] + # reloading: place kernel tensors back as a fallback. Always place, even + # when nothing is loadable (load_numel_total == 0), so parameter-alias + # buffers on such layers are restored rather than left deleted. + if info.load_numel_total > 0: # type: ignore[operator] logger.warning("%s: Failed to load weights", layer.__class__.__name__) - _place_kernel_tensors(layer, info) + _place_kernel_tensors(layer, info) # Process non-attention layers which did not load all elements. This can happen # if the created weight has extra padding elements which are not loaded @@ -388,11 +392,15 @@ def _copy_and_restore_kernel_tensors(layer: torch.nn.Module, info: LayerReloadin kernel tensor references on the layer. Preserves cudagraph references.""" assert info.kernel_tensors is not None parameters, buffers = info.kernel_tensors + non_persistent = info.kernel_non_persistent_buffers + loaded_tensor_names = {name for name, _ in info.loaded_weights} for name, param in parameters.items(): param.data.copy_(getattr(layer, name)) for name, buffer in buffers.items(): if name not in layer._buffers: continue + if name in non_persistent and name not in loaded_tensor_names: + continue buffer.data.copy_(getattr(layer, name)) _place_kernel_tensors(layer, info) @@ -404,7 +412,8 @@ def _place_kernel_tensors(layer: torch.nn.Module, info: LayerReloadingInfo): assert info.kernel_tensors is not None parameters, buffers = info.kernel_tensors + non_persistent = info.kernel_non_persistent_buffers for name, param in parameters.items(): layer.register_parameter(name, param) for name, buffer in buffers.items(): - layer.register_buffer(name, buffer) + layer.register_buffer(name, buffer, persistent=name not in non_persistent) diff --git a/vllm/model_executor/model_loader/reload/meta.py b/vllm/model_executor/model_loader/reload/meta.py index 824d5c8b0f..33af4ab71f 100644 --- a/vllm/model_executor/model_loader/reload/meta.py +++ b/vllm/model_executor/model_loader/reload/meta.py @@ -117,6 +117,7 @@ def restore_layer_on_meta(layer: torch.nn.Module, info: LayerReloadingInfo): if layer.__class__.__name__ in SKIP_MODULES: return + non_persistent = set(layer._non_persistent_buffers_set) for name in get_layer_tensors(layer): if name not in SKIP_TENSORS: delattr(layer, name) @@ -130,7 +131,7 @@ def restore_layer_on_meta(layer: torch.nn.Module, info: LayerReloadingInfo): for name, buffer in restore_buffers.items(): if name not in SKIP_TENSORS: buffer = restore_layer_refs(buffer, layer) - layer.register_buffer(name, buffer) + layer.register_buffer(name, buffer, persistent=name not in non_persistent) def materialize_layer(layer: torch.nn.Module, info: LayerReloadingInfo): diff --git a/vllm/model_executor/model_loader/reload/types.py b/vllm/model_executor/model_loader/reload/types.py index 843d605ec0..320c8d3602 100644 --- a/vllm/model_executor/model_loader/reload/types.py +++ b/vllm/model_executor/model_loader/reload/types.py @@ -29,6 +29,10 @@ class LayerReloadingInfo: # kernel formatted tensors, copied into by `_layerwise_process` when reloading kernel_tensors: LayerTensors | None = None + # non-persistent buffer names captured with `kernel_tensors`, so buffer + # persistence survives `_non_persistent_buffers_set` being mutated during reload + kernel_non_persistent_buffers: set[str] = field(default_factory=set) + def reset(self): self.__init__( # type: ignore[misc] restore_metadata=self.restore_metadata, restore_device=self.restore_device