diff --git a/tests/model_executor/model_loader/test_reload.py b/tests/model_executor/model_loader/test_reload.py index c21d9b1e02..4963f3ae5b 100644 --- a/tests/model_executor/model_loader/test_reload.py +++ b/tests/model_executor/model_loader/test_reload.py @@ -9,6 +9,9 @@ import torch from torch.nn.parameter import UninitializedParameter import vllm.model_executor.model_loader.reload.meta as reload_meta +from vllm.config import ModelConfig +from vllm.model_executor.layers.attention import MMEncoderAttention +from vllm.model_executor.layers.attention_layer_base import AttentionLayerBase from vllm.model_executor.layers.linear import QKVParallelLinear from vllm.model_executor.layers.quantization.base_config import QuantizeMethodBase from vllm.model_executor.model_loader.reload.layerwise import ( @@ -101,6 +104,39 @@ class _NonPersistentBufferLayer(torch.nn.Module): self.register_buffer("scale", torch.tensor(0.25), persistent=False) +class _ReloadableMMEncoderAttention(MMEncoderAttention): + """Minimal stand-in to test reload lifecycle without encoder initialization.""" + + def __init__(self): + torch.nn.Module.__init__(self) + self.weight = torch.nn.Parameter(torch.ones(2, 2)) + self.weight.weight_loader = default_weight_loader + self.post_load_called = False + + def process_weights_after_loading(self, act_dtype: torch.dtype) -> None: + self.post_load_called = True + + +class _ReloadableAttentionLayer( + torch.nn.Module, + AttentionLayerBase, +): + def __init__(self): + super().__init__() + self.weight = torch.nn.Parameter(torch.ones(2, 2)) + self.weight.weight_loader = default_weight_loader + self.post_load_called = False + + def get_attn_backend(self): + raise NotImplementedError + + def get_kv_cache_spec(self, vllm_config): + return None + + def process_weights_after_loading(self, act_dtype: torch.dtype) -> None: + self.post_load_called = True + + def test_move_metatensors(): tensor = torch.empty((1, 2, 3)) meta_tensor = to_meta_tensor(tensor) @@ -115,6 +151,47 @@ def test_move_metatensors(): assert tensor.__dict__ == meta_tensor.__dict__ == materialized_tensor.__dict__ +@pytest.mark.parametrize( + "layer_cls", + [_ReloadableMMEncoderAttention, _ReloadableAttentionLayer], +) +def test_attention_reload_defers_post_load(default_vllm_config, layer_cls): + default_vllm_config.model_config = ModelConfig() + layer = layer_cls() + model = torch.nn.Sequential(layer) + loaded_weight = torch.full_like(layer.weight, 7.0) + + record_metadata_for_reloading(model) + initialize_layerwise_reload(model) + layer.weight.weight_loader(layer.weight, loaded_weight) + + assert not layer.post_load_called + + finalize_layerwise_reload(model, default_vllm_config.model_config) + + assert layer.post_load_called + assert torch.equal(layer.weight, loaded_weight) + + +@pytest.mark.parametrize( + "layer_cls", + [_ReloadableMMEncoderAttention, _ReloadableAttentionLayer], +) +def test_attention_first_load_processes_weights(default_vllm_config, layer_cls): + default_vllm_config.model_config = ModelConfig() + layer = layer_cls() + model = torch.nn.Sequential(layer) + loaded_weight = torch.full_like(layer.weight, 7.0) + + initialize_online_processing(layer) + layer.weight.weight_loader(layer.weight, loaded_weight) + + finalize_layerwise_reload(model, default_vllm_config.model_config) + + assert layer.post_load_called + assert torch.equal(layer.weight, loaded_weight) + + def test_reload_lifecycle(): layer = torch.nn.Linear(2, 3) info = LayerReloadingInfo( diff --git a/vllm/model_executor/layers/attention/__init__.py b/vllm/model_executor/layers/attention/__init__.py index c9e477fb11..4b04c84103 100644 --- a/vllm/model_executor/layers/attention/__init__.py +++ b/vllm/model_executor/layers/attention/__init__.py @@ -1,6 +1,8 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project +import torch + from vllm.model_executor.layers.attention.attention import Attention from vllm.model_executor.layers.attention.chunked_local_attention import ( ChunkedLocalAttention, @@ -18,6 +20,15 @@ from vllm.model_executor.layers.attention.rswa_attention import RSWAAttention from vllm.model_executor.layers.attention.static_sink_attention import ( StaticSinkAttention, ) +from vllm.model_executor.layers.attention_layer_base import AttentionLayerBase + + +def is_deferred_attention_layer(layer: torch.nn.Module) -> bool: + """Whether an attention-like layer requires deferred post-load processing.""" + return isinstance(layer, (AttentionLayerBase, MMEncoderAttention)) and callable( + getattr(layer, "process_weights_after_loading", None) + ) + __all__ = [ "Attention", @@ -29,4 +40,5 @@ __all__ = [ "PrefillPrefixLMAttention", "RSWAAttention", "StaticSinkAttention", + "is_deferred_attention_layer", ] diff --git a/vllm/model_executor/model_loader/reload/layerwise.py b/vllm/model_executor/model_loader/reload/layerwise.py index d6d4eb4b93..5aa510a624 100644 --- a/vllm/model_executor/model_loader/reload/layerwise.py +++ b/vllm/model_executor/model_loader/reload/layerwise.py @@ -9,7 +9,7 @@ import torch from vllm.config import ModelConfig from vllm.logger import init_logger -from vllm.model_executor.layers.attention import Attention, MLAAttention +from vllm.model_executor.layers.attention import is_deferred_attention_layer from vllm.model_executor.layers.quantization.base_config import QuantizeMethodBase from vllm.model_executor.model_loader.weight_utils import default_weight_loader @@ -196,7 +196,7 @@ def make_online_process_loader(layer: torch.nn.Module, param_name: str) -> Calla ) # Do not online process attention layers, must wait until finalize - if isinstance(layer, (Attention, MLAAttention)): + if is_deferred_attention_layer(layer): return ret # Log warnings allocating excessive buffers on device @@ -249,8 +249,8 @@ def finalize_layerwise_processing(model: torch.nn.Module, model_config: ModelCon info.reset() continue - # Attention/MLA layers are processed after all other layers - if isinstance(layer, (Attention, MLAAttention)): + # Deferred attention-like layers are processed after all other layers + if is_deferred_attention_layer(layer): deferred_attn.append((layer, info)) continue @@ -293,15 +293,13 @@ def finalize_layerwise_reload(*args, **kwargs): def _finalize_attention_layer( layer: torch.nn.Module, info: LayerReloadingInfo, model_config: ModelConfig ) -> None: - if info.load_numel > 0 and info.kernel_tensors is not None: + if info.kernel_tensors is None: + if info.load_numel > 0: + _layerwise_process(layer, info) + elif info.load_numel > 0: # Reload with new scale weights from checkpoint _place_kernel_tensors(layer, info) _reload_attention_scales(layer, info) - elif info.load_numel > 0 or info.kernel_tensors is None: - raise ValueError( - "Layerwise loading of attention layers is not supported. " - "Attention must always process after linears." - ) else: _place_kernel_tensors(layer, info) layer.process_weights_after_loading(model_config.dtype) @@ -315,19 +313,18 @@ def _reload_attention_scales(layer: torch.nn.Module, info: LayerReloadingInfo) - processing, since we use .data.copy_() to preserve kernel tensor references.""" quant_method = getattr(layer, "quant_method", None) - if quant_method is None: - return - - # Re-create scale Parameters with sentinel values so unloaded scales - # are correctly detected by process_weights_after_loading - quant_method.create_weights(layer) + if quant_method is not None: + # Re-create scale Parameters with sentinel values so unloaded scales + # are correctly detected by process_weights_after_loading + quant_method.create_weights(layer) for name, args in info.loaded_weights: param = getattr(layer, name) args.arguments["param"] = param _get_weight_loader(param)(*args.args, **args.kwargs) - quant_method.process_weights_after_loading(layer) + if quant_method is not None: + quant_method.process_weights_after_loading(layer) _copy_and_restore_kernel_tensors(layer, info) diff --git a/vllm/model_executor/model_loader/utils.py b/vllm/model_executor/model_loader/utils.py index a472c75435..b7ccb78c04 100644 --- a/vllm/model_executor/model_loader/utils.py +++ b/vllm/model_executor/model_loader/utils.py @@ -14,10 +14,7 @@ from typing_extensions import assert_never import vllm.envs as envs from vllm.config import ModelConfig, VllmConfig, set_current_vllm_config from vllm.logger import init_logger -from vllm.model_executor.layers.attention import ( - MMEncoderAttention, -) -from vllm.model_executor.layers.attention_layer_base import AttentionLayerBase +from vllm.model_executor.layers.attention import is_deferred_attention_layer from vllm.model_executor.layers.hpc import HpcModule from vllm.model_executor.layers.quantization.base_config import ( QuantizationConfig, @@ -124,9 +121,7 @@ def process_weights_after_loading( # encoder. NOTE: Happens after other modules so we can easily decompress # weights. for _, module in model.named_modules(): - if isinstance(module, (AttentionLayerBase, MMEncoderAttention)) and hasattr( - module, "process_weights_after_loading" - ): + if is_deferred_attention_layer(module): # TODO(lucas): see if there is a way to unify the signatures # of process_weights_after_loading with device_loading_context(module, target_device):