[Bugfix][Model Loader] Defer post-load attention weight processing (#49519)

Signed-off-by: aoshen02 <aoshen@inferact.ai>
Co-authored-by: Roger Wang <hey@rogerw.io>
This commit is contained in:
aoshen02
2026-08-10 19:16:57 -07:00
committed by GitHub
co-authored by Roger Wang
parent f37dff22fc
commit 99e62b802c
4 changed files with 105 additions and 24 deletions
@@ -9,6 +9,9 @@ import torch
from torch.nn.parameter import UninitializedParameter from torch.nn.parameter import UninitializedParameter
import vllm.model_executor.model_loader.reload.meta as reload_meta import vllm.model_executor.model_loader.reload.meta as reload_meta
from vllm.config import ModelConfig
from vllm.model_executor.layers.attention import MMEncoderAttention
from vllm.model_executor.layers.attention_layer_base import AttentionLayerBase
from vllm.model_executor.layers.linear import QKVParallelLinear from vllm.model_executor.layers.linear import QKVParallelLinear
from vllm.model_executor.layers.quantization.base_config import QuantizeMethodBase from vllm.model_executor.layers.quantization.base_config import QuantizeMethodBase
from vllm.model_executor.model_loader.reload.layerwise import ( from vllm.model_executor.model_loader.reload.layerwise import (
@@ -101,6 +104,39 @@ class _NonPersistentBufferLayer(torch.nn.Module):
self.register_buffer("scale", torch.tensor(0.25), persistent=False) self.register_buffer("scale", torch.tensor(0.25), persistent=False)
class _ReloadableMMEncoderAttention(MMEncoderAttention):
"""Minimal stand-in to test reload lifecycle without encoder initialization."""
def __init__(self):
torch.nn.Module.__init__(self)
self.weight = torch.nn.Parameter(torch.ones(2, 2))
self.weight.weight_loader = default_weight_loader
self.post_load_called = False
def process_weights_after_loading(self, act_dtype: torch.dtype) -> None:
self.post_load_called = True
class _ReloadableAttentionLayer(
torch.nn.Module,
AttentionLayerBase,
):
def __init__(self):
super().__init__()
self.weight = torch.nn.Parameter(torch.ones(2, 2))
self.weight.weight_loader = default_weight_loader
self.post_load_called = False
def get_attn_backend(self):
raise NotImplementedError
def get_kv_cache_spec(self, vllm_config):
return None
def process_weights_after_loading(self, act_dtype: torch.dtype) -> None:
self.post_load_called = True
def test_move_metatensors(): def test_move_metatensors():
tensor = torch.empty((1, 2, 3)) tensor = torch.empty((1, 2, 3))
meta_tensor = to_meta_tensor(tensor) meta_tensor = to_meta_tensor(tensor)
@@ -115,6 +151,47 @@ def test_move_metatensors():
assert tensor.__dict__ == meta_tensor.__dict__ == materialized_tensor.__dict__ assert tensor.__dict__ == meta_tensor.__dict__ == materialized_tensor.__dict__
@pytest.mark.parametrize(
"layer_cls",
[_ReloadableMMEncoderAttention, _ReloadableAttentionLayer],
)
def test_attention_reload_defers_post_load(default_vllm_config, layer_cls):
default_vllm_config.model_config = ModelConfig()
layer = layer_cls()
model = torch.nn.Sequential(layer)
loaded_weight = torch.full_like(layer.weight, 7.0)
record_metadata_for_reloading(model)
initialize_layerwise_reload(model)
layer.weight.weight_loader(layer.weight, loaded_weight)
assert not layer.post_load_called
finalize_layerwise_reload(model, default_vllm_config.model_config)
assert layer.post_load_called
assert torch.equal(layer.weight, loaded_weight)
@pytest.mark.parametrize(
"layer_cls",
[_ReloadableMMEncoderAttention, _ReloadableAttentionLayer],
)
def test_attention_first_load_processes_weights(default_vllm_config, layer_cls):
default_vllm_config.model_config = ModelConfig()
layer = layer_cls()
model = torch.nn.Sequential(layer)
loaded_weight = torch.full_like(layer.weight, 7.0)
initialize_online_processing(layer)
layer.weight.weight_loader(layer.weight, loaded_weight)
finalize_layerwise_reload(model, default_vllm_config.model_config)
assert layer.post_load_called
assert torch.equal(layer.weight, loaded_weight)
def test_reload_lifecycle(): def test_reload_lifecycle():
layer = torch.nn.Linear(2, 3) layer = torch.nn.Linear(2, 3)
info = LayerReloadingInfo( info = LayerReloadingInfo(
@@ -1,6 +1,8 @@
# SPDX-License-Identifier: Apache-2.0 # SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import torch
from vllm.model_executor.layers.attention.attention import Attention from vllm.model_executor.layers.attention.attention import Attention
from vllm.model_executor.layers.attention.chunked_local_attention import ( from vllm.model_executor.layers.attention.chunked_local_attention import (
ChunkedLocalAttention, ChunkedLocalAttention,
@@ -18,6 +20,15 @@ from vllm.model_executor.layers.attention.rswa_attention import RSWAAttention
from vllm.model_executor.layers.attention.static_sink_attention import ( from vllm.model_executor.layers.attention.static_sink_attention import (
StaticSinkAttention, StaticSinkAttention,
) )
from vllm.model_executor.layers.attention_layer_base import AttentionLayerBase
def is_deferred_attention_layer(layer: torch.nn.Module) -> bool:
"""Whether an attention-like layer requires deferred post-load processing."""
return isinstance(layer, (AttentionLayerBase, MMEncoderAttention)) and callable(
getattr(layer, "process_weights_after_loading", None)
)
__all__ = [ __all__ = [
"Attention", "Attention",
@@ -29,4 +40,5 @@ __all__ = [
"PrefillPrefixLMAttention", "PrefillPrefixLMAttention",
"RSWAAttention", "RSWAAttention",
"StaticSinkAttention", "StaticSinkAttention",
"is_deferred_attention_layer",
] ]
@@ -9,7 +9,7 @@ import torch
from vllm.config import ModelConfig from vllm.config import ModelConfig
from vllm.logger import init_logger from vllm.logger import init_logger
from vllm.model_executor.layers.attention import Attention, MLAAttention from vllm.model_executor.layers.attention import is_deferred_attention_layer
from vllm.model_executor.layers.quantization.base_config import QuantizeMethodBase from vllm.model_executor.layers.quantization.base_config import QuantizeMethodBase
from vllm.model_executor.model_loader.weight_utils import default_weight_loader from vllm.model_executor.model_loader.weight_utils import default_weight_loader
@@ -196,7 +196,7 @@ def make_online_process_loader(layer: torch.nn.Module, param_name: str) -> Calla
) )
# Do not online process attention layers, must wait until finalize # Do not online process attention layers, must wait until finalize
if isinstance(layer, (Attention, MLAAttention)): if is_deferred_attention_layer(layer):
return ret return ret
# Log warnings allocating excessive buffers on device # Log warnings allocating excessive buffers on device
@@ -249,8 +249,8 @@ def finalize_layerwise_processing(model: torch.nn.Module, model_config: ModelCon
info.reset() info.reset()
continue continue
# Attention/MLA layers are processed after all other layers # Deferred attention-like layers are processed after all other layers
if isinstance(layer, (Attention, MLAAttention)): if is_deferred_attention_layer(layer):
deferred_attn.append((layer, info)) deferred_attn.append((layer, info))
continue continue
@@ -293,15 +293,13 @@ def finalize_layerwise_reload(*args, **kwargs):
def _finalize_attention_layer( def _finalize_attention_layer(
layer: torch.nn.Module, info: LayerReloadingInfo, model_config: ModelConfig layer: torch.nn.Module, info: LayerReloadingInfo, model_config: ModelConfig
) -> None: ) -> None:
if info.load_numel > 0 and info.kernel_tensors is not None: if info.kernel_tensors is None:
if info.load_numel > 0:
_layerwise_process(layer, info)
elif info.load_numel > 0:
# Reload with new scale weights from checkpoint # Reload with new scale weights from checkpoint
_place_kernel_tensors(layer, info) _place_kernel_tensors(layer, info)
_reload_attention_scales(layer, info) _reload_attention_scales(layer, info)
elif info.load_numel > 0 or info.kernel_tensors is None:
raise ValueError(
"Layerwise loading of attention layers is not supported. "
"Attention must always process after linears."
)
else: else:
_place_kernel_tensors(layer, info) _place_kernel_tensors(layer, info)
layer.process_weights_after_loading(model_config.dtype) layer.process_weights_after_loading(model_config.dtype)
@@ -315,19 +313,18 @@ def _reload_attention_scales(layer: torch.nn.Module, info: LayerReloadingInfo) -
processing, since we use .data.copy_() to preserve kernel tensor processing, since we use .data.copy_() to preserve kernel tensor
references.""" references."""
quant_method = getattr(layer, "quant_method", None) quant_method = getattr(layer, "quant_method", None)
if quant_method is None: if quant_method is not None:
return # Re-create scale Parameters with sentinel values so unloaded scales
# are correctly detected by process_weights_after_loading
# Re-create scale Parameters with sentinel values so unloaded scales quant_method.create_weights(layer)
# are correctly detected by process_weights_after_loading
quant_method.create_weights(layer)
for name, args in info.loaded_weights: for name, args in info.loaded_weights:
param = getattr(layer, name) param = getattr(layer, name)
args.arguments["param"] = param args.arguments["param"] = param
_get_weight_loader(param)(*args.args, **args.kwargs) _get_weight_loader(param)(*args.args, **args.kwargs)
quant_method.process_weights_after_loading(layer) if quant_method is not None:
quant_method.process_weights_after_loading(layer)
_copy_and_restore_kernel_tensors(layer, info) _copy_and_restore_kernel_tensors(layer, info)
+2 -7
View File
@@ -14,10 +14,7 @@ from typing_extensions import assert_never
import vllm.envs as envs import vllm.envs as envs
from vllm.config import ModelConfig, VllmConfig, set_current_vllm_config from vllm.config import ModelConfig, VllmConfig, set_current_vllm_config
from vllm.logger import init_logger from vllm.logger import init_logger
from vllm.model_executor.layers.attention import ( from vllm.model_executor.layers.attention import is_deferred_attention_layer
MMEncoderAttention,
)
from vllm.model_executor.layers.attention_layer_base import AttentionLayerBase
from vllm.model_executor.layers.hpc import HpcModule from vllm.model_executor.layers.hpc import HpcModule
from vllm.model_executor.layers.quantization.base_config import ( from vllm.model_executor.layers.quantization.base_config import (
QuantizationConfig, QuantizationConfig,
@@ -124,9 +121,7 @@ def process_weights_after_loading(
# encoder. NOTE: Happens after other modules so we can easily decompress # encoder. NOTE: Happens after other modules so we can easily decompress
# weights. # weights.
for _, module in model.named_modules(): for _, module in model.named_modules():
if isinstance(module, (AttentionLayerBase, MMEncoderAttention)) and hasattr( if is_deferred_attention_layer(module):
module, "process_weights_after_loading"
):
# TODO(lucas): see if there is a way to unify the signatures # TODO(lucas): see if there is a way to unify the signatures
# of process_weights_after_loading # of process_weights_after_loading
with device_loading_context(module, target_device): with device_loading_context(module, target_device):