[Bugfix][Model Loader] Defer post-load attention weight processing (#49519)
Signed-off-by: aoshen02 <aoshen@inferact.ai> Co-authored-by: Roger Wang <hey@rogerw.io>
This commit is contained in:
@@ -9,6 +9,9 @@ import torch
|
||||
from torch.nn.parameter import UninitializedParameter
|
||||
|
||||
import vllm.model_executor.model_loader.reload.meta as reload_meta
|
||||
from vllm.config import ModelConfig
|
||||
from vllm.model_executor.layers.attention import MMEncoderAttention
|
||||
from vllm.model_executor.layers.attention_layer_base import AttentionLayerBase
|
||||
from vllm.model_executor.layers.linear import QKVParallelLinear
|
||||
from vllm.model_executor.layers.quantization.base_config import QuantizeMethodBase
|
||||
from vllm.model_executor.model_loader.reload.layerwise import (
|
||||
@@ -101,6 +104,39 @@ class _NonPersistentBufferLayer(torch.nn.Module):
|
||||
self.register_buffer("scale", torch.tensor(0.25), persistent=False)
|
||||
|
||||
|
||||
class _ReloadableMMEncoderAttention(MMEncoderAttention):
|
||||
"""Minimal stand-in to test reload lifecycle without encoder initialization."""
|
||||
|
||||
def __init__(self):
|
||||
torch.nn.Module.__init__(self)
|
||||
self.weight = torch.nn.Parameter(torch.ones(2, 2))
|
||||
self.weight.weight_loader = default_weight_loader
|
||||
self.post_load_called = False
|
||||
|
||||
def process_weights_after_loading(self, act_dtype: torch.dtype) -> None:
|
||||
self.post_load_called = True
|
||||
|
||||
|
||||
class _ReloadableAttentionLayer(
|
||||
torch.nn.Module,
|
||||
AttentionLayerBase,
|
||||
):
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self.weight = torch.nn.Parameter(torch.ones(2, 2))
|
||||
self.weight.weight_loader = default_weight_loader
|
||||
self.post_load_called = False
|
||||
|
||||
def get_attn_backend(self):
|
||||
raise NotImplementedError
|
||||
|
||||
def get_kv_cache_spec(self, vllm_config):
|
||||
return None
|
||||
|
||||
def process_weights_after_loading(self, act_dtype: torch.dtype) -> None:
|
||||
self.post_load_called = True
|
||||
|
||||
|
||||
def test_move_metatensors():
|
||||
tensor = torch.empty((1, 2, 3))
|
||||
meta_tensor = to_meta_tensor(tensor)
|
||||
@@ -115,6 +151,47 @@ def test_move_metatensors():
|
||||
assert tensor.__dict__ == meta_tensor.__dict__ == materialized_tensor.__dict__
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"layer_cls",
|
||||
[_ReloadableMMEncoderAttention, _ReloadableAttentionLayer],
|
||||
)
|
||||
def test_attention_reload_defers_post_load(default_vllm_config, layer_cls):
|
||||
default_vllm_config.model_config = ModelConfig()
|
||||
layer = layer_cls()
|
||||
model = torch.nn.Sequential(layer)
|
||||
loaded_weight = torch.full_like(layer.weight, 7.0)
|
||||
|
||||
record_metadata_for_reloading(model)
|
||||
initialize_layerwise_reload(model)
|
||||
layer.weight.weight_loader(layer.weight, loaded_weight)
|
||||
|
||||
assert not layer.post_load_called
|
||||
|
||||
finalize_layerwise_reload(model, default_vllm_config.model_config)
|
||||
|
||||
assert layer.post_load_called
|
||||
assert torch.equal(layer.weight, loaded_weight)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"layer_cls",
|
||||
[_ReloadableMMEncoderAttention, _ReloadableAttentionLayer],
|
||||
)
|
||||
def test_attention_first_load_processes_weights(default_vllm_config, layer_cls):
|
||||
default_vllm_config.model_config = ModelConfig()
|
||||
layer = layer_cls()
|
||||
model = torch.nn.Sequential(layer)
|
||||
loaded_weight = torch.full_like(layer.weight, 7.0)
|
||||
|
||||
initialize_online_processing(layer)
|
||||
layer.weight.weight_loader(layer.weight, loaded_weight)
|
||||
|
||||
finalize_layerwise_reload(model, default_vllm_config.model_config)
|
||||
|
||||
assert layer.post_load_called
|
||||
assert torch.equal(layer.weight, loaded_weight)
|
||||
|
||||
|
||||
def test_reload_lifecycle():
|
||||
layer = torch.nn.Linear(2, 3)
|
||||
info = LayerReloadingInfo(
|
||||
|
||||
Reference in New Issue
Block a user