[Bugfix][Model Loader] Defer post-load attention weight processing (#49519)
Signed-off-by: aoshen02 <aoshen@inferact.ai> Co-authored-by: Roger Wang <hey@rogerw.io>
This commit is contained in:
@@ -9,6 +9,9 @@ import torch
|
|||||||
from torch.nn.parameter import UninitializedParameter
|
from torch.nn.parameter import UninitializedParameter
|
||||||
|
|
||||||
import vllm.model_executor.model_loader.reload.meta as reload_meta
|
import vllm.model_executor.model_loader.reload.meta as reload_meta
|
||||||
|
from vllm.config import ModelConfig
|
||||||
|
from vllm.model_executor.layers.attention import MMEncoderAttention
|
||||||
|
from vllm.model_executor.layers.attention_layer_base import AttentionLayerBase
|
||||||
from vllm.model_executor.layers.linear import QKVParallelLinear
|
from vllm.model_executor.layers.linear import QKVParallelLinear
|
||||||
from vllm.model_executor.layers.quantization.base_config import QuantizeMethodBase
|
from vllm.model_executor.layers.quantization.base_config import QuantizeMethodBase
|
||||||
from vllm.model_executor.model_loader.reload.layerwise import (
|
from vllm.model_executor.model_loader.reload.layerwise import (
|
||||||
@@ -101,6 +104,39 @@ class _NonPersistentBufferLayer(torch.nn.Module):
|
|||||||
self.register_buffer("scale", torch.tensor(0.25), persistent=False)
|
self.register_buffer("scale", torch.tensor(0.25), persistent=False)
|
||||||
|
|
||||||
|
|
||||||
|
class _ReloadableMMEncoderAttention(MMEncoderAttention):
|
||||||
|
"""Minimal stand-in to test reload lifecycle without encoder initialization."""
|
||||||
|
|
||||||
|
def __init__(self):
|
||||||
|
torch.nn.Module.__init__(self)
|
||||||
|
self.weight = torch.nn.Parameter(torch.ones(2, 2))
|
||||||
|
self.weight.weight_loader = default_weight_loader
|
||||||
|
self.post_load_called = False
|
||||||
|
|
||||||
|
def process_weights_after_loading(self, act_dtype: torch.dtype) -> None:
|
||||||
|
self.post_load_called = True
|
||||||
|
|
||||||
|
|
||||||
|
class _ReloadableAttentionLayer(
|
||||||
|
torch.nn.Module,
|
||||||
|
AttentionLayerBase,
|
||||||
|
):
|
||||||
|
def __init__(self):
|
||||||
|
super().__init__()
|
||||||
|
self.weight = torch.nn.Parameter(torch.ones(2, 2))
|
||||||
|
self.weight.weight_loader = default_weight_loader
|
||||||
|
self.post_load_called = False
|
||||||
|
|
||||||
|
def get_attn_backend(self):
|
||||||
|
raise NotImplementedError
|
||||||
|
|
||||||
|
def get_kv_cache_spec(self, vllm_config):
|
||||||
|
return None
|
||||||
|
|
||||||
|
def process_weights_after_loading(self, act_dtype: torch.dtype) -> None:
|
||||||
|
self.post_load_called = True
|
||||||
|
|
||||||
|
|
||||||
def test_move_metatensors():
|
def test_move_metatensors():
|
||||||
tensor = torch.empty((1, 2, 3))
|
tensor = torch.empty((1, 2, 3))
|
||||||
meta_tensor = to_meta_tensor(tensor)
|
meta_tensor = to_meta_tensor(tensor)
|
||||||
@@ -115,6 +151,47 @@ def test_move_metatensors():
|
|||||||
assert tensor.__dict__ == meta_tensor.__dict__ == materialized_tensor.__dict__
|
assert tensor.__dict__ == meta_tensor.__dict__ == materialized_tensor.__dict__
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
"layer_cls",
|
||||||
|
[_ReloadableMMEncoderAttention, _ReloadableAttentionLayer],
|
||||||
|
)
|
||||||
|
def test_attention_reload_defers_post_load(default_vllm_config, layer_cls):
|
||||||
|
default_vllm_config.model_config = ModelConfig()
|
||||||
|
layer = layer_cls()
|
||||||
|
model = torch.nn.Sequential(layer)
|
||||||
|
loaded_weight = torch.full_like(layer.weight, 7.0)
|
||||||
|
|
||||||
|
record_metadata_for_reloading(model)
|
||||||
|
initialize_layerwise_reload(model)
|
||||||
|
layer.weight.weight_loader(layer.weight, loaded_weight)
|
||||||
|
|
||||||
|
assert not layer.post_load_called
|
||||||
|
|
||||||
|
finalize_layerwise_reload(model, default_vllm_config.model_config)
|
||||||
|
|
||||||
|
assert layer.post_load_called
|
||||||
|
assert torch.equal(layer.weight, loaded_weight)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
"layer_cls",
|
||||||
|
[_ReloadableMMEncoderAttention, _ReloadableAttentionLayer],
|
||||||
|
)
|
||||||
|
def test_attention_first_load_processes_weights(default_vllm_config, layer_cls):
|
||||||
|
default_vllm_config.model_config = ModelConfig()
|
||||||
|
layer = layer_cls()
|
||||||
|
model = torch.nn.Sequential(layer)
|
||||||
|
loaded_weight = torch.full_like(layer.weight, 7.0)
|
||||||
|
|
||||||
|
initialize_online_processing(layer)
|
||||||
|
layer.weight.weight_loader(layer.weight, loaded_weight)
|
||||||
|
|
||||||
|
finalize_layerwise_reload(model, default_vllm_config.model_config)
|
||||||
|
|
||||||
|
assert layer.post_load_called
|
||||||
|
assert torch.equal(layer.weight, loaded_weight)
|
||||||
|
|
||||||
|
|
||||||
def test_reload_lifecycle():
|
def test_reload_lifecycle():
|
||||||
layer = torch.nn.Linear(2, 3)
|
layer = torch.nn.Linear(2, 3)
|
||||||
info = LayerReloadingInfo(
|
info = LayerReloadingInfo(
|
||||||
|
|||||||
@@ -1,6 +1,8 @@
|
|||||||
# SPDX-License-Identifier: Apache-2.0
|
# SPDX-License-Identifier: Apache-2.0
|
||||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||||
|
|
||||||
|
import torch
|
||||||
|
|
||||||
from vllm.model_executor.layers.attention.attention import Attention
|
from vllm.model_executor.layers.attention.attention import Attention
|
||||||
from vllm.model_executor.layers.attention.chunked_local_attention import (
|
from vllm.model_executor.layers.attention.chunked_local_attention import (
|
||||||
ChunkedLocalAttention,
|
ChunkedLocalAttention,
|
||||||
@@ -18,6 +20,15 @@ from vllm.model_executor.layers.attention.rswa_attention import RSWAAttention
|
|||||||
from vllm.model_executor.layers.attention.static_sink_attention import (
|
from vllm.model_executor.layers.attention.static_sink_attention import (
|
||||||
StaticSinkAttention,
|
StaticSinkAttention,
|
||||||
)
|
)
|
||||||
|
from vllm.model_executor.layers.attention_layer_base import AttentionLayerBase
|
||||||
|
|
||||||
|
|
||||||
|
def is_deferred_attention_layer(layer: torch.nn.Module) -> bool:
|
||||||
|
"""Whether an attention-like layer requires deferred post-load processing."""
|
||||||
|
return isinstance(layer, (AttentionLayerBase, MMEncoderAttention)) and callable(
|
||||||
|
getattr(layer, "process_weights_after_loading", None)
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
__all__ = [
|
__all__ = [
|
||||||
"Attention",
|
"Attention",
|
||||||
@@ -29,4 +40,5 @@ __all__ = [
|
|||||||
"PrefillPrefixLMAttention",
|
"PrefillPrefixLMAttention",
|
||||||
"RSWAAttention",
|
"RSWAAttention",
|
||||||
"StaticSinkAttention",
|
"StaticSinkAttention",
|
||||||
|
"is_deferred_attention_layer",
|
||||||
]
|
]
|
||||||
|
|||||||
@@ -9,7 +9,7 @@ import torch
|
|||||||
|
|
||||||
from vllm.config import ModelConfig
|
from vllm.config import ModelConfig
|
||||||
from vllm.logger import init_logger
|
from vllm.logger import init_logger
|
||||||
from vllm.model_executor.layers.attention import Attention, MLAAttention
|
from vllm.model_executor.layers.attention import is_deferred_attention_layer
|
||||||
from vllm.model_executor.layers.quantization.base_config import QuantizeMethodBase
|
from vllm.model_executor.layers.quantization.base_config import QuantizeMethodBase
|
||||||
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
|
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
|
||||||
|
|
||||||
@@ -196,7 +196,7 @@ def make_online_process_loader(layer: torch.nn.Module, param_name: str) -> Calla
|
|||||||
)
|
)
|
||||||
|
|
||||||
# Do not online process attention layers, must wait until finalize
|
# Do not online process attention layers, must wait until finalize
|
||||||
if isinstance(layer, (Attention, MLAAttention)):
|
if is_deferred_attention_layer(layer):
|
||||||
return ret
|
return ret
|
||||||
|
|
||||||
# Log warnings allocating excessive buffers on device
|
# Log warnings allocating excessive buffers on device
|
||||||
@@ -249,8 +249,8 @@ def finalize_layerwise_processing(model: torch.nn.Module, model_config: ModelCon
|
|||||||
info.reset()
|
info.reset()
|
||||||
continue
|
continue
|
||||||
|
|
||||||
# Attention/MLA layers are processed after all other layers
|
# Deferred attention-like layers are processed after all other layers
|
||||||
if isinstance(layer, (Attention, MLAAttention)):
|
if is_deferred_attention_layer(layer):
|
||||||
deferred_attn.append((layer, info))
|
deferred_attn.append((layer, info))
|
||||||
continue
|
continue
|
||||||
|
|
||||||
@@ -293,15 +293,13 @@ def finalize_layerwise_reload(*args, **kwargs):
|
|||||||
def _finalize_attention_layer(
|
def _finalize_attention_layer(
|
||||||
layer: torch.nn.Module, info: LayerReloadingInfo, model_config: ModelConfig
|
layer: torch.nn.Module, info: LayerReloadingInfo, model_config: ModelConfig
|
||||||
) -> None:
|
) -> None:
|
||||||
if info.load_numel > 0 and info.kernel_tensors is not None:
|
if info.kernel_tensors is None:
|
||||||
|
if info.load_numel > 0:
|
||||||
|
_layerwise_process(layer, info)
|
||||||
|
elif info.load_numel > 0:
|
||||||
# Reload with new scale weights from checkpoint
|
# Reload with new scale weights from checkpoint
|
||||||
_place_kernel_tensors(layer, info)
|
_place_kernel_tensors(layer, info)
|
||||||
_reload_attention_scales(layer, info)
|
_reload_attention_scales(layer, info)
|
||||||
elif info.load_numel > 0 or info.kernel_tensors is None:
|
|
||||||
raise ValueError(
|
|
||||||
"Layerwise loading of attention layers is not supported. "
|
|
||||||
"Attention must always process after linears."
|
|
||||||
)
|
|
||||||
else:
|
else:
|
||||||
_place_kernel_tensors(layer, info)
|
_place_kernel_tensors(layer, info)
|
||||||
layer.process_weights_after_loading(model_config.dtype)
|
layer.process_weights_after_loading(model_config.dtype)
|
||||||
@@ -315,19 +313,18 @@ def _reload_attention_scales(layer: torch.nn.Module, info: LayerReloadingInfo) -
|
|||||||
processing, since we use .data.copy_() to preserve kernel tensor
|
processing, since we use .data.copy_() to preserve kernel tensor
|
||||||
references."""
|
references."""
|
||||||
quant_method = getattr(layer, "quant_method", None)
|
quant_method = getattr(layer, "quant_method", None)
|
||||||
if quant_method is None:
|
if quant_method is not None:
|
||||||
return
|
# Re-create scale Parameters with sentinel values so unloaded scales
|
||||||
|
# are correctly detected by process_weights_after_loading
|
||||||
# Re-create scale Parameters with sentinel values so unloaded scales
|
quant_method.create_weights(layer)
|
||||||
# are correctly detected by process_weights_after_loading
|
|
||||||
quant_method.create_weights(layer)
|
|
||||||
|
|
||||||
for name, args in info.loaded_weights:
|
for name, args in info.loaded_weights:
|
||||||
param = getattr(layer, name)
|
param = getattr(layer, name)
|
||||||
args.arguments["param"] = param
|
args.arguments["param"] = param
|
||||||
_get_weight_loader(param)(*args.args, **args.kwargs)
|
_get_weight_loader(param)(*args.args, **args.kwargs)
|
||||||
|
|
||||||
quant_method.process_weights_after_loading(layer)
|
if quant_method is not None:
|
||||||
|
quant_method.process_weights_after_loading(layer)
|
||||||
|
|
||||||
_copy_and_restore_kernel_tensors(layer, info)
|
_copy_and_restore_kernel_tensors(layer, info)
|
||||||
|
|
||||||
|
|||||||
@@ -14,10 +14,7 @@ from typing_extensions import assert_never
|
|||||||
import vllm.envs as envs
|
import vllm.envs as envs
|
||||||
from vllm.config import ModelConfig, VllmConfig, set_current_vllm_config
|
from vllm.config import ModelConfig, VllmConfig, set_current_vllm_config
|
||||||
from vllm.logger import init_logger
|
from vllm.logger import init_logger
|
||||||
from vllm.model_executor.layers.attention import (
|
from vllm.model_executor.layers.attention import is_deferred_attention_layer
|
||||||
MMEncoderAttention,
|
|
||||||
)
|
|
||||||
from vllm.model_executor.layers.attention_layer_base import AttentionLayerBase
|
|
||||||
from vllm.model_executor.layers.hpc import HpcModule
|
from vllm.model_executor.layers.hpc import HpcModule
|
||||||
from vllm.model_executor.layers.quantization.base_config import (
|
from vllm.model_executor.layers.quantization.base_config import (
|
||||||
QuantizationConfig,
|
QuantizationConfig,
|
||||||
@@ -124,9 +121,7 @@ def process_weights_after_loading(
|
|||||||
# encoder. NOTE: Happens after other modules so we can easily decompress
|
# encoder. NOTE: Happens after other modules so we can easily decompress
|
||||||
# weights.
|
# weights.
|
||||||
for _, module in model.named_modules():
|
for _, module in model.named_modules():
|
||||||
if isinstance(module, (AttentionLayerBase, MMEncoderAttention)) and hasattr(
|
if is_deferred_attention_layer(module):
|
||||||
module, "process_weights_after_loading"
|
|
||||||
):
|
|
||||||
# TODO(lucas): see if there is a way to unify the signatures
|
# TODO(lucas): see if there is a way to unify the signatures
|
||||||
# of process_weights_after_loading
|
# of process_weights_after_loading
|
||||||
with device_loading_context(module, target_device):
|
with device_loading_context(module, target_device):
|
||||||
|
|||||||
Reference in New Issue
Block a user