[Bugfix][Model Loader] Defer post-load attention weight processing (#49519)
Signed-off-by: aoshen02 <aoshen@inferact.ai> Co-authored-by: Roger Wang <hey@rogerw.io>
This commit is contained in:
@@ -9,7 +9,7 @@ import torch
|
||||
|
||||
from vllm.config import ModelConfig
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.attention import Attention, MLAAttention
|
||||
from vllm.model_executor.layers.attention import is_deferred_attention_layer
|
||||
from vllm.model_executor.layers.quantization.base_config import QuantizeMethodBase
|
||||
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
|
||||
|
||||
@@ -196,7 +196,7 @@ def make_online_process_loader(layer: torch.nn.Module, param_name: str) -> Calla
|
||||
)
|
||||
|
||||
# Do not online process attention layers, must wait until finalize
|
||||
if isinstance(layer, (Attention, MLAAttention)):
|
||||
if is_deferred_attention_layer(layer):
|
||||
return ret
|
||||
|
||||
# Log warnings allocating excessive buffers on device
|
||||
@@ -249,8 +249,8 @@ def finalize_layerwise_processing(model: torch.nn.Module, model_config: ModelCon
|
||||
info.reset()
|
||||
continue
|
||||
|
||||
# Attention/MLA layers are processed after all other layers
|
||||
if isinstance(layer, (Attention, MLAAttention)):
|
||||
# Deferred attention-like layers are processed after all other layers
|
||||
if is_deferred_attention_layer(layer):
|
||||
deferred_attn.append((layer, info))
|
||||
continue
|
||||
|
||||
@@ -293,15 +293,13 @@ def finalize_layerwise_reload(*args, **kwargs):
|
||||
def _finalize_attention_layer(
|
||||
layer: torch.nn.Module, info: LayerReloadingInfo, model_config: ModelConfig
|
||||
) -> None:
|
||||
if info.load_numel > 0 and info.kernel_tensors is not None:
|
||||
if info.kernel_tensors is None:
|
||||
if info.load_numel > 0:
|
||||
_layerwise_process(layer, info)
|
||||
elif info.load_numel > 0:
|
||||
# Reload with new scale weights from checkpoint
|
||||
_place_kernel_tensors(layer, info)
|
||||
_reload_attention_scales(layer, info)
|
||||
elif info.load_numel > 0 or info.kernel_tensors is None:
|
||||
raise ValueError(
|
||||
"Layerwise loading of attention layers is not supported. "
|
||||
"Attention must always process after linears."
|
||||
)
|
||||
else:
|
||||
_place_kernel_tensors(layer, info)
|
||||
layer.process_weights_after_loading(model_config.dtype)
|
||||
@@ -315,19 +313,18 @@ def _reload_attention_scales(layer: torch.nn.Module, info: LayerReloadingInfo) -
|
||||
processing, since we use .data.copy_() to preserve kernel tensor
|
||||
references."""
|
||||
quant_method = getattr(layer, "quant_method", None)
|
||||
if quant_method is None:
|
||||
return
|
||||
|
||||
# Re-create scale Parameters with sentinel values so unloaded scales
|
||||
# are correctly detected by process_weights_after_loading
|
||||
quant_method.create_weights(layer)
|
||||
if quant_method is not None:
|
||||
# Re-create scale Parameters with sentinel values so unloaded scales
|
||||
# are correctly detected by process_weights_after_loading
|
||||
quant_method.create_weights(layer)
|
||||
|
||||
for name, args in info.loaded_weights:
|
||||
param = getattr(layer, name)
|
||||
args.arguments["param"] = param
|
||||
_get_weight_loader(param)(*args.args, **args.kwargs)
|
||||
|
||||
quant_method.process_weights_after_loading(layer)
|
||||
if quant_method is not None:
|
||||
quant_method.process_weights_after_loading(layer)
|
||||
|
||||
_copy_and_restore_kernel_tensors(layer, info)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user