[Bugfix][Model Loader] Defer post-load attention weight processing (#49519)
Signed-off-by: aoshen02 <aoshen@inferact.ai> Co-authored-by: Roger Wang <hey@rogerw.io>
This commit is contained in:
@@ -1,6 +1,8 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.model_executor.layers.attention.attention import Attention
|
||||
from vllm.model_executor.layers.attention.chunked_local_attention import (
|
||||
ChunkedLocalAttention,
|
||||
@@ -18,6 +20,15 @@ from vllm.model_executor.layers.attention.rswa_attention import RSWAAttention
|
||||
from vllm.model_executor.layers.attention.static_sink_attention import (
|
||||
StaticSinkAttention,
|
||||
)
|
||||
from vllm.model_executor.layers.attention_layer_base import AttentionLayerBase
|
||||
|
||||
|
||||
def is_deferred_attention_layer(layer: torch.nn.Module) -> bool:
|
||||
"""Whether an attention-like layer requires deferred post-load processing."""
|
||||
return isinstance(layer, (AttentionLayerBase, MMEncoderAttention)) and callable(
|
||||
getattr(layer, "process_weights_after_loading", None)
|
||||
)
|
||||
|
||||
|
||||
__all__ = [
|
||||
"Attention",
|
||||
@@ -29,4 +40,5 @@ __all__ = [
|
||||
"PrefillPrefixLMAttention",
|
||||
"RSWAAttention",
|
||||
"StaticSinkAttention",
|
||||
"is_deferred_attention_layer",
|
||||
]
|
||||
|
||||
Reference in New Issue
Block a user