[MM Encoder]: Migrate legacy ViT MultiHeadAttention to new MMEncoderAttention interface (#30684)
Signed-off-by: Isotr0py <mozf@mail2.sysu.edu.cn>
This commit is contained in:
@@ -14,7 +14,8 @@ from transformers import (
|
||||
CLIPVisionConfig,
|
||||
)
|
||||
|
||||
from vllm.attention.layer import Attention, MultiHeadAttention
|
||||
from vllm.attention.layer import Attention
|
||||
from vllm.attention.layers.mm_encoder_attention import MMEncoderAttention
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.config.multimodal import BaseDummyOptions
|
||||
from vllm.distributed import divide, get_tensor_model_parallel_world_size
|
||||
@@ -354,7 +355,7 @@ class CLIPAttention(nn.Module):
|
||||
quant_config: QuantizationConfig | None = None,
|
||||
*,
|
||||
prefix: str = "",
|
||||
attn_cls: type[Attention] | type[MultiHeadAttention],
|
||||
attn_cls: type[Attention] | type[MMEncoderAttention],
|
||||
) -> None:
|
||||
super().__init__()
|
||||
|
||||
@@ -449,7 +450,7 @@ class CLIPEncoderLayer(nn.Module):
|
||||
quant_config: QuantizationConfig | None = None,
|
||||
*,
|
||||
prefix: str = "",
|
||||
attn_cls: type[Attention] | type[MultiHeadAttention],
|
||||
attn_cls: type[Attention] | type[MMEncoderAttention],
|
||||
) -> None:
|
||||
super().__init__()
|
||||
self.self_attn = CLIPAttention(
|
||||
@@ -493,7 +494,7 @@ class CLIPEncoder(nn.Module):
|
||||
num_hidden_layers_override: int | None = None,
|
||||
*,
|
||||
prefix: str = "",
|
||||
attn_cls: type[Attention] | type[MultiHeadAttention],
|
||||
attn_cls: type[Attention] | type[MMEncoderAttention],
|
||||
) -> None:
|
||||
super().__init__()
|
||||
|
||||
@@ -638,7 +639,7 @@ class CLIPVisionTransformer(nn.Module):
|
||||
quant_config=quant_config,
|
||||
num_hidden_layers_override=num_hidden_layers_override,
|
||||
prefix=f"{prefix}.encoder",
|
||||
attn_cls=MultiHeadAttention,
|
||||
attn_cls=MMEncoderAttention,
|
||||
)
|
||||
|
||||
num_hidden_layers = config.num_hidden_layers
|
||||
|
||||
Reference in New Issue
Block a user