[VLM] Enable overriding whether post layernorm is used in vision encoder + fix quant args (#9217)

Co-authored-by: Isotr0py <2037008807@qq.com>
This commit is contained in:
Cyrus Leung
2024-10-23 11:27:37 +00:00
committed by GitHub
co-authored by Isotr0py
parent 3ff57ebfca
commit c18e1a3418
18 changed files with 551 additions and 253 deletions
+30 -11
View File
@@ -137,6 +137,7 @@ class InternParallelAttention(nn.Module):
quant_config: Optional[QuantizationConfig] = None,
*,
num_dummy_heads: int = 0,
prefix: str = "",
) -> None:
super().__init__()
@@ -165,6 +166,7 @@ class InternParallelAttention(nn.Module):
num_dummy_heads + self.num_heads,
bias=config.qkv_bias,
quant_config=quant_config,
prefix=f"{prefix}.qkv",
)
self.qk_normalization = config.qk_normalization
@@ -181,6 +183,7 @@ class InternParallelAttention(nn.Module):
self.dummy_dim,
self.embed_dim,
quant_config=quant_config,
prefix=f"{prefix}.proj",
)
def _apply_qk_norm(self, q: torch.Tensor, k: torch.Tensor):
@@ -284,20 +287,26 @@ class InternSdpaAttention(nn.Module):
class InternMLP(nn.Module):
def __init__(self,
config: PretrainedConfig,
quant_config: Optional[QuantizationConfig] = None):
def __init__(
self,
config: PretrainedConfig,
quant_config: Optional[QuantizationConfig] = None,
prefix: str = "",
) -> None:
super().__init__()
self.config = config
self.activation_fn = get_act_fn(config.hidden_act)
self.fc1 = ColumnParallelLinear(config.hidden_size,
config.intermediate_size,
bias=True,
quant_config=quant_config)
quant_config=quant_config,
prefix=f"{prefix}.fc1")
self.fc2 = RowParallelLinear(config.intermediate_size,
config.hidden_size,
bias=True,
quant_config=quant_config)
quant_config=quant_config,
prefix=f"{prefix}.fc2")
def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
hidden_states, _ = self.fc1(hidden_states)
@@ -315,6 +324,7 @@ class InternVisionEncoderLayer(nn.Module):
quant_config: Optional[QuantizationConfig] = None,
*,
num_dummy_heads: int = 0,
prefix: str = "",
) -> None:
super().__init__()
@@ -324,9 +334,12 @@ class InternVisionEncoderLayer(nn.Module):
self.attn = self._init_attn(config,
quant_config,
num_dummy_heads=num_dummy_heads)
num_dummy_heads=num_dummy_heads,
prefix=f"{prefix}.attn")
self.mlp = InternMLP(config, quant_config=quant_config)
self.mlp = InternMLP(config,
quant_config=quant_config,
prefix=f"{prefix}.mlp")
self.norm1 = NORM2FN[self.norm_type](self.embed_dim,
eps=config.layer_norm_eps)
self.norm2 = NORM2FN[self.norm_type](self.embed_dim,
@@ -343,6 +356,7 @@ class InternVisionEncoderLayer(nn.Module):
quant_config: Optional[QuantizationConfig],
*,
num_dummy_heads: int,
prefix: str = "",
):
# fallback to sdpa attention if tp unavailable
tp_size = get_tensor_model_parallel_world_size()
@@ -351,7 +365,8 @@ class InternVisionEncoderLayer(nn.Module):
if USE_XFORMERS_OPS and (num_heads + num_dummy_heads) % tp_size == 0:
return InternParallelAttention(config,
quant_config=quant_config,
num_dummy_heads=num_dummy_heads)
num_dummy_heads=num_dummy_heads,
prefix=prefix)
return InternSdpaAttention(config, num_dummy_heads=num_dummy_heads)
@@ -377,6 +392,7 @@ class InternVisionEncoder(nn.Module):
*,
num_hidden_layers_override: Optional[int] = None,
num_dummy_heads: int = 0,
prefix: str = "",
):
super().__init__()
@@ -390,8 +406,9 @@ class InternVisionEncoder(nn.Module):
self.layers = nn.ModuleList([
InternVisionEncoderLayer(config,
quant_config,
num_dummy_heads=num_dummy_heads)
for _ in range(num_hidden_layers)
num_dummy_heads=num_dummy_heads,
prefix=f"{prefix}.layers.{layer_idx}")
for layer_idx in range(num_hidden_layers)
])
def forward(self, inputs_embeds: torch.Tensor):
@@ -412,7 +429,8 @@ class InternVisionModel(nn.Module):
*,
num_hidden_layers_override: Optional[int] = None,
num_dummy_heads: int = 0,
):
prefix: str = "",
) -> None:
super().__init__()
self.config = config
@@ -423,6 +441,7 @@ class InternVisionModel(nn.Module):
quant_config=quant_config,
num_hidden_layers_override=num_hidden_layers_override,
num_dummy_heads=num_dummy_heads,
prefix=f"{prefix}.encoder",
)
def get_input_embeddings(self):