[Model] Standardize pooling heads (#32148)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
This commit is contained in:
@@ -8,7 +8,7 @@ from torch import nn
|
||||
from transformers import BertConfig
|
||||
|
||||
from vllm.compilation.decorators import support_torch_compile
|
||||
from vllm.config import CacheConfig, PoolerConfig, VllmConfig
|
||||
from vllm.config import CacheConfig, ModelConfig, PoolerConfig, VllmConfig
|
||||
from vllm.distributed import get_tensor_model_parallel_world_size
|
||||
from vllm.model_executor.layers.activation import get_act_fn
|
||||
from vllm.model_executor.layers.attention.encoder_only_attention import (
|
||||
@@ -24,11 +24,11 @@ from vllm.model_executor.layers.pooler import (
|
||||
Pooler,
|
||||
PoolingParamsUpdate,
|
||||
)
|
||||
from vllm.model_executor.layers.pooler.activations import LambdaPoolerActivation
|
||||
from vllm.model_executor.layers.pooler.seqwise import (
|
||||
EmbeddingPoolerHead,
|
||||
SequencePooler,
|
||||
SequencePoolerHeadOutput,
|
||||
SequencePoolerOutput,
|
||||
SequencePoolingMethodOutput,
|
||||
get_seq_pooling_method,
|
||||
)
|
||||
from vllm.model_executor.layers.pooler.tokwise import (
|
||||
@@ -94,26 +94,32 @@ class BertEmbedding(nn.Module):
|
||||
|
||||
|
||||
class BertPooler(SequencePooler):
|
||||
def __init__(self, config: BertConfig, pooler_config: PoolerConfig):
|
||||
def __init__(self, model_config: ModelConfig):
|
||||
pooler_config = model_config.pooler_config
|
||||
assert pooler_config is not None
|
||||
|
||||
config: BertConfig = model_config.hf_config
|
||||
|
||||
super().__init__(
|
||||
pooling=get_seq_pooling_method(pooler_config.seq_pooling_type),
|
||||
head=self.head,
|
||||
# We set this dummy to avoid adding parameters to nn.Module too early
|
||||
head=nn.Identity(),
|
||||
)
|
||||
|
||||
self.dense = nn.Linear(config.hidden_size, config.hidden_size)
|
||||
self.activation = nn.Tanh()
|
||||
head_dtype = model_config.head_dtype
|
||||
self.dense = nn.Linear(
|
||||
config.hidden_size,
|
||||
config.hidden_size,
|
||||
dtype=head_dtype,
|
||||
)
|
||||
self.act_fn = nn.Tanh()
|
||||
|
||||
def head(
|
||||
self,
|
||||
pooled_data: SequencePoolingMethodOutput,
|
||||
pooling_metadata: PoolingMetadata,
|
||||
) -> SequencePoolerHeadOutput:
|
||||
if isinstance(pooled_data, list):
|
||||
pooled_data = torch.stack(pooled_data)
|
||||
|
||||
pooled_data = self.dense(pooled_data)
|
||||
pooled_data = self.activation(pooled_data)
|
||||
return pooled_data
|
||||
# Use lambdas so that weights are not registered under `self.head`
|
||||
self.head = EmbeddingPoolerHead(
|
||||
projector=lambda x: self.dense(x),
|
||||
head_dtype=head_dtype,
|
||||
activation=LambdaPoolerActivation(self.act_fn),
|
||||
)
|
||||
|
||||
|
||||
class BertEncoder(nn.Module):
|
||||
@@ -449,12 +455,7 @@ class BertPoolingModel(BertModel):
|
||||
embedding_class=embedding_class,
|
||||
)
|
||||
|
||||
config = vllm_config.model_config.hf_config
|
||||
|
||||
pooler_config = vllm_config.model_config.pooler_config
|
||||
assert pooler_config is not None
|
||||
|
||||
self.pooler = BertPooler(config, pooler_config)
|
||||
self.pooler = BertPooler(vllm_config.model_config)
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
other_weights, loaded_stacked_params = self._load_weights(weights)
|
||||
|
||||
@@ -466,10 +466,7 @@ class BertWithRope(nn.Module, SupportsQuant):
|
||||
)
|
||||
|
||||
if add_pooling_layer:
|
||||
pooler_config = vllm_config.model_config.pooler_config
|
||||
assert pooler_config is not None
|
||||
|
||||
self.pooler = BertPooler(self.config, pooler_config)
|
||||
self.pooler = BertPooler(vllm_config.model_config)
|
||||
else:
|
||||
self.pooler = None
|
||||
|
||||
|
||||
@@ -5,7 +5,7 @@ from collections.abc import Set
|
||||
import numpy as np
|
||||
import torch
|
||||
|
||||
from vllm.config import ModelConfig, PoolerConfig, VllmConfig
|
||||
from vllm.config import ModelConfig, VllmConfig
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.pooler import (
|
||||
DispatchPooler,
|
||||
@@ -13,8 +13,8 @@ from vllm.model_executor.layers.pooler import (
|
||||
)
|
||||
from vllm.model_executor.layers.pooler.activations import PoolerNormalize
|
||||
from vllm.model_executor.layers.pooler.seqwise import (
|
||||
EmbeddingPoolerHead,
|
||||
SequencePooler,
|
||||
SequencePoolerHeadOutput,
|
||||
SequencePoolingMethod,
|
||||
SequencePoolingMethodOutput,
|
||||
get_seq_pooling_method,
|
||||
@@ -178,25 +178,22 @@ class GritLMMeanPool(SequencePoolingMethod):
|
||||
|
||||
|
||||
class GritLMPooler(SequencePooler):
|
||||
def __init__(self, model_config: ModelConfig, pooler_config: PoolerConfig):
|
||||
def __init__(self, model_config: ModelConfig):
|
||||
pooler_config = model_config.pooler_config
|
||||
assert pooler_config is not None
|
||||
|
||||
super().__init__(
|
||||
pooling=(
|
||||
GritLMMeanPool(model_config)
|
||||
if pooler_config.seq_pooling_type == "MEAN"
|
||||
else get_seq_pooling_method(pooler_config.seq_pooling_type)
|
||||
),
|
||||
head=self.head,
|
||||
head=EmbeddingPoolerHead(
|
||||
head_dtype=model_config.head_dtype,
|
||||
activation=PoolerNormalize(),
|
||||
),
|
||||
)
|
||||
|
||||
self.activation = PoolerNormalize()
|
||||
|
||||
def head(
|
||||
self,
|
||||
pooled_data: SequencePoolingMethodOutput,
|
||||
pooling_metadata: PoolingMetadata,
|
||||
) -> SequencePoolerHeadOutput:
|
||||
return self.activation(pooled_data)
|
||||
|
||||
|
||||
@default_pooling_type(seq_pooling_type="MEAN")
|
||||
class GritLM(LlamaForCausalLM):
|
||||
@@ -240,6 +237,6 @@ class GritLM(LlamaForCausalLM):
|
||||
self.pooler = DispatchPooler(
|
||||
{
|
||||
"token_embed": pooler_for_token_embed(pooler_config),
|
||||
"embed": GritLMPooler(vllm_config.model_config, pooler_config),
|
||||
"embed": GritLMPooler(vllm_config.model_config),
|
||||
}
|
||||
)
|
||||
|
||||
@@ -8,17 +8,17 @@ from transformers import ModernBertConfig
|
||||
from transformers.activations import ACT2FN
|
||||
|
||||
from vllm.compilation.decorators import support_torch_compile
|
||||
from vllm.config import PoolerConfig, VllmConfig
|
||||
from vllm.config import ModelConfig, VllmConfig
|
||||
from vllm.distributed import get_tensor_model_parallel_world_size
|
||||
from vllm.model_executor.layers.attention.encoder_only_attention import (
|
||||
EncoderOnlyAttention,
|
||||
)
|
||||
from vllm.model_executor.layers.linear import QKVParallelLinear, RowParallelLinear
|
||||
from vllm.model_executor.layers.pooler import DispatchPooler
|
||||
from vllm.model_executor.layers.pooler.activations import LambdaPoolerActivation
|
||||
from vllm.model_executor.layers.pooler.seqwise import (
|
||||
EmbeddingPoolerHead,
|
||||
SequencePooler,
|
||||
SequencePoolerHeadOutput,
|
||||
SequencePoolingMethodOutput,
|
||||
get_seq_pooling_method,
|
||||
)
|
||||
from vllm.model_executor.layers.pooler.tokwise import pooler_for_token_classify
|
||||
@@ -26,7 +26,6 @@ from vllm.model_executor.layers.rotary_embedding import get_rope
|
||||
from vllm.model_executor.layers.vocab_parallel_embedding import VocabParallelEmbedding
|
||||
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
|
||||
from vllm.sequence import IntermediateTensors
|
||||
from vllm.v1.pool.metadata import PoolingMetadata
|
||||
|
||||
from .interfaces import SupportsCrossEncoding
|
||||
from .interfaces_base import attn_type, default_pooling_type
|
||||
@@ -282,7 +281,11 @@ class ModernBertModel(nn.Module):
|
||||
|
||||
|
||||
class ModernBertPooler(SequencePooler):
|
||||
def __init__(self, config: ModernBertConfig, pooler_config: PoolerConfig):
|
||||
def __init__(self, model_config: ModelConfig):
|
||||
pooler_config = model_config.pooler_config
|
||||
assert pooler_config is not None
|
||||
|
||||
config: ModernBertConfig = model_config.hf_config
|
||||
hf_pooling_type = config.classifier_pooling.upper()
|
||||
# vllm_pooling_type = pooler_config.seq_pooling_type
|
||||
# Currently we don't have a way to see if the user set the pooling type
|
||||
@@ -290,27 +293,30 @@ class ModernBertPooler(SequencePooler):
|
||||
|
||||
super().__init__(
|
||||
pooling=get_seq_pooling_method(hf_pooling_type),
|
||||
head=self.head,
|
||||
# We set this dummy to avoid adding parameters to nn.Module too early
|
||||
head=nn.Identity(),
|
||||
)
|
||||
|
||||
head_dtype = model_config.head_dtype
|
||||
self.dense = nn.Linear(
|
||||
config.hidden_size, config.hidden_size, config.classifier_bias
|
||||
config.hidden_size,
|
||||
config.hidden_size,
|
||||
config.classifier_bias,
|
||||
dtype=head_dtype,
|
||||
)
|
||||
self.act = nn.GELU()
|
||||
self.norm = nn.LayerNorm(
|
||||
config.hidden_size, eps=config.norm_eps, bias=config.norm_bias
|
||||
config.hidden_size,
|
||||
eps=config.norm_eps,
|
||||
bias=config.norm_bias,
|
||||
)
|
||||
|
||||
def head(
|
||||
self,
|
||||
pooled_data: SequencePoolingMethodOutput,
|
||||
pooling_metadata: PoolingMetadata,
|
||||
) -> SequencePoolerHeadOutput:
|
||||
if isinstance(pooled_data, list):
|
||||
pooled_data = torch.stack(pooled_data)
|
||||
|
||||
pooled_data = pooled_data.to(self.dense.weight.dtype)
|
||||
return self.norm(self.act(self.dense(pooled_data)))
|
||||
# Use lambdas so that weights are not registered under `self.head`
|
||||
self.head = EmbeddingPoolerHead(
|
||||
projector=lambda x: self.dense(x),
|
||||
head_dtype=head_dtype,
|
||||
activation=LambdaPoolerActivation(lambda x: self.norm(self.act(x))),
|
||||
)
|
||||
|
||||
|
||||
@default_pooling_type(seq_pooling_type="CLS")
|
||||
@@ -335,7 +341,7 @@ class ModernBertForSequenceClassification(nn.Module, SupportsCrossEncoding):
|
||||
pooler_config = vllm_config.model_config.pooler_config
|
||||
assert pooler_config is not None
|
||||
|
||||
self.pooling = ModernBertPooler(config, pooler_config)
|
||||
self.pooling = ModernBertPooler(vllm_config.model_config)
|
||||
|
||||
self.pooler = DispatchPooler.for_seq_cls(
|
||||
pooler_config,
|
||||
|
||||
Reference in New Issue
Block a user