[Refactor] Separate sequence and token pooling types (#32026)

Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
This commit is contained in:
Cyrus Leung
2026-01-10 04:53:24 +00:00
committed by GitHub
parent 52d428295d
commit 583a90e005
42 changed files with 324 additions and 204 deletions
@@ -46,7 +46,8 @@ def test_model_loading_with_params(vllm_runner, monkeypatch):
assert model_config.encoder_config["do_lower_case"] assert model_config.encoder_config["do_lower_case"]
# asserts on the pooling config files # asserts on the pooling config files
assert model_config.pooler_config.pooling_type == "CLS" assert model_config.pooler_config.seq_pooling_type == "CLS"
assert model_config.pooler_config.tok_pooling_type == "ALL"
assert model_config.pooler_config.normalize assert model_config.pooler_config.normalize
# asserts on the tokenizer loaded # asserts on the tokenizer loaded
@@ -90,7 +91,8 @@ def test_roberta_model_loading_with_params(vllm_runner, monkeypatch):
assert not model_config.encoder_config["do_lower_case"] assert not model_config.encoder_config["do_lower_case"]
# asserts on the pooling config files # asserts on the pooling config files
assert model_config.pooler_config.pooling_type == "MEAN" assert model_config.pooler_config.seq_pooling_type == "MEAN"
assert model_config.pooler_config.tok_pooling_type == "ALL"
assert model_config.pooler_config.normalize assert model_config.pooler_config.normalize
# asserts on the tokenizer loaded # asserts on the tokenizer loaded
@@ -54,7 +54,7 @@ def test_models(
vllm_extra_kwargs = {} vllm_extra_kwargs = {}
if model == "ssmits/Qwen2-7B-Instruct-embed-base": if model == "ssmits/Qwen2-7B-Instruct-embed-base":
vllm_extra_kwargs["pooler_config"] = PoolerConfig( vllm_extra_kwargs["pooler_config"] = PoolerConfig(
pooling_type="MEAN", normalize=False seq_pooling_type="MEAN", normalize=False
) )
max_model_len: int | None = 512 max_model_len: int | None = 512
@@ -88,7 +88,7 @@ def test_gemma_multimodal(
convert="classify", convert="classify",
load_format="auto", load_format="auto",
hf_overrides=update_config, hf_overrides=update_config,
pooler_config=PoolerConfig(pooling_type="LAST"), pooler_config=PoolerConfig(seq_pooling_type="LAST"),
max_model_len=512, max_model_len=512,
enforce_eager=True, enforce_eager=True,
tensor_parallel_size=1, tensor_parallel_size=1,
@@ -162,8 +162,11 @@ def mteb_test_embed_models(
assert model_info.architecture in model_config.architectures assert model_info.architecture in model_config.architectures
# Confirm whether the important configs in model_config are correct. # Confirm whether the important configs in model_config are correct.
if model_info.pooling_type is not None: pooler_config = model_config.pooler_config
assert model_config.pooler_config.pooling_type == model_info.pooling_type if model_info.seq_pooling_type is not None:
assert pooler_config.seq_pooling_type == model_info.seq_pooling_type
if model_info.tok_pooling_type is not None:
assert pooler_config.tok_pooling_type == model_info.tok_pooling_type
if model_info.attn_type is not None: if model_info.attn_type is not None:
assert model_config.attn_type == model_info.attn_type assert model_config.attn_type == model_info.attn_type
if model_info.is_prefix_caching_supported is not None: if model_info.is_prefix_caching_supported is not None:
@@ -254,8 +254,11 @@ def mteb_test_rerank_models(
assert model_config.hf_config.num_labels == 1 assert model_config.hf_config.num_labels == 1
# Confirm whether the important configs in model_config are correct. # Confirm whether the important configs in model_config are correct.
if model_info.pooling_type is not None: pooler_config = model_config.pooler_config
assert model_config.pooler_config.pooling_type == model_info.pooling_type if model_info.seq_pooling_type is not None:
assert pooler_config.seq_pooling_type == model_info.seq_pooling_type
if model_info.tok_pooling_type is not None:
assert pooler_config.tok_pooling_type == model_info.tok_pooling_type
if model_info.attn_type is not None: if model_info.attn_type is not None:
assert model_config.attn_type == model_info.attn_type assert model_config.attn_type == model_info.attn_type
if model_info.is_prefix_caching_supported is not None: if model_info.is_prefix_caching_supported is not None:
@@ -17,7 +17,7 @@ MODELS = [
"BAAI/bge-base-en", "BAAI/bge-base-en",
architecture="BertModel", architecture="BertModel",
mteb_score=0.779336792, mteb_score=0.779336792,
pooling_type="CLS", seq_pooling_type="CLS",
attn_type="encoder_only", attn_type="encoder_only",
is_prefix_caching_supported=False, is_prefix_caching_supported=False,
is_chunked_prefill_supported=False, is_chunked_prefill_supported=False,
@@ -54,7 +54,7 @@ MODELS = [
"BAAI/bge-m3", "BAAI/bge-m3",
architecture="XLMRobertaModel", architecture="XLMRobertaModel",
mteb_score=0.787343078, mteb_score=0.787343078,
pooling_type="CLS", seq_pooling_type="CLS",
attn_type="encoder_only", attn_type="encoder_only",
is_prefix_caching_supported=False, is_prefix_caching_supported=False,
is_chunked_prefill_supported=False, is_chunked_prefill_supported=False,
@@ -65,7 +65,7 @@ MODELS = [
"BAAI/bge-code-v1", "BAAI/bge-code-v1",
architecture="Qwen2Model", architecture="Qwen2Model",
mteb_score=0.75724465, mteb_score=0.75724465,
pooling_type="LAST", seq_pooling_type="LAST",
attn_type="decoder", attn_type="decoder",
is_prefix_caching_supported=True, is_prefix_caching_supported=True,
is_chunked_prefill_supported=True, is_chunked_prefill_supported=True,
@@ -79,7 +79,7 @@ RERANK_MODELS = [
"BAAI/bge-reranker-base", "BAAI/bge-reranker-base",
architecture="XLMRobertaForSequenceClassification", architecture="XLMRobertaForSequenceClassification",
mteb_score=0.32398, mteb_score=0.32398,
pooling_type="CLS", seq_pooling_type="CLS",
attn_type="encoder_only", attn_type="encoder_only",
is_prefix_caching_supported=False, is_prefix_caching_supported=False,
is_chunked_prefill_supported=False, is_chunked_prefill_supported=False,
@@ -26,7 +26,7 @@ RERANK_MODELS = [
"method": "no_post_processing", "method": "no_post_processing",
}, },
mteb_score=0.33757, mteb_score=0.33757,
pooling_type="LAST", seq_pooling_type="LAST",
attn_type="decoder", attn_type="decoder",
is_prefix_caching_supported=True, is_prefix_caching_supported=True,
is_chunked_prefill_supported=True, is_chunked_prefill_supported=True,
@@ -12,7 +12,7 @@ RERANK_MODELS = [
RerankModelInfo( RerankModelInfo(
"cross-encoder/ms-marco-TinyBERT-L-2-v2", "cross-encoder/ms-marco-TinyBERT-L-2-v2",
architecture="BertForSequenceClassification", architecture="BertForSequenceClassification",
pooling_type="CLS", seq_pooling_type="CLS",
attn_type="encoder_only", attn_type="encoder_only",
is_prefix_caching_supported=False, is_prefix_caching_supported=False,
is_chunked_prefill_supported=False, is_chunked_prefill_supported=False,
@@ -21,7 +21,7 @@ RERANK_MODELS = [
RerankModelInfo( RerankModelInfo(
"tomaarsen/Qwen3-Reranker-0.6B-seq-cls", "tomaarsen/Qwen3-Reranker-0.6B-seq-cls",
architecture="Qwen3ForSequenceClassification", architecture="Qwen3ForSequenceClassification",
pooling_type="LAST", seq_pooling_type="LAST",
attn_type="decoder", attn_type="decoder",
is_prefix_caching_supported=True, is_prefix_caching_supported=True,
is_chunked_prefill_supported=True, is_chunked_prefill_supported=True,
@@ -18,7 +18,7 @@ MODELS = [
"thenlper/gte-large", "thenlper/gte-large",
mteb_score=0.76807651, mteb_score=0.76807651,
architecture="BertModel", architecture="BertModel",
pooling_type="MEAN", seq_pooling_type="MEAN",
attn_type="encoder_only", attn_type="encoder_only",
is_prefix_caching_supported=False, is_prefix_caching_supported=False,
is_chunked_prefill_supported=False, is_chunked_prefill_supported=False,
@@ -44,7 +44,7 @@ MODELS = [
architecture="GteNewModel", architecture="GteNewModel",
mteb_score=0.775074696, mteb_score=0.775074696,
hf_overrides={"architectures": ["GteNewModel"]}, hf_overrides={"architectures": ["GteNewModel"]},
pooling_type="CLS", seq_pooling_type="CLS",
attn_type="encoder_only", attn_type="encoder_only",
is_prefix_caching_supported=False, is_prefix_caching_supported=False,
is_chunked_prefill_supported=False, is_chunked_prefill_supported=False,
@@ -67,7 +67,7 @@ MODELS = [
"Alibaba-NLP/gte-Qwen2-1.5B-instruct", "Alibaba-NLP/gte-Qwen2-1.5B-instruct",
mteb_score=0.758473459018872, mteb_score=0.758473459018872,
architecture="Qwen2ForCausalLM", architecture="Qwen2ForCausalLM",
pooling_type="LAST", seq_pooling_type="LAST",
attn_type="encoder_only", attn_type="encoder_only",
is_prefix_caching_supported=False, is_prefix_caching_supported=False,
is_chunked_prefill_supported=False, is_chunked_prefill_supported=False,
@@ -78,7 +78,7 @@ MODELS = [
"Alibaba-NLP/gte-modernbert-base", "Alibaba-NLP/gte-modernbert-base",
mteb_score=0.748193353, mteb_score=0.748193353,
architecture="ModernBertModel", architecture="ModernBertModel",
pooling_type="CLS", seq_pooling_type="CLS",
attn_type="encoder_only", attn_type="encoder_only",
is_prefix_caching_supported=False, is_prefix_caching_supported=False,
is_chunked_prefill_supported=False, is_chunked_prefill_supported=False,
@@ -89,7 +89,7 @@ MODELS = [
"Qwen/Qwen3-Embedding-0.6B", "Qwen/Qwen3-Embedding-0.6B",
mteb_score=0.771163695, mteb_score=0.771163695,
architecture="Qwen3ForCausalLM", architecture="Qwen3ForCausalLM",
pooling_type="LAST", seq_pooling_type="LAST",
attn_type="decoder", attn_type="decoder",
is_prefix_caching_supported=True, is_prefix_caching_supported=True,
is_chunked_prefill_supported=True, is_chunked_prefill_supported=True,
@@ -108,7 +108,7 @@ RERANK_MODELS = [
"Alibaba-NLP/gte-reranker-modernbert-base", "Alibaba-NLP/gte-reranker-modernbert-base",
mteb_score=0.33386, mteb_score=0.33386,
architecture="ModernBertForSequenceClassification", architecture="ModernBertForSequenceClassification",
pooling_type="CLS", seq_pooling_type="CLS",
attn_type="encoder_only", attn_type="encoder_only",
is_prefix_caching_supported=False, is_prefix_caching_supported=False,
is_chunked_prefill_supported=False, is_chunked_prefill_supported=False,
@@ -119,7 +119,7 @@ RERANK_MODELS = [
mteb_score=0.33062, mteb_score=0.33062,
architecture="GteNewForSequenceClassification", architecture="GteNewForSequenceClassification",
hf_overrides={"architectures": ["GteNewForSequenceClassification"]}, hf_overrides={"architectures": ["GteNewForSequenceClassification"]},
pooling_type="CLS", seq_pooling_type="CLS",
attn_type="encoder_only", attn_type="encoder_only",
is_prefix_caching_supported=False, is_prefix_caching_supported=False,
is_chunked_prefill_supported=False, is_chunked_prefill_supported=False,
@@ -13,7 +13,7 @@ MODELS = [
"intfloat/e5-small", "intfloat/e5-small",
architecture="BertModel", architecture="BertModel",
mteb_score=0.742285423, mteb_score=0.742285423,
pooling_type="MEAN", seq_pooling_type="MEAN",
attn_type="encoder_only", attn_type="encoder_only",
is_prefix_caching_supported=False, is_prefix_caching_supported=False,
is_chunked_prefill_supported=False, is_chunked_prefill_supported=False,
@@ -29,7 +29,7 @@ MODELS = [
"intfloat/multilingual-e5-base", "intfloat/multilingual-e5-base",
architecture="XLMRobertaModel", architecture="XLMRobertaModel",
mteb_score=0.779325955, mteb_score=0.779325955,
pooling_type="MEAN", seq_pooling_type="MEAN",
attn_type="encoder_only", attn_type="encoder_only",
is_prefix_caching_supported=False, is_prefix_caching_supported=False,
is_chunked_prefill_supported=False, is_chunked_prefill_supported=False,
@@ -24,7 +24,7 @@ EMBEDDING_MODELS = [
mteb_score=0.824413164, mteb_score=0.824413164,
architecture="XLMRobertaModel", architecture="XLMRobertaModel",
is_matryoshka=True, is_matryoshka=True,
pooling_type="MEAN", seq_pooling_type="MEAN",
attn_type="encoder_only", attn_type="encoder_only",
is_prefix_caching_supported=False, is_prefix_caching_supported=False,
is_chunked_prefill_supported=False, is_chunked_prefill_supported=False,
@@ -36,7 +36,7 @@ RERANK_MODELS = [
"jinaai/jina-reranker-v2-base-multilingual", "jinaai/jina-reranker-v2-base-multilingual",
mteb_score=0.33643, mteb_score=0.33643,
architecture="XLMRobertaForSequenceClassification", architecture="XLMRobertaForSequenceClassification",
pooling_type="CLS", seq_pooling_type="CLS",
attn_type="encoder_only", attn_type="encoder_only",
is_prefix_caching_supported=False, is_prefix_caching_supported=False,
is_chunked_prefill_supported=False, is_chunked_prefill_supported=False,
@@ -24,7 +24,7 @@ RERANK_MODELS = [
"mixedbread-ai/mxbai-rerank-base-v2", "mixedbread-ai/mxbai-rerank-base-v2",
architecture="Qwen2ForSequenceClassification", architecture="Qwen2ForSequenceClassification",
hf_overrides=mxbai_rerank_hf_overrides, hf_overrides=mxbai_rerank_hf_overrides,
pooling_type="LAST", seq_pooling_type="LAST",
attn_type="decoder", attn_type="decoder",
is_prefix_caching_supported=True, is_prefix_caching_supported=True,
is_chunked_prefill_supported=True, is_chunked_prefill_supported=True,
@@ -19,7 +19,7 @@ EMBEDDING_MODELS = [
"nvidia/llama-nemotron-embed-1b-v2", "nvidia/llama-nemotron-embed-1b-v2",
architecture="LlamaBidirectionalModel", architecture="LlamaBidirectionalModel",
mteb_score=0.689164662128673, mteb_score=0.689164662128673,
pooling_type="MEAN", seq_pooling_type="MEAN",
attn_type="encoder_only", attn_type="encoder_only",
is_prefix_caching_supported=False, is_prefix_caching_supported=False,
is_chunked_prefill_supported=False, is_chunked_prefill_supported=False,
@@ -32,7 +32,7 @@ RERANK_MODELS = [
architecture="LlamaBidirectionalForSequenceClassification", architecture="LlamaBidirectionalForSequenceClassification",
chat_template_name="nemotron-rerank.jinja", chat_template_name="nemotron-rerank.jinja",
mteb_score=0.33994, mteb_score=0.33994,
pooling_type="MEAN", seq_pooling_type="MEAN",
attn_type="encoder_only", attn_type="encoder_only",
is_prefix_caching_supported=False, is_prefix_caching_supported=False,
is_chunked_prefill_supported=False, is_chunked_prefill_supported=False,
@@ -14,7 +14,7 @@ MODELS = [
architecture="NomicBertModel", architecture="NomicBertModel",
mteb_score=0.737568559, mteb_score=0.737568559,
enable_test=True, enable_test=True,
pooling_type="MEAN", seq_pooling_type="MEAN",
attn_type="encoder_only", attn_type="encoder_only",
is_prefix_caching_supported=False, is_prefix_caching_supported=False,
is_chunked_prefill_supported=False, is_chunked_prefill_supported=False,
@@ -32,7 +32,7 @@ MODELS = [
architecture="NomicBertModel", architecture="NomicBertModel",
mteb_score=0.715488912, mteb_score=0.715488912,
enable_test=True, enable_test=True,
pooling_type="MEAN", seq_pooling_type="MEAN",
attn_type="encoder_only", attn_type="encoder_only",
is_prefix_caching_supported=False, is_prefix_caching_supported=False,
is_chunked_prefill_supported=False, is_chunked_prefill_supported=False,
@@ -27,7 +27,7 @@ RERANK_MODELS = [
architecture="Qwen3ForSequenceClassification", architecture="Qwen3ForSequenceClassification",
hf_overrides=qwen3_reranker_hf_overrides, hf_overrides=qwen3_reranker_hf_overrides,
chat_template_name="qwen3_reranker.jinja", chat_template_name="qwen3_reranker.jinja",
pooling_type="LAST", seq_pooling_type="LAST",
attn_type="decoder", attn_type="decoder",
is_prefix_caching_supported=True, is_prefix_caching_supported=True,
is_chunked_prefill_supported=True, is_chunked_prefill_supported=True,
@@ -14,7 +14,7 @@ MODELS = [
is_matryoshka=False, is_matryoshka=False,
architecture="BertModel", architecture="BertModel",
mteb_score=0.714927797, mteb_score=0.714927797,
pooling_type="CLS", seq_pooling_type="CLS",
attn_type="encoder_only", attn_type="encoder_only",
is_prefix_caching_supported=False, is_prefix_caching_supported=False,
is_chunked_prefill_supported=False, is_chunked_prefill_supported=False,
@@ -37,7 +37,7 @@ MODELS = [
is_matryoshka=False, is_matryoshka=False,
architecture="NomicBertModel", architecture="NomicBertModel",
mteb_score=0.681146831, mteb_score=0.681146831,
pooling_type="CLS", seq_pooling_type="CLS",
attn_type="encoder_only", attn_type="encoder_only",
is_prefix_caching_supported=False, is_prefix_caching_supported=False,
is_chunked_prefill_supported=False, is_chunked_prefill_supported=False,
@@ -54,7 +54,7 @@ MODELS = [
is_matryoshka=True, is_matryoshka=True,
architecture="BertModel", architecture="BertModel",
mteb_score=0.649088363, mteb_score=0.649088363,
pooling_type="CLS", seq_pooling_type="CLS",
attn_type="encoder_only", attn_type="encoder_only",
is_prefix_caching_supported=False, is_prefix_caching_supported=False,
is_chunked_prefill_supported=False, is_chunked_prefill_supported=False,
@@ -65,7 +65,7 @@ MODELS = [
is_matryoshka=True, is_matryoshka=True,
architecture="XLMRobertaModel", architecture="XLMRobertaModel",
mteb_score=0.712258299, mteb_score=0.712258299,
pooling_type="CLS", seq_pooling_type="CLS",
attn_type="encoder_only", attn_type="encoder_only",
is_prefix_caching_supported=False, is_prefix_caching_supported=False,
is_chunked_prefill_supported=False, is_chunked_prefill_supported=False,
@@ -76,7 +76,7 @@ MODELS = [
is_matryoshka=True, is_matryoshka=True,
architecture="GteModel", architecture="GteModel",
mteb_score=0.706622444, mteb_score=0.706622444,
pooling_type="CLS", seq_pooling_type="CLS",
attn_type="encoder_only", attn_type="encoder_only",
is_prefix_caching_supported=False, is_prefix_caching_supported=False,
is_chunked_prefill_supported=False, is_chunked_prefill_supported=False,
@@ -14,7 +14,7 @@ ST_PROJECTOR_MODELS = [
"TencentBAC/Conan-embedding-v1", "TencentBAC/Conan-embedding-v1",
architecture="BertModel", architecture="BertModel",
mteb_score=0.688611955, mteb_score=0.688611955,
pooling_type="MEAN", seq_pooling_type="MEAN",
attn_type="encoder_only", attn_type="encoder_only",
is_prefix_caching_supported=False, is_prefix_caching_supported=False,
is_chunked_prefill_supported=False, is_chunked_prefill_supported=False,
@@ -24,7 +24,7 @@ ST_PROJECTOR_MODELS = [
"google/embeddinggemma-300m", "google/embeddinggemma-300m",
architecture="Gemma3TextModel", architecture="Gemma3TextModel",
mteb_score=0.7473819294684156, mteb_score=0.7473819294684156,
pooling_type="MEAN", seq_pooling_type="MEAN",
attn_type="encoder_only", attn_type="encoder_only",
is_prefix_caching_supported=False, is_prefix_caching_supported=False,
is_chunked_prefill_supported=False, is_chunked_prefill_supported=False,
+3 -1
View File
@@ -11,6 +11,7 @@ import torch.nn.functional as F
from transformers import PretrainedConfig from transformers import PretrainedConfig
from vllm.config.model import AttnTypeStr, ModelConfig, ModelDType, RunnerOption from vllm.config.model import AttnTypeStr, ModelConfig, ModelDType, RunnerOption
from vllm.config.pooler import SequencePoolingType, TokenPoolingType
from vllm.logprobs import Logprob, PromptLogprobs, SampleLogprobs from vllm.logprobs import Logprob, PromptLogprobs, SampleLogprobs
from vllm.multimodal.processing import InputProcessingContext from vllm.multimodal.processing import InputProcessingContext
from vllm.tokenizers import cached_tokenizer_from_config from vllm.tokenizers import cached_tokenizer_from_config
@@ -379,7 +380,8 @@ class ModelInfo:
max_model_len: int | None = None max_model_len: int | None = None
hf_dtype: str = "float32" hf_dtype: str = "float32"
hf_overrides: dict[str, Any] | None = None hf_overrides: dict[str, Any] | None = None
pooling_type: str | None = None seq_pooling_type: SequencePoolingType | None = None
tok_pooling_type: TokenPoolingType | None = None
attn_type: AttnTypeStr | None = None attn_type: AttnTypeStr | None = None
is_prefix_caching_supported: bool | None = None is_prefix_caching_supported: bool | None = None
is_chunked_prefill_supported: bool | None = None is_chunked_prefill_supported: bool | None = None
+49 -37
View File
@@ -161,7 +161,8 @@ def test_get_pooling_config():
assert model_config.pooler_config is not None assert model_config.pooler_config is not None
assert model_config.pooler_config.normalize assert model_config.pooler_config.normalize
assert model_config.pooler_config.pooling_type == "MEAN" assert model_config.pooler_config.seq_pooling_type == "MEAN"
assert model_config.pooler_config.tok_pooling_type == "ALL"
@pytest.mark.skipif( @pytest.mark.skipif(
@@ -169,7 +170,7 @@ def test_get_pooling_config():
) )
def test_get_pooling_config_from_args(): def test_get_pooling_config_from_args():
model_id = "sentence-transformers/all-MiniLM-L12-v2" model_id = "sentence-transformers/all-MiniLM-L12-v2"
pooler_config = PoolerConfig(pooling_type="CLS", normalize=True) pooler_config = PoolerConfig(seq_pooling_type="CLS", normalize=True)
model_config = ModelConfig(model_id, pooler_config=pooler_config) model_config = ModelConfig(model_id, pooler_config=pooler_config)
assert asdict(model_config.pooler_config) == asdict(pooler_config) assert asdict(model_config.pooler_config) == asdict(pooler_config)
@@ -180,14 +181,25 @@ def test_get_pooling_config_from_args():
[ [
("tomaarsen/Qwen3-Reranker-0.6B-seq-cls", "LAST", "LAST"), # LLM ("tomaarsen/Qwen3-Reranker-0.6B-seq-cls", "LAST", "LAST"), # LLM
("intfloat/e5-small", "CLS", "MEAN"), # BertModel ("intfloat/e5-small", "CLS", "MEAN"), # BertModel
],
)
def test_default_seq_pooling_type(model_id, default_pooling_type, pooling_type):
model_config = ModelConfig(model_id)
assert model_config._model_info.default_seq_pooling_type == default_pooling_type
assert model_config.pooler_config.seq_pooling_type == pooling_type
@pytest.mark.parametrize(
("model_id", "default_pooling_type", "pooling_type"),
[
("Qwen/Qwen2.5-Math-RM-72B", "ALL", "ALL"), # reward ("Qwen/Qwen2.5-Math-RM-72B", "ALL", "ALL"), # reward
("Qwen/Qwen2.5-Math-PRM-7B", "STEP", "STEP"), # step reward ("Qwen/Qwen2.5-Math-PRM-7B", "STEP", "STEP"), # step reward
], ],
) )
def test_default_pooling_type(model_id, default_pooling_type, pooling_type): def test_default_tok_pooling_type(model_id, default_pooling_type, pooling_type):
model_config = ModelConfig(model_id) model_config = ModelConfig(model_id)
assert model_config._model_info.default_pooling_type == default_pooling_type assert model_config._model_info.default_tok_pooling_type == default_pooling_type
assert model_config.pooler_config.pooling_type == pooling_type assert model_config.pooler_config.tok_pooling_type == pooling_type
@pytest.mark.parametrize( @pytest.mark.parametrize(
@@ -554,100 +566,100 @@ def test_s3_url_different_models_create_different_directories(mock_pull_files):
"jason9693/Qwen2.5-1.5B-apeach", "jason9693/Qwen2.5-1.5B-apeach",
"decoder", "decoder",
True, True,
"Pooling models with causal attn and last pooling support chunked prefill.", "Pooling models with causal attn and LAST/ALL pooling support chunked prefill.", # noqa: E501
), ),
( (
"Qwen/Qwen3-Embedding-0.6B", "Qwen/Qwen3-Embedding-0.6B",
"decoder", "decoder",
True, True,
"Pooling models with causal attn and last pooling support chunked prefill.", "Pooling models with causal attn and LAST/ALL pooling support chunked prefill.", # noqa: E501
), ),
( (
"Qwen/Qwen2.5-Math-PRM-7B", "Qwen/Qwen2.5-Math-PRM-7B",
"decoder", "decoder",
False, False,
"Pooling models with step pooling does not support chunked prefill.", "Pooling models with causal attn and LAST/STEP pooling do not support chunked prefill.", # noqa: E501
), ),
( (
"internlm/internlm2-1_8b-reward", "internlm/internlm2-1_8b-reward",
"decoder", "decoder",
True, True,
"Pooling models with causal attn and all pooling support chunked prefill.", "Pooling models with causal attn and LAST/ALL pooling support chunked prefill.", # noqa: E501
), ),
( (
"BAAI/bge-base-en", "BAAI/bge-base-en",
"encoder_only", "encoder_only",
False, False,
"Pooling models with bidirectional attn does not support chunked prefill.", "Pooling models with bidirectional attn do not support chunked prefill.", # noqa: E501
), ),
( (
"boltuix/NeuroBERT-NER", "boltuix/NeuroBERT-NER",
"encoder_only", "encoder_only",
False, False,
"Pooling models with bidirectional attn does not support chunked prefill.", "Pooling models with bidirectional attn do not support chunked prefill.", # noqa: E501
), ),
( (
"papluca/xlm-roberta-base-language-detection", "papluca/xlm-roberta-base-language-detection",
"encoder_only", "encoder_only",
False, False,
"Pooling models with bidirectional attn does not support chunked prefill.", "Pooling models with bidirectional attn do not support chunked prefill.", # noqa: E501
), ),
( (
"Alibaba-NLP/gte-Qwen2-1.5B-instruct", "Alibaba-NLP/gte-Qwen2-1.5B-instruct",
"encoder_only", "encoder_only",
False, False,
"Pooling models with bidirectional attn does not support chunked prefill.", "Pooling models with bidirectional attn do not support chunked prefill.", # noqa: E501
), ),
( (
"intfloat/e5-small", "intfloat/e5-small",
"encoder_only", "encoder_only",
False, False,
"Pooling models with bidirectional attn does not support chunked prefill.", "Pooling models with bidirectional attn do not support chunked prefill.", # noqa: E501
), ),
# multimodal models # multimodal models
( (
"openai/clip-vit-base-patch32", "openai/clip-vit-base-patch32",
"decoder", "decoder",
True, True,
"Pooling models with causal attn and last pooling support chunked prefill.", "Pooling models with causal attn and LAST/ALL pooling support chunked prefill.", # noqa: E501
), ),
( (
"google/siglip-base-patch16-224", "google/siglip-base-patch16-224",
"encoder_only", "encoder_only",
False, False,
"Pooling models with bidirectional attn does not support chunked prefill.", "Pooling models with bidirectional attn do not support chunked prefill.", # noqa: E501
), ),
# generate models # generate models
( (
"Qwen/Qwen3-0.6B", "Qwen/Qwen3-0.6B",
"decoder", "decoder",
True, True,
"Generative models support chunked prefill.", "Generative models support chunked prefill.", # noqa: E501
), ),
( (
"Qwen/Qwen3-Next-80B-A3B-Instruct", "Qwen/Qwen3-Next-80B-A3B-Instruct",
"hybrid", "hybrid",
True, True,
"Generative models support chunked prefill.", "Generative models support chunked prefill.", # noqa: E501
), ),
( (
"ibm-granite/granite-4.0-h-small", "ibm-granite/granite-4.0-h-small",
"hybrid", "hybrid",
True, True,
"Generative models support chunked prefill.", "Generative models support chunked prefill.", # noqa: E501
), ),
( (
"state-spaces/mamba-130m-hf", "state-spaces/mamba-130m-hf",
"attention_free", "attention_free",
True, True,
"Generative models support chunked prefill.", "Generative models support chunked prefill.", # noqa: E501
), ),
# encoder_decoder models # encoder_decoder models
( (
"openai/whisper-small", "openai/whisper-small",
"encoder_decoder", "encoder_decoder",
False, False,
"Encoder decoder models does not support chunked prefill.", "Encoder decoder models do not support chunked prefill.", # noqa: E501
), ),
], ],
) )
@@ -673,100 +685,100 @@ def test_is_chunked_prefill_supported(
"jason9693/Qwen2.5-1.5B-apeach", "jason9693/Qwen2.5-1.5B-apeach",
"decoder", "decoder",
True, True,
"Pooling models with causal attn and last pooling support prefix caching.", "Pooling models with causal attn and LAST/ALL pooling support prefix caching.", # noqa: E501
), ),
( (
"Qwen/Qwen3-Embedding-0.6B", "Qwen/Qwen3-Embedding-0.6B",
"decoder", "decoder",
True, True,
"Pooling models with causal attn and last pooling support prefix caching.", "Pooling models with causal attn and LAST/ALL pooling support prefix caching.", # noqa: E501
), ),
( (
"Qwen/Qwen2.5-Math-PRM-7B", "Qwen/Qwen2.5-Math-PRM-7B",
"decoder", "decoder",
False, False,
"Pooling models with step pooling does not support prefix caching.", "Pooling models with causal attn and LAST/STEP pooling do not support prefix caching.", # noqa: E501
), ),
( (
"internlm/internlm2-1_8b-reward", "internlm/internlm2-1_8b-reward",
"decoder", "decoder",
True, True,
"Pooling models with causal attn and all pooling support prefix caching.", "Pooling models with causal attn and LAST/ALL pooling support prefix caching.", # noqa: E501
), ),
( (
"BAAI/bge-base-en", "BAAI/bge-base-en",
"encoder_only", "encoder_only",
False, False,
"Pooling models with bidirectional attn does not support prefix caching.", "Pooling models with bidirectional attn do not support prefix caching.", # noqa: E501
), ),
( (
"boltuix/NeuroBERT-NER", "boltuix/NeuroBERT-NER",
"encoder_only", "encoder_only",
False, False,
"Pooling models with bidirectional attn does not support prefix caching.", "Pooling models with bidirectional attn do not support prefix caching.", # noqa: E501
), ),
( (
"papluca/xlm-roberta-base-language-detection", "papluca/xlm-roberta-base-language-detection",
"encoder_only", "encoder_only",
False, False,
"Pooling models with bidirectional attn does not support prefix caching.", "Pooling models with bidirectional attn do not support prefix caching.", # noqa: E501
), ),
( (
"Alibaba-NLP/gte-Qwen2-1.5B-instruct", "Alibaba-NLP/gte-Qwen2-1.5B-instruct",
"encoder_only", "encoder_only",
False, False,
"Pooling models with bidirectional attn does not support prefix caching.", "Pooling models with bidirectional attn do not support prefix caching.", # noqa: E501
), ),
( (
"intfloat/e5-small", "intfloat/e5-small",
"encoder_only", "encoder_only",
False, False,
"Pooling models with bidirectional attn does not support prefix caching.", "Pooling models with bidirectional attn do not support prefix caching.", # noqa: E501
), ),
# multimodal models # multimodal models
( (
"openai/clip-vit-base-patch32", "openai/clip-vit-base-patch32",
"decoder", "decoder",
True, True,
"Pooling models with causal attn and last pooling support prefix caching.", "Pooling models with causal attn and LAST/ALL pooling support prefix caching.", # noqa: E501
), ),
( (
"google/siglip-base-patch16-224", "google/siglip-base-patch16-224",
"encoder_only", "encoder_only",
False, False,
"Pooling models with bidirectional attn does not support prefix caching.", "Pooling models with bidirectional attn do not support prefix caching.", # noqa: E501
), ),
# generate models # generate models
( (
"Qwen/Qwen3-0.6B", "Qwen/Qwen3-0.6B",
"decoder", "decoder",
True, True,
"Generative models support prefix caching.", "Generative models support prefix caching.", # noqa: E501
), ),
( (
"Qwen/Qwen3-Next-80B-A3B-Instruct", "Qwen/Qwen3-Next-80B-A3B-Instruct",
"hybrid", "hybrid",
False, False,
"Hybrid models does not support prefix caching since the feature is still experimental.", # noqa: E501 "Hybrid models do not support prefix caching since the feature is still experimental.", # noqa: E501
), ),
( (
"ibm-granite/granite-4.0-h-small", "ibm-granite/granite-4.0-h-small",
"hybrid", "hybrid",
False, False,
"Hybrid models does not support prefix caching since the feature is still experimental.", # noqa: E501 "Hybrid models do not support prefix caching since the feature is still experimental.", # noqa: E501
), ),
( (
"state-spaces/mamba-130m-hf", "state-spaces/mamba-130m-hf",
"attention_free", "attention_free",
False, False,
"Attention free models does not support prefix caching since the feature is still experimental.", # noqa: E501 "Attention free models do not support prefix caching since the feature is still experimental.", # noqa: E501
), ),
# encoder_decoder models # encoder_decoder models
( (
"openai/whisper-small", "openai/whisper-small",
"encoder_decoder", "encoder_decoder",
False, False,
"Encoder decoder models does not support prefix caching.", "Encoder decoder models do not support prefix caching.", # noqa: E501
), ),
], ],
) )
+4 -4
View File
@@ -40,7 +40,7 @@ def test_task():
def test_embed(): def test_embed():
task = "embed" task = "embed"
model_config = MockModelConfig(pooler_config=PoolerConfig(pooling_type="CLS")) model_config = MockModelConfig(pooler_config=PoolerConfig(seq_pooling_type="CLS"))
pooling_params = PoolingParams(normalize=None) pooling_params = PoolingParams(normalize=None)
pooling_params.verify(task=task, model_config=model_config) pooling_params.verify(task=task, model_config=model_config)
@@ -86,7 +86,7 @@ def test_embed_dimensions(model_info: EmbedModelInfo):
@pytest.mark.parametrize("task", ["score", "classify"]) @pytest.mark.parametrize("task", ["score", "classify"])
def test_classify(task): def test_classify(task):
model_config = MockModelConfig(pooler_config=PoolerConfig(pooling_type="CLS")) model_config = MockModelConfig(pooler_config=PoolerConfig(seq_pooling_type="CLS"))
pooling_params = PoolingParams(use_activation=None) pooling_params = PoolingParams(use_activation=None)
pooling_params.verify(task=task, model_config=model_config) pooling_params.verify(task=task, model_config=model_config)
@@ -108,7 +108,7 @@ def test_classify(task):
def test_token_embed(pooling_type: str): def test_token_embed(pooling_type: str):
task = "token_embed" task = "token_embed"
model_config = MockModelConfig( model_config = MockModelConfig(
pooler_config=PoolerConfig(pooling_type=pooling_type) pooler_config=PoolerConfig(tok_pooling_type=pooling_type)
) )
pooling_params = PoolingParams(normalize=None) pooling_params = PoolingParams(normalize=None)
@@ -134,7 +134,7 @@ def test_token_embed(pooling_type: str):
def test_token_classify(pooling_type: str): def test_token_classify(pooling_type: str):
task = "token_classify" task = "token_classify"
model_config = MockModelConfig( model_config = MockModelConfig(
pooler_config=PoolerConfig(pooling_type=pooling_type) pooler_config=PoolerConfig(tok_pooling_type=pooling_type)
) )
pooling_params = PoolingParams(use_activation=None) pooling_params = PoolingParams(use_activation=None)
+55 -39
View File
@@ -539,9 +539,12 @@ class ModelConfig:
if getattr(self.pooler_config, k) is None: if getattr(self.pooler_config, k) is None:
setattr(self.pooler_config, k, v) setattr(self.pooler_config, k, v)
default_pooling_type = self._model_info.default_pooling_type default_seq_pooling_type = self._model_info.default_seq_pooling_type
if self.pooler_config.pooling_type is None: if self.pooler_config.seq_pooling_type is None:
self.pooler_config.pooling_type = default_pooling_type self.pooler_config.seq_pooling_type = default_seq_pooling_type
default_tok_pooling_type = self._model_info.default_tok_pooling_type
if self.pooler_config.tok_pooling_type is None:
self.pooler_config.tok_pooling_type = default_tok_pooling_type
self.dtype: torch.dtype = _get_and_verify_dtype( self.dtype: torch.dtype = _get_and_verify_dtype(
self.model, self.model,
@@ -1543,8 +1546,8 @@ class ModelConfig:
@property @property
def attn_type(self) -> AttnTypeStr: def attn_type(self) -> AttnTypeStr:
if self.pooler_config is not None: if self.pooler_config is not None:
pooling_type = self._model_info.default_pooling_type.lower() seq_pooling_type = self._model_info.default_seq_pooling_type
if pooling_type == "cls": if seq_pooling_type == "CLS":
return "encoder_only" return "encoder_only"
else: else:
is_causal = getattr(self.hf_config, "is_causal", True) is_causal = getattr(self.hf_config, "is_causal", True)
@@ -1561,89 +1564,102 @@ class ModelConfig:
@property @property
def is_chunked_prefill_supported(self) -> bool: def is_chunked_prefill_supported(self) -> bool:
attn_type = self.attn_type attn_type = self.attn_type
if self.pooler_config is not None:
if pooler_config := self.pooler_config:
# for pooling models # for pooling models
if attn_type == "encoder_only": if attn_type == "encoder_only":
logger.debug( logger.debug(
"Pooling models with bidirectional attn does not support " "Pooling models with bidirectional attn "
"chunked prefill." "do not support chunked prefill."
) )
return False return False
elif attn_type == "decoder":
pooling_type = self.pooler_config.pooling_type.lower() if attn_type == "decoder":
if pooling_type in ["mean", "step", "cls"]: if (
pooler_config.seq_pooling_type in ("MEAN", "CLS")
or pooler_config.tok_pooling_type == "STEP"
):
logger.debug( logger.debug(
"Pooling models with %s pooling does not " "Pooling models with causal attn and %s/%s pooling "
"support chunked prefill.", "do not support chunked prefill.",
pooling_type, pooler_config.seq_pooling_type,
pooler_config.tok_pooling_type,
) )
return False return False
elif pooling_type in ["all", "last"]: else:
logger.debug( logger.debug(
"Pooling models with causal attn and %s pooling support " "Pooling models with causal attn and %s/%s pooling "
"chunked prefill.", "support chunked prefill.",
pooling_type, pooler_config.seq_pooling_type,
pooler_config.tok_pooling_type,
) )
return True return True
else:
raise ValueError(f"{pooling_type=} not supported.")
# vllm currently does not have pooling models using hybrid, # vllm currently does not have pooling models using hybrid,
# attention_free or encoder_decoder attn types. # attention_free or encoder_decoder attn types.
return attn_type != "encoder_decoder" return attn_type != "encoder_decoder"
else: else:
# for generative models
if attn_type == "encoder_decoder": if attn_type == "encoder_decoder":
logger.debug("Encoder decoder models does not support chunked prefill.") logger.debug("Encoder decoder models do not support chunked prefill.")
return False return False
logger.debug("Generative models support chunked prefill.") logger.debug("Generative models support chunked prefill.")
return True return True
@property @property
def is_prefix_caching_supported(self) -> bool: def is_prefix_caching_supported(self) -> bool:
attn_type = self.attn_type attn_type = self.attn_type
if self.pooler_config is not None:
if pooler_config := self.pooler_config:
# for pooling models # for pooling models
if attn_type == "encoder_only": if attn_type == "encoder_only":
logger.debug( logger.debug(
"Pooling models with bidirectional attn does not " "Pooling models with bidirectional attn "
"support prefix caching." "do not support prefix caching."
) )
return False return False
elif attn_type == "decoder":
pooling_type = self.pooler_config.pooling_type.lower() if attn_type == "decoder":
if pooling_type in ["mean", "step", "cls"]: if (
pooler_config.seq_pooling_type in ("MEAN", "CLS")
or pooler_config.tok_pooling_type == "STEP"
):
logger.debug( logger.debug(
"Pooling models with %s pooling does not " "Pooling models with causal attn and %s/%s pooling "
"support prefix caching.", "do not support prefix caching.",
pooling_type, pooler_config.seq_pooling_type,
pooler_config.tok_pooling_type,
) )
return False return False
elif pooling_type in ["all", "last"]: else:
logger.debug( logger.debug(
"Pooling models with causal attn and %s pooling support " "Pooling models with causal attn and %s/%s pooling "
"prefix caching.", "support prefix caching.",
pooling_type, pooler_config.seq_pooling_type,
pooler_config.tok_pooling_type,
) )
return True return True
else:
raise ValueError(f"{pooling_type=} not supported.")
# vllm currently does not have pooling models using hybrid, # vllm currently does not have pooling models using hybrid,
# attention_free or encoder_decoder attn types. # attention_free or encoder_decoder attn types.
return False return False
else: else:
# for generative models
if attn_type == "hybrid": if attn_type == "hybrid":
logger.debug( logger.debug(
"Hybrid models does not support prefix caching since the feature " "Hybrid models do not support prefix caching since the feature "
"is still experimental." "is still experimental."
) )
return False return False
elif attn_type == "attention_free": elif attn_type == "attention_free":
logger.debug( logger.debug(
"Attention free models does not support prefix caching since the " "Attention free models do not support prefix caching since the "
"feature is still experimental." "feature is still experimental."
) )
return False return False
elif attn_type == "encoder_decoder": elif attn_type == "encoder_decoder":
logger.debug("Encoder decoder models does not support prefix caching.") logger.debug("Encoder decoder models do not support prefix caching.")
return False return False
else: # attn_type == "decoder" else: # attn_type == "decoder"
logger.debug("Generative models support prefix caching.") logger.debug("Generative models support prefix caching.")
+59 -7
View File
@@ -1,7 +1,7 @@
# SPDX-License-Identifier: Apache-2.0 # SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from typing import Any, Literal from typing import Any, Literal, get_args
from pydantic.dataclasses import dataclass from pydantic.dataclasses import dataclass
@@ -11,7 +11,11 @@ from vllm.utils.hashing import safe_hash
logger = init_logger(__name__) logger = init_logger(__name__)
PoolingTypeStr = Literal["LAST", "ALL", "CLS", "STEP", "MEAN"] SequencePoolingType = Literal["CLS", "LAST", "MEAN"]
SEQ_POOLING_TYPES: tuple[SequencePoolingType, ...] = get_args(SequencePoolingType)
TokenPoolingType = Literal["ALL", "STEP"]
TOK_POOLING_TYPES: tuple[TokenPoolingType, ...] = get_args(TokenPoolingType)
@config @config
@@ -19,9 +23,26 @@ PoolingTypeStr = Literal["LAST", "ALL", "CLS", "STEP", "MEAN"]
class PoolerConfig: class PoolerConfig:
"""Controls the behavior of output pooling in pooling models.""" """Controls the behavior of output pooling in pooling models."""
pooling_type: PoolingTypeStr | None = None pooling_type: SequencePoolingType | TokenPoolingType | None = None
""" """
The pooling method of the pooling model. The pooling method used for pooling.
If set, `seq_pooling_type` or `tok_pooling_type` are automatically populated
with this field. Alternatively, users can set `seq_pooling_type` and
`tok_pooling_type` explicitly.
This field is mainly for user convenience. Internal code should always use
`seq_pooling_type` or `tok_pooling_type` instead of `pooling_type`.
"""
seq_pooling_type: SequencePoolingType | None = None
"""
The pooling method used for sequence pooling.
"""
tok_pooling_type: TokenPoolingType | None = None
"""
The pooling method used for tokenwise pooling.
""" """
## for embeddings models ## for embeddings models
@@ -88,9 +109,40 @@ class PoolerConfig:
# raise deprecated warning for softmax and activation # raise deprecated warning for softmax and activation
self.use_activation = get_use_activation(self) self.use_activation = get_use_activation(self)
def get_pooling_type(self) -> PoolingTypeStr: if pooling_type := self.pooling_type:
assert self.pooling_type is not None, "Should be resolved by ModelConfig" if self.seq_pooling_type is not None:
return self.pooling_type raise ValueError(
"Cannot set both `pooling_type` and `seq_pooling_type`"
)
if self.tok_pooling_type is not None:
raise ValueError(
"Cannot set both `pooling_type` and `tok_pooling_type`"
)
if pooling_type in SEQ_POOLING_TYPES:
logger.debug(
"Resolved `pooling_type=%r` to `seq_pooling_type=%r`.",
pooling_type,
pooling_type,
)
self.seq_pooling_type = pooling_type
elif pooling_type in TOK_POOLING_TYPES:
logger.debug(
"Resolved `pooling_type=%r` to `tok_pooling_type=%r`.",
pooling_type,
pooling_type,
)
self.tok_pooling_type = pooling_type
else:
raise NotImplementedError(pooling_type)
def get_seq_pooling_type(self) -> SequencePoolingType:
assert self.seq_pooling_type is not None, "Should be resolved by ModelConfig"
return self.seq_pooling_type
def get_tok_pooling_type(self) -> TokenPoolingType:
assert self.tok_pooling_type is not None, "Should be resolved by ModelConfig"
return self.tok_pooling_type
def compute_hash(self) -> str: def compute_hash(self) -> str:
""" """
+1 -1
View File
@@ -172,7 +172,7 @@ class LLM:
The available overrides depend on the model that is being run. The available overrides depend on the model that is being run.
For example, for Phi-3-Vision: `{"num_crops": 4}`. For example, for Phi-3-Vision: `{"num_crops": 4}`.
pooler_config: Initialize non-default pooling config for the pooling pooler_config: Initialize non-default pooling config for the pooling
model. e.g. `PoolerConfig(pooling_type="mean", normalize=False)`. model. e.g. `PoolerConfig(seq_pooling_type="MEAN", normalize=False)`.
compilation_config: Either an integer or a dictionary. If it is an compilation_config: Either an integer or a dictionary. If it is an
integer, it is used as the mode of compilation optimization. If it integer, it is used as the mode of compilation optimization. If it
is a dictionary, it can specify the full compilation configuration. is a dictionary, it can specify the full compilation configuration.
@@ -7,7 +7,7 @@ from typing import TypeAlias
import torch import torch
import torch.nn as nn import torch.nn as nn
from vllm.config.pooler import PoolingTypeStr from vllm.config.pooler import SequencePoolingType
from vllm.model_executor.layers.pooler import PoolingParamsUpdate from vllm.model_executor.layers.pooler import PoolingParamsUpdate
from vllm.tasks import PoolingTask from vllm.tasks import PoolingTask
from vllm.v1.pool.metadata import PoolingMetadata from vllm.v1.pool.metadata import PoolingMetadata
@@ -82,11 +82,11 @@ class MeanPool(SequencePoolingMethod):
) / prompt_lens.unsqueeze(1) ) / prompt_lens.unsqueeze(1)
def get_seq_pooling_method(pooling_type: PoolingTypeStr | str): def get_seq_pooling_method(pooling_type: SequencePoolingType | str):
if pooling_type == "LAST":
return LastPool()
if pooling_type == "CLS": if pooling_type == "CLS":
return CLSPool() return CLSPool()
if pooling_type == "LAST":
return LastPool()
if pooling_type == "MEAN": if pooling_type == "MEAN":
return MeanPool() return MeanPool()
@@ -85,7 +85,7 @@ class SequencePooler(Pooler):
def pooler_for_embed(pooler_config: PoolerConfig): def pooler_for_embed(pooler_config: PoolerConfig):
pooling = get_seq_pooling_method(pooler_config.get_pooling_type()) pooling = get_seq_pooling_method(pooler_config.get_seq_pooling_type())
head = EmbeddingPoolerHead() head = EmbeddingPoolerHead()
return SequencePooler(pooling=pooling, head=head) return SequencePooler(pooling=pooling, head=head)
@@ -99,7 +99,7 @@ def pooler_for_classify(
act_fn: PoolerActivation | str | None = None, act_fn: PoolerActivation | str | None = None,
): ):
if pooling is None: if pooling is None:
pooling = get_seq_pooling_method(pooler_config.get_pooling_type()) pooling = get_seq_pooling_method(pooler_config.get_seq_pooling_type())
head = ClassifierPoolerHead(classifier=classifier, act_fn=act_fn) head = ClassifierPoolerHead(classifier=classifier, act_fn=act_fn)
@@ -8,7 +8,7 @@ import torch
import torch.nn as nn import torch.nn as nn
from vllm.config import get_current_vllm_config from vllm.config import get_current_vllm_config
from vllm.config.pooler import PoolingTypeStr from vllm.config.pooler import TokenPoolingType
from vllm.model_executor.layers.pooler import PoolingParamsUpdate from vllm.model_executor.layers.pooler import PoolingParamsUpdate
from vllm.tasks import PoolingTask from vllm.tasks import PoolingTask
from vllm.v1.pool.metadata import PoolingMetadata from vllm.v1.pool.metadata import PoolingMetadata
@@ -113,12 +113,10 @@ class StepPool(AllPool):
return pooled_data return pooled_data
def get_tok_pooling_method(pooling_type: PoolingTypeStr | str): def get_tok_pooling_method(pooling_type: TokenPoolingType | str):
if pooling_type == "ALL": if pooling_type == "ALL":
return AllPool() return AllPool()
if pooling_type == "STEP": if pooling_type == "STEP":
return StepPool() return StepPool()
# TODO: Separate seq and tok pooling types so we don't need this fallback
return AllPool()
raise NotImplementedError(f"Unknown tokenwise pooling type: {pooling_type!r}") raise NotImplementedError(f"Unknown tokenwise pooling type: {pooling_type!r}")
@@ -85,7 +85,7 @@ class TokenPooler(Pooler):
def pooler_for_token_embed(pooler_config: PoolerConfig): def pooler_for_token_embed(pooler_config: PoolerConfig):
pooling = get_tok_pooling_method(pooler_config.get_pooling_type()) pooling = get_tok_pooling_method(pooler_config.get_tok_pooling_type())
head = TokenEmbeddingPoolerHead() head = TokenEmbeddingPoolerHead()
return TokenPooler(pooling=pooling, head=head) return TokenPooler(pooling=pooling, head=head)
@@ -99,7 +99,7 @@ def pooler_for_token_classify(
act_fn: PoolerActivation | str | None = None, act_fn: PoolerActivation | str | None = None,
): ):
if pooling is None: if pooling is None:
pooling = get_tok_pooling_method(pooler_config.get_pooling_type()) pooling = get_tok_pooling_method(pooler_config.get_tok_pooling_type())
head = TokenClassifierPoolerHead(classifier=classifier, act_fn=act_fn) head = TokenClassifierPoolerHead(classifier=classifier, act_fn=act_fn)
+5 -5
View File
@@ -357,7 +357,7 @@ class BertOutput(nn.Module):
@support_torch_compile @support_torch_compile
@default_pooling_type("CLS") @default_pooling_type(seq_pooling_type="CLS")
class BertModel(nn.Module, SupportsQuant): class BertModel(nn.Module, SupportsQuant):
is_pooling_model = True is_pooling_model = True
@@ -461,7 +461,7 @@ class BertPoolingModel(BertModel):
return loaded_params return loaded_params
@default_pooling_type("CLS") @default_pooling_type(seq_pooling_type="CLS")
class BertEmbeddingModel(nn.Module, SupportsQuant): class BertEmbeddingModel(nn.Module, SupportsQuant):
"""A model that uses Bert to provide embedding functionalities. """A model that uses Bert to provide embedding functionalities.
@@ -675,7 +675,7 @@ class SPLADESparsePooler(Pooler):
return torch.stack(pooled_list, dim=0).contiguous() return torch.stack(pooled_list, dim=0).contiguous()
@default_pooling_type("CLS") @default_pooling_type(seq_pooling_type="CLS")
class BertSpladeSparseEmbeddingModel(BertEmbeddingModel): class BertSpladeSparseEmbeddingModel(BertEmbeddingModel):
""" """
BertEmbeddingModel + SPLADE sparse embedding. BertEmbeddingModel + SPLADE sparse embedding.
@@ -780,7 +780,7 @@ class BertSpladeSparseEmbeddingModel(BertEmbeddingModel):
return loaded return loaded
@default_pooling_type("CLS") @default_pooling_type(seq_pooling_type="CLS")
class BertForSequenceClassification(nn.Module, SupportsCrossEncoding, SupportsQuant): class BertForSequenceClassification(nn.Module, SupportsCrossEncoding, SupportsQuant):
"""A model that uses Bert to provide embedding functionalities. """A model that uses Bert to provide embedding functionalities.
@@ -849,7 +849,7 @@ class BertForSequenceClassification(nn.Module, SupportsCrossEncoding, SupportsQu
@attn_type("encoder_only") @attn_type("encoder_only")
@default_pooling_type("ALL") @default_pooling_type(tok_pooling_type="ALL")
class BertForTokenClassification(nn.Module): class BertForTokenClassification(nn.Module):
is_pooling_model = True is_pooling_model = True
+2 -2
View File
@@ -441,7 +441,7 @@ class BertWithRopeEncoder(nn.Module):
@support_torch_compile @support_torch_compile
@default_pooling_type("CLS") @default_pooling_type(seq_pooling_type="CLS")
class BertWithRope(nn.Module, SupportsQuant): class BertWithRope(nn.Module, SupportsQuant):
hf_to_vllm_mapper = WeightsMapper(orig_to_new_prefix={"model.": ""}) hf_to_vllm_mapper = WeightsMapper(orig_to_new_prefix={"model.": ""})
@@ -670,7 +670,7 @@ class JinaRobertaModel(BertWithRope):
return super().load_weights(weights) return super().load_weights(weights)
@default_pooling_type("CLS") @default_pooling_type(seq_pooling_type="CLS")
class GteNewForSequenceClassification(nn.Module, SupportsCrossEncoding): class GteNewForSequenceClassification(nn.Module, SupportsCrossEncoding):
is_pooling_model = True is_pooling_model = True
+3 -3
View File
@@ -145,7 +145,7 @@ class CLIPProcessingInfo(BaseProcessingInfo):
image_width=image_width, image_width=image_width,
image_height=image_height, image_height=image_height,
), ),
_get_vision_feature_select_strategy(pooler_config.pooling_type), _get_vision_feature_select_strategy(pooler_config.seq_pooling_type),
) )
def get_image_size_with_most_features(self) -> ImageSize: def get_image_size_with_most_features(self) -> ImageSize:
@@ -819,7 +819,7 @@ class CLIPVisionModel(nn.Module):
# Assume EOS token corresponds to LAST token in text model # Assume EOS token corresponds to LAST token in text model
@default_pooling_type("LAST") @default_pooling_type(seq_pooling_type="LAST")
@MULTIMODAL_REGISTRY.register_processor( @MULTIMODAL_REGISTRY.register_processor(
CLIPMultiModalProcessor, CLIPMultiModalProcessor,
info=CLIPProcessingInfo, info=CLIPProcessingInfo,
@@ -908,7 +908,7 @@ class CLIPEmbeddingModel(nn.Module, SupportsMultiModal, SupportsQuant):
) -> torch.Tensor: ) -> torch.Tensor:
if feature_select_strategy is None: if feature_select_strategy is None:
feature_select_strategy = _get_vision_feature_select_strategy( feature_select_strategy = _get_vision_feature_select_strategy(
self.pooler_config.pooling_type self.pooler_config.seq_pooling_type
) )
pooled_output = self.vision_model( pooled_output = self.vision_model(
+5 -4
View File
@@ -94,12 +94,12 @@ class JinaRobertaModelConfig(VerifyAndUpdateConfig):
class LlamaBidirectionalConfig(VerifyAndUpdateConfig): class LlamaBidirectionalConfig(VerifyAndUpdateConfig):
@staticmethod @staticmethod
def verify_and_update_model_config(model_config: "ModelConfig") -> None: def verify_and_update_model_config(model_config: "ModelConfig") -> None:
from vllm.config.pooler import PoolingTypeStr from vllm.config.pooler import SequencePoolingType
hf_config = model_config.hf_config hf_config = model_config.hf_config
hf_config.is_causal = False hf_config.is_causal = False
pooling_type_map: dict[str, PoolingTypeStr] = { pooling_type_map: dict[str, SequencePoolingType] = {
"avg": "MEAN", "avg": "MEAN",
"cls": "CLS", "cls": "CLS",
"last": "LAST", "last": "LAST",
@@ -107,8 +107,9 @@ class LlamaBidirectionalConfig(VerifyAndUpdateConfig):
pooling_type = pooling_type_map.get(hf_config.pooling, None) pooling_type = pooling_type_map.get(hf_config.pooling, None)
if pooling_type is None: if pooling_type is None:
raise ValueError(f"pool_type {hf_config.pooling} not supported") raise ValueError(f"pool_type {hf_config.pooling!r} not supported")
model_config.pooler_config.pooling_type = pooling_type
model_config.pooler_config.seq_pooling_type = pooling_type
class NomicBertModelConfig(VerifyAndUpdateConfig): class NomicBertModelConfig(VerifyAndUpdateConfig):
+1 -1
View File
@@ -193,7 +193,7 @@ class GritLMPooler(SequencePooler):
return self.activation(pooled_data) return self.activation(pooled_data)
@default_pooling_type("MEAN") @default_pooling_type(seq_pooling_type="MEAN")
class GritLM(LlamaForCausalLM): class GritLM(LlamaForCausalLM):
"""This class implements the embedding model for parasail-ai/GritLM-7B-vllm. """This class implements the embedding model for parasail-ai/GritLM-7B-vllm.
+33 -9
View File
@@ -20,12 +20,13 @@ from vllm.utils.func_utils import supports_kw
if TYPE_CHECKING: if TYPE_CHECKING:
from vllm.config import VllmConfig from vllm.config import VllmConfig
from vllm.config.model import AttnTypeStr from vllm.config.model import AttnTypeStr
from vllm.config.pooler import PoolingTypeStr from vllm.config.pooler import SequencePoolingType, TokenPoolingType
from vllm.model_executor.layers.pooler import Pooler from vllm.model_executor.layers.pooler import Pooler
else: else:
VllmConfig = Any VllmConfig = Any
Pooler = Any Pooler = Any
PoolingTypeStr = Any SequencePoolingType = Any
TokenPoolingType = Any
AttnTypeStr = Any AttnTypeStr = Any
logger = init_logger(__name__) logger = init_logger(__name__)
@@ -155,9 +156,19 @@ class VllmModelForPooling(VllmModel[T_co], Protocol[T_co]):
MRO of your model class. MRO of your model class.
""" """
default_pooling_type: ClassVar[PoolingTypeStr] = "LAST" default_seq_pooling_type: ClassVar[SequencePoolingType] = "LAST"
""" """
Indicates the [vllm.config.pooler.PoolerConfig.pooling_type][] Indicates the [vllm.config.pooler.PoolerConfig.seq_pooling_type][]
to use by default.
You can use the
[vllm.model_executor.models.interfaces_base.default_pooling_type][]
decorator to conveniently set this field.
"""
default_tok_pooling_type: ClassVar[TokenPoolingType] = "ALL"
"""
Indicates the [vllm.config.pooler.PoolerConfig.tok_pooling_type][]
to use by default. to use by default.
You can use the You can use the
@@ -200,18 +211,31 @@ def is_pooling_model(
_T = TypeVar("_T", bound=type[nn.Module]) _T = TypeVar("_T", bound=type[nn.Module])
def default_pooling_type(pooling_type: PoolingTypeStr): def default_pooling_type(
"""Decorator to set `VllmModelForPooling.default_pooling_type`.""" *,
seq_pooling_type: SequencePoolingType = "LAST",
tok_pooling_type: TokenPoolingType = "ALL",
):
"""Decorator to set `VllmModelForPooling.default_*_pooling_type`."""
def func(model: _T) -> _T: def func(model: _T) -> _T:
model.default_pooling_type = pooling_type # type: ignore model.default_seq_pooling_type = seq_pooling_type # type: ignore
model.default_tok_pooling_type = tok_pooling_type # type: ignore
return model return model
return func return func
def get_default_pooling_type(model: type[object] | object) -> PoolingTypeStr: def get_default_seq_pooling_type(
return getattr(model, "default_pooling_type", "LAST") model: type[object] | object,
) -> SequencePoolingType:
return getattr(model, "default_seq_pooling_type", "LAST")
def get_default_tok_pooling_type(
model: type[object] | object,
) -> TokenPoolingType:
return getattr(model, "default_tok_pooling_type", "ALL")
def attn_type(attn_type: AttnTypeStr): def attn_type(attn_type: AttnTypeStr):
+1 -1
View File
@@ -402,7 +402,7 @@ class InternLM2ForCausalLM(nn.Module, SupportsPP, SupportsLoRA):
return loaded_params return loaded_params
@default_pooling_type("ALL") @default_pooling_type(tok_pooling_type="ALL")
class InternLM2ForRewardModel(InternLM2ForCausalLM): class InternLM2ForRewardModel(InternLM2ForCausalLM):
is_pooling_model = True is_pooling_model = True
+3 -3
View File
@@ -221,7 +221,7 @@ class ModernBertEncoderLayer(nn.Module):
@support_torch_compile @support_torch_compile
@default_pooling_type("CLS") @default_pooling_type(seq_pooling_type="CLS")
class ModernBertModel(nn.Module): class ModernBertModel(nn.Module):
hf_to_vllm_mapper = WeightsMapper( hf_to_vllm_mapper = WeightsMapper(
orig_to_new_prefix={"layers.": "encoder_layer.layers."} orig_to_new_prefix={"layers.": "encoder_layer.layers."}
@@ -308,7 +308,7 @@ class ModernBertPooler(SequencePooler):
return self.norm(self.act(self.dense(pooled_data))) return self.norm(self.act(self.dense(pooled_data)))
@default_pooling_type("CLS") @default_pooling_type(seq_pooling_type="CLS")
class ModernBertForSequenceClassification(nn.Module, SupportsCrossEncoding): class ModernBertForSequenceClassification(nn.Module, SupportsCrossEncoding):
is_pooling_model = True is_pooling_model = True
@@ -395,7 +395,7 @@ class ModernBertPredictionHead(nn.Module):
@attn_type("encoder_only") @attn_type("encoder_only")
@default_pooling_type("ALL") @default_pooling_type(tok_pooling_type="ALL")
class ModernBertForTokenClassification(nn.Module): class ModernBertForTokenClassification(nn.Module):
is_pooling_model = True is_pooling_model = True
+2 -2
View File
@@ -96,7 +96,7 @@ class Qwen2RewardBaseModel(nn.Module, SupportsLoRA, SupportsPP):
return loader.load_weights(weights) return loader.load_weights(weights)
@default_pooling_type("ALL") @default_pooling_type(tok_pooling_type="ALL")
class Qwen2ForRewardModel(Qwen2RewardBaseModel): class Qwen2ForRewardModel(Qwen2RewardBaseModel):
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""): def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
vllm_config.model_config.hf_config.num_labels = 1 vllm_config.model_config.hf_config.num_labels = 1
@@ -108,7 +108,7 @@ class Qwen2ForRewardModel(Qwen2RewardBaseModel):
self.pooler = pooler_for_token_classify(pooler_config) self.pooler = pooler_for_token_classify(pooler_config)
@default_pooling_type("STEP") @default_pooling_type(tok_pooling_type="STEP")
class Qwen2ForProcessRewardModel(Qwen2RewardBaseModel): class Qwen2ForProcessRewardModel(Qwen2RewardBaseModel):
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""): def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
vllm_config.model_config.hf_config.num_labels = 2 vllm_config.model_config.hf_config.num_labels = 2
+9 -5
View File
@@ -35,10 +35,11 @@ from vllm.utils.hashing import safe_hash
if TYPE_CHECKING: if TYPE_CHECKING:
from vllm.config.model import AttnTypeStr from vllm.config.model import AttnTypeStr
from vllm.config.pooler import PoolingTypeStr from vllm.config.pooler import SequencePoolingType, TokenPoolingType
else: else:
AttnTypeStr = Any AttnTypeStr = Any
PoolingTypeStr = Any SequencePoolingType = Any
TokenPoolingType = Any
from .interfaces import ( from .interfaces import (
@@ -57,7 +58,8 @@ from .interfaces import (
) )
from .interfaces_base import ( from .interfaces_base import (
get_attn_type, get_attn_type,
get_default_pooling_type, get_default_seq_pooling_type,
get_default_tok_pooling_type,
is_pooling_model, is_pooling_model,
is_text_generation_model, is_text_generation_model,
) )
@@ -548,7 +550,8 @@ class _ModelInfo:
is_text_generation_model: bool is_text_generation_model: bool
is_pooling_model: bool is_pooling_model: bool
attn_type: AttnTypeStr attn_type: AttnTypeStr
default_pooling_type: PoolingTypeStr default_seq_pooling_type: SequencePoolingType
default_tok_pooling_type: TokenPoolingType
supports_cross_encoding: bool supports_cross_encoding: bool
supports_multimodal: bool supports_multimodal: bool
supports_multimodal_raw_input_only: bool supports_multimodal_raw_input_only: bool
@@ -569,7 +572,8 @@ class _ModelInfo:
architecture=model.__name__, architecture=model.__name__,
is_text_generation_model=is_text_generation_model(model), is_text_generation_model=is_text_generation_model(model),
is_pooling_model=is_pooling_model(model), is_pooling_model=is_pooling_model(model),
default_pooling_type=get_default_pooling_type(model), default_seq_pooling_type=get_default_seq_pooling_type(model),
default_tok_pooling_type=get_default_tok_pooling_type(model),
attn_type=get_attn_type(model), attn_type=get_attn_type(model),
supports_cross_encoding=supports_cross_encoding(model), supports_cross_encoding=supports_cross_encoding(model),
supports_multimodal=supports_multimodal(model), supports_multimodal=supports_multimodal(model),
+2 -2
View File
@@ -93,7 +93,7 @@ class RobertaClassificationHead(nn.Module):
return x return x
@default_pooling_type("CLS") @default_pooling_type(seq_pooling_type="CLS")
class RobertaEmbeddingModel(BertEmbeddingModel): class RobertaEmbeddingModel(BertEmbeddingModel):
"""A model that uses Roberta to provide embedding functionalities.""" """A model that uses Roberta to provide embedding functionalities."""
@@ -150,7 +150,7 @@ class RobertaEmbeddingModel(BertEmbeddingModel):
return loader.load_weights(weights_list, mapper=mapper) return loader.load_weights(weights_list, mapper=mapper)
@default_pooling_type("CLS") @default_pooling_type(seq_pooling_type="CLS")
class RobertaForSequenceClassification(nn.Module, SupportsCrossEncoding): class RobertaForSequenceClassification(nn.Module, SupportsCrossEncoding):
"""A model that uses Roberta to provide embedding functionalities. """A model that uses Roberta to provide embedding functionalities.
+3 -3
View File
@@ -129,7 +129,7 @@ class SiglipProcessingInfo(BaseProcessingInfo):
image_width=image_width, image_width=image_width,
image_height=image_height, image_height=image_height,
), ),
_get_vision_feature_select_strategy(pooler_config.pooling_type), _get_vision_feature_select_strategy(pooler_config.seq_pooling_type),
) )
def get_image_size_with_most_features(self) -> ImageSize: def get_image_size_with_most_features(self) -> ImageSize:
@@ -998,7 +998,7 @@ class SiglipTextEmbeddings(nn.Module):
# Assume EOS token corresponds to CLS token in text model # Assume EOS token corresponds to CLS token in text model
@default_pooling_type("CLS") @default_pooling_type(seq_pooling_type="CLS")
@MULTIMODAL_REGISTRY.register_processor( @MULTIMODAL_REGISTRY.register_processor(
SiglipMultiModalProcessor, SiglipMultiModalProcessor,
info=SiglipProcessingInfo, info=SiglipProcessingInfo,
@@ -1125,7 +1125,7 @@ class SiglipEmbeddingModel(nn.Module, SupportsMultiModal, SupportsQuant):
) -> torch.Tensor: ) -> torch.Tensor:
if feature_select_strategy is None: if feature_select_strategy is None:
feature_select_strategy = _get_vision_feature_select_strategy( feature_select_strategy = _get_vision_feature_select_strategy(
self.pooler_config.pooling_type self.pooler_config.seq_pooling_type
) )
pooled_output = self.vision_model( pooled_output = self.vision_model(
+1 -1
View File
@@ -140,7 +140,7 @@ class PoolingParams(
self, pooler_config: "PoolerConfig", valid_parameters: list[str] self, pooler_config: "PoolerConfig", valid_parameters: list[str]
): ):
step_pooling_parameters = ["step_tag_id", "returned_token_ids"] step_pooling_parameters = ["step_tag_id", "returned_token_ids"]
if pooler_config.pooling_type != "STEP": if pooler_config.tok_pooling_type != "STEP":
invalid_parameters = [] invalid_parameters = []
for k in step_pooling_parameters: for k in step_pooling_parameters:
if getattr(self, k, None) is not None: if getattr(self, k, None) is not None:
+2 -2
View File
@@ -3,11 +3,11 @@
from typing import Literal, get_args from typing import Literal, get_args
GenerationTask = Literal["generate", "transcription"] GenerationTask = Literal["generate", "transcription"]
GENERATION_TASKS = get_args(GenerationTask) GENERATION_TASKS: tuple[GenerationTask, ...] = get_args(GenerationTask)
PoolingTask = Literal[ PoolingTask = Literal[
"embed", "classify", "score", "token_embed", "token_classify", "plugin" "embed", "classify", "score", "token_embed", "token_classify", "plugin"
] ]
POOLING_TASKS = get_args(PoolingTask) POOLING_TASKS: tuple[PoolingTask, ...] = get_args(PoolingTask)
SupportedTask = Literal[GenerationTask, PoolingTask] SupportedTask = Literal[GenerationTask, PoolingTask]
+24 -21
View File
@@ -10,9 +10,7 @@ from pathlib import Path
from typing import Any, Literal, TypeAlias from typing import Any, Literal, TypeAlias
import huggingface_hub import huggingface_hub
from huggingface_hub import ( from huggingface_hub import get_safetensors_metadata
get_safetensors_metadata,
)
from packaging.version import Version from packaging.version import Version
from transformers import GenerationConfig, PretrainedConfig from transformers import GenerationConfig, PretrainedConfig
from transformers.models.auto.image_processing_auto import get_image_processor_config from transformers.models.auto.image_processing_auto import get_image_processor_config
@@ -742,7 +740,10 @@ def get_config(
@cache @cache
def get_pooling_config(model: str, revision: str | None = "main") -> dict | None: def get_pooling_config(
model: str,
revision: str | None = "main",
) -> dict[str, Any] | None:
""" """
This function gets the pooling and normalize This function gets the pooling and normalize
config from the model - only applies to config from the model - only applies to
@@ -793,38 +794,40 @@ def get_pooling_config(model: str, revision: str | None = "main") -> dict | None
) )
if pooling: if pooling:
pooling_file_name = "{}/config.json".format(pooling["path"]) from vllm.config.pooler import SEQ_POOLING_TYPES, TOK_POOLING_TYPES
pooling_dict = get_hf_file_to_dict(pooling_file_name, model, revision)
pooling_type_name = next(
(item for item, val in pooling_dict.items() if val is True), None
)
if pooling_type_name is not None: pooling_file_name = "{}/config.json".format(pooling["path"])
pooling_type_name = get_pooling_config_name(pooling_type_name) pooling_dict = get_hf_file_to_dict(pooling_file_name, model, revision) or {}
logger.info("Found pooling configuration.") logger.info("Found pooling configuration.")
return {"pooling_type": pooling_type_name, "normalize": normalize}
config: dict[str, Any] = {"normalize": normalize}
for key, val in pooling_dict.items():
if val is True:
pooling_type = parse_pooling_type(key)
if pooling_type in SEQ_POOLING_TYPES:
config["seq_pooling_type"] = pooling_type
elif pooling_type in TOK_POOLING_TYPES:
config["tok_pooling_type"] = pooling_type
else:
logger.debug("Skipping unrelated field: %r=%r", key, val)
return config
return None return None
def get_pooling_config_name(pooling_name: str) -> str | None: def parse_pooling_type(pooling_name: str):
if "pooling_mode_" in pooling_name: if "pooling_mode_" in pooling_name:
pooling_name = pooling_name.replace("pooling_mode_", "") pooling_name = pooling_name.replace("pooling_mode_", "")
if "_" in pooling_name: if "_" in pooling_name:
pooling_name = pooling_name.split("_")[0] pooling_name = pooling_name.split("_", 1)[0]
if "lasttoken" in pooling_name: if "lasttoken" in pooling_name:
pooling_name = "last" pooling_name = "last"
supported_pooling_types = ["LAST", "ALL", "CLS", "STEP", "MEAN"] return pooling_name.upper()
pooling_type_name = pooling_name.upper()
if pooling_type_name in supported_pooling_types:
return pooling_type_name
raise NotImplementedError(f"Pooling type {pooling_type_name} not supported")
@cache @cache