[Bugfix] Support Sentence Transformers 5.4+ serialized configs (#54533)

Signed-off-by: maireneu <maireneu@gmail.com>
Co-authored-by: Codex <codex@openai.com>
This commit is contained in:
Eli Nabro Yeongmin Oh
2026-09-01 15:53:38 +00:00
committed by GitHub
co-authored by Codex
parent f1e5fdd7f2
commit cdefd9d499
3 changed files with 150 additions and 21 deletions
+3 -3
View File
@@ -292,12 +292,12 @@ classification activation is applied.
| Field | Source precedence | How to override |
| ----- | ----------------- | --------------- |
| Pooling method (`pooling_type`) | `--pooler-config` > boolean `pooling_mode_*` fields in the Pooling module referenced by Sentence Transformers `modules.json` > architecture default (`LAST` for sequence pooling and `ALL` for token pooling unless the architecture overrides it) | Set `{"pooling_type": "CLS"}`, or set `seq_pooling_type` / `tok_pooling_type` explicitly. |
| Pooling method (`pooling_type`) | `--pooler-config` > compact `pooling_mode` or legacy boolean `pooling_mode_*` fields in the Pooling module referenced by Sentence Transformers `modules.json` > architecture default (`LAST` for sequence pooling and `ALL` for token pooling unless the architecture overrides it) | Set `{"pooling_type": "CLS"}`, or set `seq_pooling_type` / `tok_pooling_type` explicitly. |
| Embedding normalization (`use_activation`) | `--pooler-config` > Sentence Transformers modules (`true` when a Normalize module is present, otherwise `false`) > pooling-task default (`true`) when no Sentence Transformers Pooling module is found | Set `{"use_activation": false}` to return unnormalized embeddings. |
| Classification activation function | Hugging Face `problem_type` > Sentence Transformers activation metadata > sigmoid or softmax selected from the label count | The function cannot be selected through `--pooler-config`; set `{"use_activation": false}` to return logits instead. |
Sentence Transformers configurations using the newer compact `pooling_mode`
string are not currently parsed; see [issue #45995](https://github.com/vllm-project/vllm/issues/45995).
Both compact `pooling_mode` values written by Sentence Transformers 5.4+ and
legacy boolean `pooling_mode_*` fields are supported.
For converted models and predefined models using the standard DispatchPooler
adapters, `embed` and `token_embed` construct an L2-normalization head, while
+108
View File
@@ -1,9 +1,11 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import json
import logging
import os
from dataclasses import MISSING, Field, asdict, dataclass, field
from pathlib import Path
from types import SimpleNamespace
from typing import cast
from unittest.mock import patch
@@ -39,11 +41,20 @@ from vllm.config.speculative import _validate_qwen3_omni_dspark
from vllm.config.utils import get_field
from vllm.config.vllm import OPTIMIZATION_LEVEL_TO_CONFIG, OptimizationLevel
from vllm.platforms import current_platform
from vllm.transformers_utils.config import (
get_pooling_config,
try_get_dense_modules,
)
from vllm.v1.attention.backend import AttentionCGSupport
DEVICE_TYPE = current_platform.device_type
def _write_json(path: Path, value: object) -> None:
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(json.dumps(value), encoding="utf-8")
def test_kda_recoverssm_derivation_is_revalidated():
config = SimpleNamespace(
cache_config=SimpleNamespace(
@@ -1031,6 +1042,103 @@ def test_get_pooling_config():
assert model_config.pooler_config.tok_pooling_type == "ALL"
@pytest.mark.parametrize(
("pooling_module_type", "normalize_module_type", "pooling_config", "expected"),
[
(
"sentence_transformers.models.Pooling",
"sentence_transformers.models.Normalize",
{"pooling_mode_mean_tokens": True},
{"use_activation": True, "seq_pooling_type": "MEAN"},
),
(
"sentence_transformers.sentence_transformer.modules.pooling.Pooling",
"sentence_transformers.sentence_transformer.modules.normalize.Normalize",
{"pooling_mode": "lasttoken"},
{"use_activation": True, "seq_pooling_type": "LAST"},
),
(
"sentence_transformers.sentence_transformer.modules.pooling.Pooling",
"sentence_transformers.base.modules.normalize.Normalize",
{"pooling_mode": "mean"},
{"use_activation": True, "seq_pooling_type": "MEAN"},
),
],
)
def test_get_pooling_config_supports_sentence_transformers_schemas(
tmp_path,
caplog,
pooling_module_type,
normalize_module_type,
pooling_config,
expected,
):
modules = [
{"idx": 1, "name": "1", "path": "1_Pooling", "type": pooling_module_type},
{
"idx": 2,
"name": "2",
"path": "2_Normalize",
"type": normalize_module_type,
},
]
_write_json(tmp_path / "modules.json", modules)
_write_json(tmp_path / "1_Pooling" / "config.json", pooling_config)
with caplog.at_level(logging.WARNING):
config = get_pooling_config(str(tmp_path))
assert config == expected
assert "Unable to determine Sentence Transformers pooling type" not in caplog.text
def test_get_pooling_config_warns_when_pooling_mode_is_unknown(tmp_path, caplog):
modules = [
{
"idx": 1,
"name": "1",
"path": "1_Pooling",
"type": "sentence_transformers.models.Pooling",
}
]
_write_json(tmp_path / "modules.json", modules)
_write_json(
tmp_path / "1_Pooling" / "config.json",
{"pooling_mode": "unsupported"},
)
with caplog.at_level(logging.WARNING):
config = get_pooling_config(str(tmp_path))
assert config == {"use_activation": False}
assert "Unable to determine Sentence Transformers pooling type" in caplog.text
@pytest.mark.parametrize(
"dense_module_type",
[
"sentence_transformers.models.Dense",
"sentence_transformers.base.modules.dense.Dense",
],
)
def test_get_dense_modules_supports_sentence_transformers_schemas(
tmp_path, dense_module_type
):
modules = [{"idx": 2, "name": "2", "path": "2_Dense", "type": dense_module_type}]
dense_config = {
"in_features": 768,
"out_features": 3072,
"bias": False,
"activation_function": "torch.nn.modules.linear.Identity",
}
_write_json(tmp_path / "modules.json", modules)
_write_json(tmp_path / "2_Dense" / "config.json", dense_config)
assert try_get_dense_modules(str(tmp_path)) == [
{**dense_config, "folder": "2_Dense"}
]
@pytest.mark.skipif(
current_platform.is_rocm(), reason="Xformers backend is not supported on ROCm."
)
+39 -18
View File
@@ -52,6 +52,21 @@ MISTRAL_CONFIG_NAME = "params.json"
logger = init_logger(__name__)
_ST_POOLING_MODULE_TYPES = {
"sentence_transformers.models.Pooling",
"sentence_transformers.sentence_transformer.modules.pooling.Pooling",
}
_ST_NORMALIZE_MODULE_TYPES = {
"sentence_transformers.models.Normalize",
"sentence_transformers.base.modules.normalize.Normalize",
"sentence_transformers.sentence_transformer.modules.normalize.Normalize",
}
_DENSE_MODULE_TYPES = {
"sentence_transformers.models.Dense",
"sentence_transformers.base.modules.dense.Dense",
"pylate.models.Dense.Dense",
}
if Version(version("transformers")) < Version("5.0.0"):
raise ImportError(
"Support for Transformers v4 is deprecated and was removed in vLLM v0.24.0. "
@@ -852,11 +867,7 @@ def get_pooling_config(
logger.info("Found sentence-transformers modules configuration.")
pooling = next(
(
item
for item in modules_dict
if item["type"] == "sentence_transformers.models.Pooling"
),
(item for item in modules_dict if item["type"] in _ST_POOLING_MODULE_TYPES),
None,
)
normalize = bool(
@@ -864,7 +875,7 @@ def get_pooling_config(
(
item
for item in modules_dict
if item["type"] == "sentence_transformers.models.Normalize"
if item["type"] in _ST_NORMALIZE_MODULE_TYPES
),
False,
)
@@ -880,14 +891,28 @@ def get_pooling_config(
config: dict[str, Any] = {"use_activation": normalize}
for key, val in pooling_dict.items():
if val is True:
pooling_type = parse_pooling_type(key)
if pooling_type in SEQ_POOLING_TYPES:
config["seq_pooling_type"] = pooling_type
elif pooling_type in TOK_POOLING_TYPES:
config["tok_pooling_type"] = pooling_type
else:
logger.debug("Skipping unrelated field: %r=%r", key, val)
if key == "pooling_mode" and isinstance(val, str):
pooling_name = val
elif val is True:
pooling_name = key
else:
continue
pooling_type = parse_pooling_type(pooling_name)
if pooling_type in SEQ_POOLING_TYPES:
config["seq_pooling_type"] = pooling_type
elif pooling_type in TOK_POOLING_TYPES:
config["tok_pooling_type"] = pooling_type
else:
logger.debug("Skipping unrelated field: %r=%r", key, val)
if not {"seq_pooling_type", "tok_pooling_type"} & config.keys():
logger.warning(
"Unable to determine Sentence Transformers pooling type from %s; "
"unless configured explicitly, vLLM will fall back to the model "
"architecture default.",
pooling_file_name,
)
return config
@@ -1151,10 +1176,6 @@ def try_get_dense_modules(
if isinstance(modules, dict):
modules = modules.get("modules", [])
_DENSE_MODULE_TYPES = {
"sentence_transformers.models.Dense",
"pylate.models.Dense.Dense",
}
dense_modules = [m for m in modules if m.get("type") in _DENSE_MODULE_TYPES]
if not dense_modules:
return None