From cdefd9d4997f00da72dc6245cc60678b50761b7e Mon Sep 17 00:00:00 2001 From: Eli Nabro Yeongmin Oh <43892971+maireneu@users.noreply.github.com> Date: Wed, 2 Sep 2026 00:53:38 +0900 Subject: [PATCH] [Bugfix] Support Sentence Transformers 5.4+ serialized configs (#54533) Signed-off-by: maireneu Co-authored-by: Codex --- docs/models/pooling_models/README.md | 6 +- tests/test_config.py | 108 +++++++++++++++++++++++++++ vllm/transformers_utils/config.py | 57 +++++++++----- 3 files changed, 150 insertions(+), 21 deletions(-) diff --git a/docs/models/pooling_models/README.md b/docs/models/pooling_models/README.md index 4bad45d7a8..12f27f0bd8 100644 --- a/docs/models/pooling_models/README.md +++ b/docs/models/pooling_models/README.md @@ -292,12 +292,12 @@ classification activation is applied. | Field | Source precedence | How to override | | ----- | ----------------- | --------------- | -| Pooling method (`pooling_type`) | `--pooler-config` > boolean `pooling_mode_*` fields in the Pooling module referenced by Sentence Transformers `modules.json` > architecture default (`LAST` for sequence pooling and `ALL` for token pooling unless the architecture overrides it) | Set `{"pooling_type": "CLS"}`, or set `seq_pooling_type` / `tok_pooling_type` explicitly. | +| Pooling method (`pooling_type`) | `--pooler-config` > compact `pooling_mode` or legacy boolean `pooling_mode_*` fields in the Pooling module referenced by Sentence Transformers `modules.json` > architecture default (`LAST` for sequence pooling and `ALL` for token pooling unless the architecture overrides it) | Set `{"pooling_type": "CLS"}`, or set `seq_pooling_type` / `tok_pooling_type` explicitly. | | Embedding normalization (`use_activation`) | `--pooler-config` > Sentence Transformers modules (`true` when a Normalize module is present, otherwise `false`) > pooling-task default (`true`) when no Sentence Transformers Pooling module is found | Set `{"use_activation": false}` to return unnormalized embeddings. | | Classification activation function | Hugging Face `problem_type` > Sentence Transformers activation metadata > sigmoid or softmax selected from the label count | The function cannot be selected through `--pooler-config`; set `{"use_activation": false}` to return logits instead. | -Sentence Transformers configurations using the newer compact `pooling_mode` -string are not currently parsed; see [issue #45995](https://github.com/vllm-project/vllm/issues/45995). +Both compact `pooling_mode` values written by Sentence Transformers 5.4+ and +legacy boolean `pooling_mode_*` fields are supported. For converted models and predefined models using the standard DispatchPooler adapters, `embed` and `token_embed` construct an L2-normalization head, while diff --git a/tests/test_config.py b/tests/test_config.py index aa0c838e3f..ea688f7d8c 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -1,9 +1,11 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project +import json import logging import os from dataclasses import MISSING, Field, asdict, dataclass, field +from pathlib import Path from types import SimpleNamespace from typing import cast from unittest.mock import patch @@ -39,11 +41,20 @@ from vllm.config.speculative import _validate_qwen3_omni_dspark from vllm.config.utils import get_field from vllm.config.vllm import OPTIMIZATION_LEVEL_TO_CONFIG, OptimizationLevel from vllm.platforms import current_platform +from vllm.transformers_utils.config import ( + get_pooling_config, + try_get_dense_modules, +) from vllm.v1.attention.backend import AttentionCGSupport DEVICE_TYPE = current_platform.device_type +def _write_json(path: Path, value: object) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(value), encoding="utf-8") + + def test_kda_recoverssm_derivation_is_revalidated(): config = SimpleNamespace( cache_config=SimpleNamespace( @@ -1031,6 +1042,103 @@ def test_get_pooling_config(): assert model_config.pooler_config.tok_pooling_type == "ALL" +@pytest.mark.parametrize( + ("pooling_module_type", "normalize_module_type", "pooling_config", "expected"), + [ + ( + "sentence_transformers.models.Pooling", + "sentence_transformers.models.Normalize", + {"pooling_mode_mean_tokens": True}, + {"use_activation": True, "seq_pooling_type": "MEAN"}, + ), + ( + "sentence_transformers.sentence_transformer.modules.pooling.Pooling", + "sentence_transformers.sentence_transformer.modules.normalize.Normalize", + {"pooling_mode": "lasttoken"}, + {"use_activation": True, "seq_pooling_type": "LAST"}, + ), + ( + "sentence_transformers.sentence_transformer.modules.pooling.Pooling", + "sentence_transformers.base.modules.normalize.Normalize", + {"pooling_mode": "mean"}, + {"use_activation": True, "seq_pooling_type": "MEAN"}, + ), + ], +) +def test_get_pooling_config_supports_sentence_transformers_schemas( + tmp_path, + caplog, + pooling_module_type, + normalize_module_type, + pooling_config, + expected, +): + modules = [ + {"idx": 1, "name": "1", "path": "1_Pooling", "type": pooling_module_type}, + { + "idx": 2, + "name": "2", + "path": "2_Normalize", + "type": normalize_module_type, + }, + ] + _write_json(tmp_path / "modules.json", modules) + _write_json(tmp_path / "1_Pooling" / "config.json", pooling_config) + + with caplog.at_level(logging.WARNING): + config = get_pooling_config(str(tmp_path)) + + assert config == expected + assert "Unable to determine Sentence Transformers pooling type" not in caplog.text + + +def test_get_pooling_config_warns_when_pooling_mode_is_unknown(tmp_path, caplog): + modules = [ + { + "idx": 1, + "name": "1", + "path": "1_Pooling", + "type": "sentence_transformers.models.Pooling", + } + ] + _write_json(tmp_path / "modules.json", modules) + _write_json( + tmp_path / "1_Pooling" / "config.json", + {"pooling_mode": "unsupported"}, + ) + + with caplog.at_level(logging.WARNING): + config = get_pooling_config(str(tmp_path)) + + assert config == {"use_activation": False} + assert "Unable to determine Sentence Transformers pooling type" in caplog.text + + +@pytest.mark.parametrize( + "dense_module_type", + [ + "sentence_transformers.models.Dense", + "sentence_transformers.base.modules.dense.Dense", + ], +) +def test_get_dense_modules_supports_sentence_transformers_schemas( + tmp_path, dense_module_type +): + modules = [{"idx": 2, "name": "2", "path": "2_Dense", "type": dense_module_type}] + dense_config = { + "in_features": 768, + "out_features": 3072, + "bias": False, + "activation_function": "torch.nn.modules.linear.Identity", + } + _write_json(tmp_path / "modules.json", modules) + _write_json(tmp_path / "2_Dense" / "config.json", dense_config) + + assert try_get_dense_modules(str(tmp_path)) == [ + {**dense_config, "folder": "2_Dense"} + ] + + @pytest.mark.skipif( current_platform.is_rocm(), reason="Xformers backend is not supported on ROCm." ) diff --git a/vllm/transformers_utils/config.py b/vllm/transformers_utils/config.py index b5b0ccd8f0..dcbc0ef9f7 100644 --- a/vllm/transformers_utils/config.py +++ b/vllm/transformers_utils/config.py @@ -52,6 +52,21 @@ MISTRAL_CONFIG_NAME = "params.json" logger = init_logger(__name__) +_ST_POOLING_MODULE_TYPES = { + "sentence_transformers.models.Pooling", + "sentence_transformers.sentence_transformer.modules.pooling.Pooling", +} +_ST_NORMALIZE_MODULE_TYPES = { + "sentence_transformers.models.Normalize", + "sentence_transformers.base.modules.normalize.Normalize", + "sentence_transformers.sentence_transformer.modules.normalize.Normalize", +} +_DENSE_MODULE_TYPES = { + "sentence_transformers.models.Dense", + "sentence_transformers.base.modules.dense.Dense", + "pylate.models.Dense.Dense", +} + if Version(version("transformers")) < Version("5.0.0"): raise ImportError( "Support for Transformers v4 is deprecated and was removed in vLLM v0.24.0. " @@ -852,11 +867,7 @@ def get_pooling_config( logger.info("Found sentence-transformers modules configuration.") pooling = next( - ( - item - for item in modules_dict - if item["type"] == "sentence_transformers.models.Pooling" - ), + (item for item in modules_dict if item["type"] in _ST_POOLING_MODULE_TYPES), None, ) normalize = bool( @@ -864,7 +875,7 @@ def get_pooling_config( ( item for item in modules_dict - if item["type"] == "sentence_transformers.models.Normalize" + if item["type"] in _ST_NORMALIZE_MODULE_TYPES ), False, ) @@ -880,14 +891,28 @@ def get_pooling_config( config: dict[str, Any] = {"use_activation": normalize} for key, val in pooling_dict.items(): - if val is True: - pooling_type = parse_pooling_type(key) - if pooling_type in SEQ_POOLING_TYPES: - config["seq_pooling_type"] = pooling_type - elif pooling_type in TOK_POOLING_TYPES: - config["tok_pooling_type"] = pooling_type - else: - logger.debug("Skipping unrelated field: %r=%r", key, val) + if key == "pooling_mode" and isinstance(val, str): + pooling_name = val + elif val is True: + pooling_name = key + else: + continue + + pooling_type = parse_pooling_type(pooling_name) + if pooling_type in SEQ_POOLING_TYPES: + config["seq_pooling_type"] = pooling_type + elif pooling_type in TOK_POOLING_TYPES: + config["tok_pooling_type"] = pooling_type + else: + logger.debug("Skipping unrelated field: %r=%r", key, val) + + if not {"seq_pooling_type", "tok_pooling_type"} & config.keys(): + logger.warning( + "Unable to determine Sentence Transformers pooling type from %s; " + "unless configured explicitly, vLLM will fall back to the model " + "architecture default.", + pooling_file_name, + ) return config @@ -1151,10 +1176,6 @@ def try_get_dense_modules( if isinstance(modules, dict): modules = modules.get("modules", []) - _DENSE_MODULE_TYPES = { - "sentence_transformers.models.Dense", - "pylate.models.Dense.Dense", - } dense_modules = [m for m in modules if m.get("type") in _DENSE_MODULE_TYPES] if not dense_modules: return None