[Bugfix] Support Sentence Transformers 5.4+ serialized configs (#54533)
Signed-off-by: maireneu <maireneu@gmail.com> Co-authored-by: Codex <codex@openai.com>
This commit is contained in:
co-authored by
Codex
parent
f1e5fdd7f2
commit
cdefd9d499
@@ -292,12 +292,12 @@ classification activation is applied.
|
||||
|
||||
| Field | Source precedence | How to override |
|
||||
| ----- | ----------------- | --------------- |
|
||||
| Pooling method (`pooling_type`) | `--pooler-config` > boolean `pooling_mode_*` fields in the Pooling module referenced by Sentence Transformers `modules.json` > architecture default (`LAST` for sequence pooling and `ALL` for token pooling unless the architecture overrides it) | Set `{"pooling_type": "CLS"}`, or set `seq_pooling_type` / `tok_pooling_type` explicitly. |
|
||||
| Pooling method (`pooling_type`) | `--pooler-config` > compact `pooling_mode` or legacy boolean `pooling_mode_*` fields in the Pooling module referenced by Sentence Transformers `modules.json` > architecture default (`LAST` for sequence pooling and `ALL` for token pooling unless the architecture overrides it) | Set `{"pooling_type": "CLS"}`, or set `seq_pooling_type` / `tok_pooling_type` explicitly. |
|
||||
| Embedding normalization (`use_activation`) | `--pooler-config` > Sentence Transformers modules (`true` when a Normalize module is present, otherwise `false`) > pooling-task default (`true`) when no Sentence Transformers Pooling module is found | Set `{"use_activation": false}` to return unnormalized embeddings. |
|
||||
| Classification activation function | Hugging Face `problem_type` > Sentence Transformers activation metadata > sigmoid or softmax selected from the label count | The function cannot be selected through `--pooler-config`; set `{"use_activation": false}` to return logits instead. |
|
||||
|
||||
Sentence Transformers configurations using the newer compact `pooling_mode`
|
||||
string are not currently parsed; see [issue #45995](https://github.com/vllm-project/vllm/issues/45995).
|
||||
Both compact `pooling_mode` values written by Sentence Transformers 5.4+ and
|
||||
legacy boolean `pooling_mode_*` fields are supported.
|
||||
|
||||
For converted models and predefined models using the standard DispatchPooler
|
||||
adapters, `embed` and `token_embed` construct an L2-normalization head, while
|
||||
|
||||
@@ -1,9 +1,11 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
from dataclasses import MISSING, Field, asdict, dataclass, field
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
from typing import cast
|
||||
from unittest.mock import patch
|
||||
@@ -39,11 +41,20 @@ from vllm.config.speculative import _validate_qwen3_omni_dspark
|
||||
from vllm.config.utils import get_field
|
||||
from vllm.config.vllm import OPTIMIZATION_LEVEL_TO_CONFIG, OptimizationLevel
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.transformers_utils.config import (
|
||||
get_pooling_config,
|
||||
try_get_dense_modules,
|
||||
)
|
||||
from vllm.v1.attention.backend import AttentionCGSupport
|
||||
|
||||
DEVICE_TYPE = current_platform.device_type
|
||||
|
||||
|
||||
def _write_json(path: Path, value: object) -> None:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text(json.dumps(value), encoding="utf-8")
|
||||
|
||||
|
||||
def test_kda_recoverssm_derivation_is_revalidated():
|
||||
config = SimpleNamespace(
|
||||
cache_config=SimpleNamespace(
|
||||
@@ -1031,6 +1042,103 @@ def test_get_pooling_config():
|
||||
assert model_config.pooler_config.tok_pooling_type == "ALL"
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("pooling_module_type", "normalize_module_type", "pooling_config", "expected"),
|
||||
[
|
||||
(
|
||||
"sentence_transformers.models.Pooling",
|
||||
"sentence_transformers.models.Normalize",
|
||||
{"pooling_mode_mean_tokens": True},
|
||||
{"use_activation": True, "seq_pooling_type": "MEAN"},
|
||||
),
|
||||
(
|
||||
"sentence_transformers.sentence_transformer.modules.pooling.Pooling",
|
||||
"sentence_transformers.sentence_transformer.modules.normalize.Normalize",
|
||||
{"pooling_mode": "lasttoken"},
|
||||
{"use_activation": True, "seq_pooling_type": "LAST"},
|
||||
),
|
||||
(
|
||||
"sentence_transformers.sentence_transformer.modules.pooling.Pooling",
|
||||
"sentence_transformers.base.modules.normalize.Normalize",
|
||||
{"pooling_mode": "mean"},
|
||||
{"use_activation": True, "seq_pooling_type": "MEAN"},
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_get_pooling_config_supports_sentence_transformers_schemas(
|
||||
tmp_path,
|
||||
caplog,
|
||||
pooling_module_type,
|
||||
normalize_module_type,
|
||||
pooling_config,
|
||||
expected,
|
||||
):
|
||||
modules = [
|
||||
{"idx": 1, "name": "1", "path": "1_Pooling", "type": pooling_module_type},
|
||||
{
|
||||
"idx": 2,
|
||||
"name": "2",
|
||||
"path": "2_Normalize",
|
||||
"type": normalize_module_type,
|
||||
},
|
||||
]
|
||||
_write_json(tmp_path / "modules.json", modules)
|
||||
_write_json(tmp_path / "1_Pooling" / "config.json", pooling_config)
|
||||
|
||||
with caplog.at_level(logging.WARNING):
|
||||
config = get_pooling_config(str(tmp_path))
|
||||
|
||||
assert config == expected
|
||||
assert "Unable to determine Sentence Transformers pooling type" not in caplog.text
|
||||
|
||||
|
||||
def test_get_pooling_config_warns_when_pooling_mode_is_unknown(tmp_path, caplog):
|
||||
modules = [
|
||||
{
|
||||
"idx": 1,
|
||||
"name": "1",
|
||||
"path": "1_Pooling",
|
||||
"type": "sentence_transformers.models.Pooling",
|
||||
}
|
||||
]
|
||||
_write_json(tmp_path / "modules.json", modules)
|
||||
_write_json(
|
||||
tmp_path / "1_Pooling" / "config.json",
|
||||
{"pooling_mode": "unsupported"},
|
||||
)
|
||||
|
||||
with caplog.at_level(logging.WARNING):
|
||||
config = get_pooling_config(str(tmp_path))
|
||||
|
||||
assert config == {"use_activation": False}
|
||||
assert "Unable to determine Sentence Transformers pooling type" in caplog.text
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"dense_module_type",
|
||||
[
|
||||
"sentence_transformers.models.Dense",
|
||||
"sentence_transformers.base.modules.dense.Dense",
|
||||
],
|
||||
)
|
||||
def test_get_dense_modules_supports_sentence_transformers_schemas(
|
||||
tmp_path, dense_module_type
|
||||
):
|
||||
modules = [{"idx": 2, "name": "2", "path": "2_Dense", "type": dense_module_type}]
|
||||
dense_config = {
|
||||
"in_features": 768,
|
||||
"out_features": 3072,
|
||||
"bias": False,
|
||||
"activation_function": "torch.nn.modules.linear.Identity",
|
||||
}
|
||||
_write_json(tmp_path / "modules.json", modules)
|
||||
_write_json(tmp_path / "2_Dense" / "config.json", dense_config)
|
||||
|
||||
assert try_get_dense_modules(str(tmp_path)) == [
|
||||
{**dense_config, "folder": "2_Dense"}
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
current_platform.is_rocm(), reason="Xformers backend is not supported on ROCm."
|
||||
)
|
||||
|
||||
@@ -52,6 +52,21 @@ MISTRAL_CONFIG_NAME = "params.json"
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
_ST_POOLING_MODULE_TYPES = {
|
||||
"sentence_transformers.models.Pooling",
|
||||
"sentence_transformers.sentence_transformer.modules.pooling.Pooling",
|
||||
}
|
||||
_ST_NORMALIZE_MODULE_TYPES = {
|
||||
"sentence_transformers.models.Normalize",
|
||||
"sentence_transformers.base.modules.normalize.Normalize",
|
||||
"sentence_transformers.sentence_transformer.modules.normalize.Normalize",
|
||||
}
|
||||
_DENSE_MODULE_TYPES = {
|
||||
"sentence_transformers.models.Dense",
|
||||
"sentence_transformers.base.modules.dense.Dense",
|
||||
"pylate.models.Dense.Dense",
|
||||
}
|
||||
|
||||
if Version(version("transformers")) < Version("5.0.0"):
|
||||
raise ImportError(
|
||||
"Support for Transformers v4 is deprecated and was removed in vLLM v0.24.0. "
|
||||
@@ -852,11 +867,7 @@ def get_pooling_config(
|
||||
logger.info("Found sentence-transformers modules configuration.")
|
||||
|
||||
pooling = next(
|
||||
(
|
||||
item
|
||||
for item in modules_dict
|
||||
if item["type"] == "sentence_transformers.models.Pooling"
|
||||
),
|
||||
(item for item in modules_dict if item["type"] in _ST_POOLING_MODULE_TYPES),
|
||||
None,
|
||||
)
|
||||
normalize = bool(
|
||||
@@ -864,7 +875,7 @@ def get_pooling_config(
|
||||
(
|
||||
item
|
||||
for item in modules_dict
|
||||
if item["type"] == "sentence_transformers.models.Normalize"
|
||||
if item["type"] in _ST_NORMALIZE_MODULE_TYPES
|
||||
),
|
||||
False,
|
||||
)
|
||||
@@ -880,14 +891,28 @@ def get_pooling_config(
|
||||
|
||||
config: dict[str, Any] = {"use_activation": normalize}
|
||||
for key, val in pooling_dict.items():
|
||||
if val is True:
|
||||
pooling_type = parse_pooling_type(key)
|
||||
if pooling_type in SEQ_POOLING_TYPES:
|
||||
config["seq_pooling_type"] = pooling_type
|
||||
elif pooling_type in TOK_POOLING_TYPES:
|
||||
config["tok_pooling_type"] = pooling_type
|
||||
else:
|
||||
logger.debug("Skipping unrelated field: %r=%r", key, val)
|
||||
if key == "pooling_mode" and isinstance(val, str):
|
||||
pooling_name = val
|
||||
elif val is True:
|
||||
pooling_name = key
|
||||
else:
|
||||
continue
|
||||
|
||||
pooling_type = parse_pooling_type(pooling_name)
|
||||
if pooling_type in SEQ_POOLING_TYPES:
|
||||
config["seq_pooling_type"] = pooling_type
|
||||
elif pooling_type in TOK_POOLING_TYPES:
|
||||
config["tok_pooling_type"] = pooling_type
|
||||
else:
|
||||
logger.debug("Skipping unrelated field: %r=%r", key, val)
|
||||
|
||||
if not {"seq_pooling_type", "tok_pooling_type"} & config.keys():
|
||||
logger.warning(
|
||||
"Unable to determine Sentence Transformers pooling type from %s; "
|
||||
"unless configured explicitly, vLLM will fall back to the model "
|
||||
"architecture default.",
|
||||
pooling_file_name,
|
||||
)
|
||||
|
||||
return config
|
||||
|
||||
@@ -1151,10 +1176,6 @@ def try_get_dense_modules(
|
||||
if isinstance(modules, dict):
|
||||
modules = modules.get("modules", [])
|
||||
|
||||
_DENSE_MODULE_TYPES = {
|
||||
"sentence_transformers.models.Dense",
|
||||
"pylate.models.Dense.Dense",
|
||||
}
|
||||
dense_modules = [m for m in modules if m.get("type") in _DENSE_MODULE_TYPES]
|
||||
if not dense_modules:
|
||||
return None
|
||||
|
||||
Reference in New Issue
Block a user