fix(pooling): validate BGE-M3 combined task ownership (#51823)
Signed-off-by: Zhe Li <2843409461@qq.com> Co-authored-by: OpenAI Codex <noreply@openai.com>
This commit is contained in:
@@ -365,36 +365,99 @@ curl -s http://localhost:8000/rerank -H "Content-Type: application/json" -d '{
|
||||
|
||||
## BAAI/bge-m3
|
||||
|
||||
The `BAAI/bge-m3` model comes with extra weights for sparse and colbert embeddings but unfortunately in its `config.json`
|
||||
the architecture is declared as `XLMRobertaModel`, which makes `vLLM` load it as a vanilla ROBERTA model without the
|
||||
extra weights. To load the full model weights, override its architecture like this:
|
||||
`BAAI/bge-m3` supports dense retrieval, lexical matching, and ColBERT-style
|
||||
multi-vector retrieval. Its `config.json` declares `XLMRobertaModel`, so vLLM
|
||||
otherwise loads it as a vanilla RoBERTa model without the extra sparse and
|
||||
ColBERT weights. The examples below therefore override the architecture with
|
||||
`BgeM3EmbeddingModel`.
|
||||
|
||||
The three retrieval modes map to concrete pooling tasks as follows:
|
||||
|
||||
| Retrieval mode | Pooling task | Output |
|
||||
| -------------- | ------------ | ------ |
|
||||
| Dense | `embed` | One embedding vector per input |
|
||||
| Lexical/sparse | `token_classify` | One scalar weight per non-special token |
|
||||
| ColBERT multi-vector | `token_embed` | One embedding vector per non-special token |
|
||||
|
||||
Serve one concrete mode by selecting its task at load time:
|
||||
|
||||
```shell
|
||||
vllm serve BAAI/bge-m3 --hf-overrides '{"architectures": ["BgeM3EmbeddingModel"]}'
|
||||
vllm serve BAAI/bge-m3 \
|
||||
--runner pooling \
|
||||
--hf-overrides '{"architectures": ["BgeM3EmbeddingModel"]}' \
|
||||
--pooler-config.task <task>
|
||||
```
|
||||
|
||||
Then you obtain the sparse embeddings like this:
|
||||
For dense embeddings, replace `<task>` with `embed` and use the Embeddings API:
|
||||
|
||||
```shell
|
||||
curl -s http://localhost:8000/v1/embeddings \
|
||||
-H "Content-Type: application/json" -d '{
|
||||
"model": "BAAI/bge-m3",
|
||||
"input": ["What is BGE M3?", "Definition of BM25"]
|
||||
}'
|
||||
```
|
||||
|
||||
For lexical weights, replace `<task>` with `token_classify` and use the
|
||||
Pooling API:
|
||||
|
||||
```shell
|
||||
curl -s http://localhost:8000/pooling -H "Content-Type: application/json" -d '{
|
||||
"model": "BAAI/bge-m3",
|
||||
"task": "token_classify",
|
||||
"input": ["What is BGE M3?", "Definition of BM25"]
|
||||
"model": "BAAI/bge-m3",
|
||||
"task": "token_classify",
|
||||
"input": ["What is BGE M3?", "Definition of BM25"]
|
||||
}'
|
||||
```
|
||||
|
||||
Due to limitations in the output schema, the output consists of a list of
|
||||
token scores for each token for each input. This means that you'll have to call
|
||||
`/tokenize` as well to be able to pair tokens with scores.
|
||||
Refer to the tests in `tests/models/language/pooling/test_bge_m3.py` to see how
|
||||
to do that.
|
||||
token scores for each input. Call `/tokenize` as well to pair token IDs with
|
||||
their scores. See
|
||||
[`test_bge_m3.py`](../../../tests/models/language/pooling/test_bge_m3.py) for a
|
||||
complete example that also combines repeated token IDs.
|
||||
|
||||
You can obtain the colbert embeddings like this:
|
||||
For ColBERT vectors, replace `<task>` with `token_embed` and use the Pooling API:
|
||||
|
||||
```shell
|
||||
curl -s http://localhost:8000/pooling -H "Content-Type: application/json" -d '{
|
||||
"model": "BAAI/bge-m3",
|
||||
"task": "token_embed",
|
||||
"input": ["What is BGE M3?", "Definition of BM25"]
|
||||
"model": "BAAI/bge-m3",
|
||||
"task": "token_embed",
|
||||
"input": ["What is BGE M3?", "Definition of BM25"]
|
||||
}'
|
||||
```
|
||||
|
||||
### Dense and sparse output through an IO processor plugin
|
||||
|
||||
The source tree includes a reference
|
||||
[BGE-M3 IO processor plugin](../../../tests/plugins/bge_m3_sparse_plugin) that
|
||||
formats dense embeddings, sparse token weights, or both in one response. From
|
||||
a source checkout, install it in the vLLM environment and load it as follows:
|
||||
|
||||
```shell
|
||||
uv pip install ./tests/plugins/bge_m3_sparse_plugin
|
||||
|
||||
vllm serve BAAI/bge-m3 \
|
||||
--runner pooling \
|
||||
--hf-overrides '{"architectures": ["BgeM3EmbeddingModel"]}' \
|
||||
--io-processor-plugin bge_m3_sparse_plugin
|
||||
```
|
||||
|
||||
The plugin selects the internal `embed&token_classify` task so the model
|
||||
computes dense and lexical outputs together. Public requests must use task
|
||||
`plugin` and put the plugin-specific fields under `data`:
|
||||
|
||||
```shell
|
||||
curl -s http://localhost:8000/pooling -H "Content-Type: application/json" -d '{
|
||||
"model": "BAAI/bge-m3",
|
||||
"task": "plugin",
|
||||
"data": {
|
||||
"input": ["What is BGE M3?", "Definition of BM25"],
|
||||
"embed_task": "dense&sparse",
|
||||
"return_tokens": true
|
||||
}
|
||||
}'
|
||||
```
|
||||
|
||||
`embed_task` accepts `dense`, `sparse`, or `dense&sparse`. The combined
|
||||
`embed&token_classify` task is an internal execution contract for this plugin,
|
||||
not a generic Pooling API response format. Without the plugin, select one of
|
||||
the three concrete tasks above.
|
||||
|
||||
@@ -0,0 +1,94 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm.entrypoints.pooling import factories
|
||||
from vllm.entrypoints.pooling.factories import init_pooling_io_processors
|
||||
from vllm.entrypoints.pooling.pooling.io_processor import (
|
||||
PluginWithIOProcessorPlugins,
|
||||
UnsupportedCombinedTaskIOProcessor,
|
||||
)
|
||||
|
||||
|
||||
def _bge_m3_config(io_processor_plugin=None):
|
||||
model_config = MagicMock()
|
||||
model_config.get_pooling_task.return_value = "embed&token_classify"
|
||||
model_config.io_processor_plugin = io_processor_plugin
|
||||
model_config.hf_config.to_dict.return_value = {}
|
||||
model_config.architecture = "BgeM3EmbeddingModel"
|
||||
|
||||
vllm_config = MagicMock(model_config=model_config)
|
||||
renderer = MagicMock()
|
||||
renderer._executor = MagicMock()
|
||||
chat_template_config = MagicMock(
|
||||
chat_template=None,
|
||||
chat_template_content_format="auto",
|
||||
trust_request_chat_template=False,
|
||||
)
|
||||
return vllm_config, renderer, chat_template_config
|
||||
|
||||
|
||||
def test_combined_task_without_plugin_uses_rejection_processor():
|
||||
vllm_config, renderer, chat_template_config = _bge_m3_config()
|
||||
|
||||
processors = init_pooling_io_processors(
|
||||
supported_tasks=("embed", "embed&token_classify"),
|
||||
vllm_config=vllm_config,
|
||||
renderer=renderer,
|
||||
chat_template_config=chat_template_config,
|
||||
)
|
||||
|
||||
assert processors.keys() == {"embed&token_classify"}
|
||||
assert isinstance(
|
||||
processors["embed&token_classify"], UnsupportedCombinedTaskIOProcessor
|
||||
)
|
||||
|
||||
|
||||
def test_combined_task_with_plugin_uses_plugin_processor(monkeypatch):
|
||||
vllm_config, renderer, chat_template_config = _bge_m3_config("bge_m3_sparse_plugin")
|
||||
monkeypatch.setattr(factories, "has_io_processor", lambda *_: True)
|
||||
monkeypatch.setattr(
|
||||
"vllm.entrypoints.pooling.pooling.io_processor.get_io_processor",
|
||||
lambda *_: MagicMock(),
|
||||
)
|
||||
|
||||
processors = init_pooling_io_processors(
|
||||
supported_tasks=("embed", "embed&token_classify"),
|
||||
vllm_config=vllm_config,
|
||||
renderer=renderer,
|
||||
chat_template_config=chat_template_config,
|
||||
)
|
||||
|
||||
assert processors.keys() == {"embed&token_classify", "plugin"}
|
||||
assert isinstance(processors["plugin"], PluginWithIOProcessorPlugins)
|
||||
|
||||
|
||||
def test_combined_task_plain_pooling_request_has_actionable_error(monkeypatch):
|
||||
from vllm.entrypoints.pooling.pooling.protocol import PoolingCompletionRequest
|
||||
from vllm.entrypoints.pooling.pooling.serving import ServingPooling
|
||||
|
||||
vllm_config, renderer, chat_template_config = _bge_m3_config("bge_m3_sparse_plugin")
|
||||
monkeypatch.setattr(factories, "has_io_processor", lambda *_: True)
|
||||
monkeypatch.setattr(
|
||||
"vllm.entrypoints.pooling.pooling.io_processor.get_io_processor",
|
||||
lambda *_: MagicMock(),
|
||||
)
|
||||
|
||||
engine_client = MagicMock(renderer=renderer, vllm_config=vllm_config)
|
||||
models = MagicMock(model_config=vllm_config.model_config)
|
||||
serving = ServingPooling(
|
||||
engine_client,
|
||||
models,
|
||||
supported_tasks=("embed", "embed&token_classify"),
|
||||
request_logger=None,
|
||||
chat_template_config=chat_template_config,
|
||||
)
|
||||
request = PoolingCompletionRequest(model="BAAI/bge-m3", input=["hola"])
|
||||
|
||||
assert serving.io_processors.keys() == {"embed&token_classify", "plugin"}
|
||||
io_processor = serving.get_io_processor(request)
|
||||
with pytest.raises(ValueError, match="plugin request with a 'data' field"):
|
||||
io_processor.create_pooling_params(request)
|
||||
@@ -65,10 +65,16 @@ def init_pooling_io_processors(
|
||||
|
||||
processors["token_embed"] = TokenEmbedIOProcessor
|
||||
|
||||
if has_io_processor(
|
||||
if pooling_task == "embed&token_classify":
|
||||
from .pooling.io_processor import UnsupportedCombinedTaskIOProcessor
|
||||
|
||||
processors[pooling_task] = UnsupportedCombinedTaskIOProcessor
|
||||
|
||||
has_plugin = has_io_processor(
|
||||
vllm_config,
|
||||
model_config.io_processor_plugin,
|
||||
):
|
||||
)
|
||||
if has_plugin:
|
||||
from .pooling.io_processor import PluginWithIOProcessorPlugins
|
||||
|
||||
processors["plugin"] = PluginWithIOProcessorPlugins
|
||||
|
||||
@@ -24,6 +24,18 @@ from .protocol import IOProcessorRequest, IOProcessorResponse
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
class UnsupportedCombinedTaskIOProcessor(PoolingIOProcessor):
|
||||
name = "embed&token_classify"
|
||||
|
||||
def create_pooling_params(self, request):
|
||||
raise ValueError(
|
||||
"The 'embed&token_classify' pooling task is only available "
|
||||
"through an IO processor plugin. Send a plugin request with "
|
||||
"a 'data' field, "
|
||||
"or select a concrete task with --pooler-config.task."
|
||||
)
|
||||
|
||||
|
||||
class PluginWithoutIOProcessorPlugins(PoolingIOProcessor):
|
||||
# Some models, such as Terratorch (tests/models/test_terratorch.py),
|
||||
# use plugin tasks in the pooler but do not use IO Processor plugins.
|
||||
|
||||
Reference in New Issue
Block a user