From 95901ce70a625cd152e465d1b15441fd199bde24 Mon Sep 17 00:00:00 2001 From: Ola <114643959+030611@users.noreply.github.com> Date: Mon, 17 Aug 2026 18:30:35 +0800 Subject: [PATCH] fix(pooling): validate BGE-M3 combined task ownership (#51823) Signed-off-by: Zhe Li <2843409461@qq.com> Co-authored-by: OpenAI Codex --- docs/models/pooling_models/specific_models.md | 95 +++++++++++++++---- tests/entrypoints/pooling/test_factories.py | 94 ++++++++++++++++++ vllm/entrypoints/pooling/factories.py | 10 +- .../pooling/pooling/io_processor.py | 12 +++ 4 files changed, 193 insertions(+), 18 deletions(-) create mode 100644 tests/entrypoints/pooling/test_factories.py diff --git a/docs/models/pooling_models/specific_models.md b/docs/models/pooling_models/specific_models.md index 8753f1fd07..0c7b0a1bf3 100644 --- a/docs/models/pooling_models/specific_models.md +++ b/docs/models/pooling_models/specific_models.md @@ -365,36 +365,99 @@ curl -s http://localhost:8000/rerank -H "Content-Type: application/json" -d '{ ## BAAI/bge-m3 -The `BAAI/bge-m3` model comes with extra weights for sparse and colbert embeddings but unfortunately in its `config.json` -the architecture is declared as `XLMRobertaModel`, which makes `vLLM` load it as a vanilla ROBERTA model without the -extra weights. To load the full model weights, override its architecture like this: +`BAAI/bge-m3` supports dense retrieval, lexical matching, and ColBERT-style +multi-vector retrieval. Its `config.json` declares `XLMRobertaModel`, so vLLM +otherwise loads it as a vanilla RoBERTa model without the extra sparse and +ColBERT weights. The examples below therefore override the architecture with +`BgeM3EmbeddingModel`. + +The three retrieval modes map to concrete pooling tasks as follows: + +| Retrieval mode | Pooling task | Output | +| -------------- | ------------ | ------ | +| Dense | `embed` | One embedding vector per input | +| Lexical/sparse | `token_classify` | One scalar weight per non-special token | +| ColBERT multi-vector | `token_embed` | One embedding vector per non-special token | + +Serve one concrete mode by selecting its task at load time: ```shell -vllm serve BAAI/bge-m3 --hf-overrides '{"architectures": ["BgeM3EmbeddingModel"]}' +vllm serve BAAI/bge-m3 \ + --runner pooling \ + --hf-overrides '{"architectures": ["BgeM3EmbeddingModel"]}' \ + --pooler-config.task ``` -Then you obtain the sparse embeddings like this: +For dense embeddings, replace `` with `embed` and use the Embeddings API: + +```shell +curl -s http://localhost:8000/v1/embeddings \ + -H "Content-Type: application/json" -d '{ + "model": "BAAI/bge-m3", + "input": ["What is BGE M3?", "Definition of BM25"] + }' +``` + +For lexical weights, replace `` with `token_classify` and use the +Pooling API: ```shell curl -s http://localhost:8000/pooling -H "Content-Type: application/json" -d '{ - "model": "BAAI/bge-m3", - "task": "token_classify", - "input": ["What is BGE M3?", "Definition of BM25"] + "model": "BAAI/bge-m3", + "task": "token_classify", + "input": ["What is BGE M3?", "Definition of BM25"] }' ``` Due to limitations in the output schema, the output consists of a list of -token scores for each token for each input. This means that you'll have to call -`/tokenize` as well to be able to pair tokens with scores. -Refer to the tests in `tests/models/language/pooling/test_bge_m3.py` to see how -to do that. +token scores for each input. Call `/tokenize` as well to pair token IDs with +their scores. See +[`test_bge_m3.py`](../../../tests/models/language/pooling/test_bge_m3.py) for a +complete example that also combines repeated token IDs. -You can obtain the colbert embeddings like this: +For ColBERT vectors, replace `` with `token_embed` and use the Pooling API: ```shell curl -s http://localhost:8000/pooling -H "Content-Type: application/json" -d '{ - "model": "BAAI/bge-m3", - "task": "token_embed", - "input": ["What is BGE M3?", "Definition of BM25"] + "model": "BAAI/bge-m3", + "task": "token_embed", + "input": ["What is BGE M3?", "Definition of BM25"] }' ``` + +### Dense and sparse output through an IO processor plugin + +The source tree includes a reference +[BGE-M3 IO processor plugin](../../../tests/plugins/bge_m3_sparse_plugin) that +formats dense embeddings, sparse token weights, or both in one response. From +a source checkout, install it in the vLLM environment and load it as follows: + +```shell +uv pip install ./tests/plugins/bge_m3_sparse_plugin + +vllm serve BAAI/bge-m3 \ + --runner pooling \ + --hf-overrides '{"architectures": ["BgeM3EmbeddingModel"]}' \ + --io-processor-plugin bge_m3_sparse_plugin +``` + +The plugin selects the internal `embed&token_classify` task so the model +computes dense and lexical outputs together. Public requests must use task +`plugin` and put the plugin-specific fields under `data`: + +```shell +curl -s http://localhost:8000/pooling -H "Content-Type: application/json" -d '{ + "model": "BAAI/bge-m3", + "task": "plugin", + "data": { + "input": ["What is BGE M3?", "Definition of BM25"], + "embed_task": "dense&sparse", + "return_tokens": true + } +}' +``` + +`embed_task` accepts `dense`, `sparse`, or `dense&sparse`. The combined +`embed&token_classify` task is an internal execution contract for this plugin, +not a generic Pooling API response format. Without the plugin, select one of +the three concrete tasks above. diff --git a/tests/entrypoints/pooling/test_factories.py b/tests/entrypoints/pooling/test_factories.py new file mode 100644 index 0000000000..8972ac49c9 --- /dev/null +++ b/tests/entrypoints/pooling/test_factories.py @@ -0,0 +1,94 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +from unittest.mock import MagicMock + +import pytest + +from vllm.entrypoints.pooling import factories +from vllm.entrypoints.pooling.factories import init_pooling_io_processors +from vllm.entrypoints.pooling.pooling.io_processor import ( + PluginWithIOProcessorPlugins, + UnsupportedCombinedTaskIOProcessor, +) + + +def _bge_m3_config(io_processor_plugin=None): + model_config = MagicMock() + model_config.get_pooling_task.return_value = "embed&token_classify" + model_config.io_processor_plugin = io_processor_plugin + model_config.hf_config.to_dict.return_value = {} + model_config.architecture = "BgeM3EmbeddingModel" + + vllm_config = MagicMock(model_config=model_config) + renderer = MagicMock() + renderer._executor = MagicMock() + chat_template_config = MagicMock( + chat_template=None, + chat_template_content_format="auto", + trust_request_chat_template=False, + ) + return vllm_config, renderer, chat_template_config + + +def test_combined_task_without_plugin_uses_rejection_processor(): + vllm_config, renderer, chat_template_config = _bge_m3_config() + + processors = init_pooling_io_processors( + supported_tasks=("embed", "embed&token_classify"), + vllm_config=vllm_config, + renderer=renderer, + chat_template_config=chat_template_config, + ) + + assert processors.keys() == {"embed&token_classify"} + assert isinstance( + processors["embed&token_classify"], UnsupportedCombinedTaskIOProcessor + ) + + +def test_combined_task_with_plugin_uses_plugin_processor(monkeypatch): + vllm_config, renderer, chat_template_config = _bge_m3_config("bge_m3_sparse_plugin") + monkeypatch.setattr(factories, "has_io_processor", lambda *_: True) + monkeypatch.setattr( + "vllm.entrypoints.pooling.pooling.io_processor.get_io_processor", + lambda *_: MagicMock(), + ) + + processors = init_pooling_io_processors( + supported_tasks=("embed", "embed&token_classify"), + vllm_config=vllm_config, + renderer=renderer, + chat_template_config=chat_template_config, + ) + + assert processors.keys() == {"embed&token_classify", "plugin"} + assert isinstance(processors["plugin"], PluginWithIOProcessorPlugins) + + +def test_combined_task_plain_pooling_request_has_actionable_error(monkeypatch): + from vllm.entrypoints.pooling.pooling.protocol import PoolingCompletionRequest + from vllm.entrypoints.pooling.pooling.serving import ServingPooling + + vllm_config, renderer, chat_template_config = _bge_m3_config("bge_m3_sparse_plugin") + monkeypatch.setattr(factories, "has_io_processor", lambda *_: True) + monkeypatch.setattr( + "vllm.entrypoints.pooling.pooling.io_processor.get_io_processor", + lambda *_: MagicMock(), + ) + + engine_client = MagicMock(renderer=renderer, vllm_config=vllm_config) + models = MagicMock(model_config=vllm_config.model_config) + serving = ServingPooling( + engine_client, + models, + supported_tasks=("embed", "embed&token_classify"), + request_logger=None, + chat_template_config=chat_template_config, + ) + request = PoolingCompletionRequest(model="BAAI/bge-m3", input=["hola"]) + + assert serving.io_processors.keys() == {"embed&token_classify", "plugin"} + io_processor = serving.get_io_processor(request) + with pytest.raises(ValueError, match="plugin request with a 'data' field"): + io_processor.create_pooling_params(request) diff --git a/vllm/entrypoints/pooling/factories.py b/vllm/entrypoints/pooling/factories.py index dd3d873b31..c7615054dd 100644 --- a/vllm/entrypoints/pooling/factories.py +++ b/vllm/entrypoints/pooling/factories.py @@ -65,10 +65,16 @@ def init_pooling_io_processors( processors["token_embed"] = TokenEmbedIOProcessor - if has_io_processor( + if pooling_task == "embed&token_classify": + from .pooling.io_processor import UnsupportedCombinedTaskIOProcessor + + processors[pooling_task] = UnsupportedCombinedTaskIOProcessor + + has_plugin = has_io_processor( vllm_config, model_config.io_processor_plugin, - ): + ) + if has_plugin: from .pooling.io_processor import PluginWithIOProcessorPlugins processors["plugin"] = PluginWithIOProcessorPlugins diff --git a/vllm/entrypoints/pooling/pooling/io_processor.py b/vllm/entrypoints/pooling/pooling/io_processor.py index ecc356c477..4b18cfc9e1 100644 --- a/vllm/entrypoints/pooling/pooling/io_processor.py +++ b/vllm/entrypoints/pooling/pooling/io_processor.py @@ -24,6 +24,18 @@ from .protocol import IOProcessorRequest, IOProcessorResponse logger = init_logger(__name__) +class UnsupportedCombinedTaskIOProcessor(PoolingIOProcessor): + name = "embed&token_classify" + + def create_pooling_params(self, request): + raise ValueError( + "The 'embed&token_classify' pooling task is only available " + "through an IO processor plugin. Send a plugin request with " + "a 'data' field, " + "or select a concrete task with --pooler-config.task." + ) + + class PluginWithoutIOProcessorPlugins(PoolingIOProcessor): # Some models, such as Terratorch (tests/models/test_terratorch.py), # use plugin tasks in the pooler but do not use IO Processor plugins.