[CI] Shard (H200 MIG 35GB / MI355 DPX) Entrypoints Integration (Pooling) into named jobs (#58652)
Signed-off-by: Thang Nguyen <thangnguyenvn647@gmail.com> Signed-off-by: khluu <khluu000@gmail.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com> Co-authored-by: khluu <khluu000@gmail.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
khluu
parent
004e37ece2
commit
413ae4b167
@@ -0,0 +1,94 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm.entrypoints.pooling import factories
|
||||
from vllm.entrypoints.pooling.factories import init_pooling_io_processors
|
||||
from vllm.entrypoints.pooling.pooling.io_processor import (
|
||||
PluginWithIOProcessorPlugins,
|
||||
UnsupportedCombinedTaskIOProcessor,
|
||||
)
|
||||
|
||||
|
||||
def _bge_m3_config(io_processor_plugin=None):
|
||||
model_config = MagicMock()
|
||||
model_config.get_pooling_task.return_value = "embed&token_classify"
|
||||
model_config.io_processor_plugin = io_processor_plugin
|
||||
model_config.hf_config.to_dict.return_value = {}
|
||||
model_config.architecture = "BgeM3EmbeddingModel"
|
||||
|
||||
vllm_config = MagicMock(model_config=model_config)
|
||||
renderer = MagicMock()
|
||||
renderer._executor = MagicMock()
|
||||
chat_template_config = MagicMock(
|
||||
chat_template=None,
|
||||
chat_template_content_format="auto",
|
||||
trust_request_chat_template=False,
|
||||
)
|
||||
return vllm_config, renderer, chat_template_config
|
||||
|
||||
|
||||
def test_combined_task_without_plugin_uses_rejection_processor():
|
||||
vllm_config, renderer, chat_template_config = _bge_m3_config()
|
||||
|
||||
processors = init_pooling_io_processors(
|
||||
supported_tasks=("embed", "embed&token_classify"),
|
||||
vllm_config=vllm_config,
|
||||
renderer=renderer,
|
||||
chat_template_config=chat_template_config,
|
||||
)
|
||||
|
||||
assert processors.keys() == {"embed&token_classify"}
|
||||
assert isinstance(
|
||||
processors["embed&token_classify"], UnsupportedCombinedTaskIOProcessor
|
||||
)
|
||||
|
||||
|
||||
def test_combined_task_with_plugin_uses_plugin_processor(monkeypatch):
|
||||
vllm_config, renderer, chat_template_config = _bge_m3_config("bge_m3_sparse_plugin")
|
||||
monkeypatch.setattr(factories, "has_io_processor", lambda *_: True)
|
||||
monkeypatch.setattr(
|
||||
"vllm.entrypoints.pooling.pooling.io_processor.get_io_processor",
|
||||
lambda *_: MagicMock(),
|
||||
)
|
||||
|
||||
processors = init_pooling_io_processors(
|
||||
supported_tasks=("embed", "embed&token_classify"),
|
||||
vllm_config=vllm_config,
|
||||
renderer=renderer,
|
||||
chat_template_config=chat_template_config,
|
||||
)
|
||||
|
||||
assert processors.keys() == {"embed&token_classify", "plugin"}
|
||||
assert isinstance(processors["plugin"], PluginWithIOProcessorPlugins)
|
||||
|
||||
|
||||
def test_combined_task_plain_pooling_request_has_actionable_error(monkeypatch):
|
||||
from vllm.entrypoints.pooling.pooling.protocol import PoolingCompletionRequest
|
||||
from vllm.entrypoints.pooling.pooling.serving import ServingPooling
|
||||
|
||||
vllm_config, renderer, chat_template_config = _bge_m3_config("bge_m3_sparse_plugin")
|
||||
monkeypatch.setattr(factories, "has_io_processor", lambda *_: True)
|
||||
monkeypatch.setattr(
|
||||
"vllm.entrypoints.pooling.pooling.io_processor.get_io_processor",
|
||||
lambda *_: MagicMock(),
|
||||
)
|
||||
|
||||
engine_client = MagicMock(renderer=renderer, vllm_config=vllm_config)
|
||||
models = MagicMock(model_config=vllm_config.model_config)
|
||||
serving = ServingPooling(
|
||||
engine_client,
|
||||
models,
|
||||
supported_tasks=("embed", "embed&token_classify"),
|
||||
request_logger=None,
|
||||
chat_template_config=chat_template_config,
|
||||
)
|
||||
request = PoolingCompletionRequest(model="BAAI/bge-m3", input=["hola"])
|
||||
|
||||
assert serving.io_processors.keys() == {"embed&token_classify", "plugin"}
|
||||
io_processor = serving.get_io_processor(request)
|
||||
with pytest.raises(ValueError, match="plugin request with a 'data' field"):
|
||||
io_processor.create_pooling_params(request)
|
||||
@@ -0,0 +1,223 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm import PoolingParams
|
||||
from vllm.entrypoints.pooling.base.io_processor import PoolingIOProcessor
|
||||
from vllm.entrypoints.pooling.typing import OfflineEncodeInputsContext
|
||||
from vllm.exceptions import VLLMValidationError
|
||||
from vllm.renderers import TokenizeParams
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def processor() -> PoolingIOProcessor:
|
||||
return object.__new__(PoolingIOProcessor)
|
||||
|
||||
|
||||
def test_rejects_untrusted_request_chat_template(processor: PoolingIOProcessor):
|
||||
with pytest.raises(VLLMValidationError) as exc_info:
|
||||
processor._validate_chat_template("template", None, False)
|
||||
|
||||
assert str(exc_info.value) == (
|
||||
"Chat template is passed with request, but "
|
||||
"--trust-request-chat-template is not set. "
|
||||
"Refused request with untrusted chat template."
|
||||
)
|
||||
assert exc_info.value.parameter is None
|
||||
assert exc_info.value.value is None
|
||||
|
||||
|
||||
def test_rejects_mismatched_pooling_params(processor: PoolingIOProcessor):
|
||||
with pytest.raises(VLLMValidationError) as exc_info:
|
||||
processor._params_to_seq([PoolingParams()], num_requests=2)
|
||||
|
||||
assert str(exc_info.value) == (
|
||||
"The lengths of prompts (2) and params (1) must be the same."
|
||||
)
|
||||
assert exc_info.value.parameter is None
|
||||
assert exc_info.value.value is None
|
||||
|
||||
|
||||
def test_rejects_mismatched_lora_requests(processor: PoolingIOProcessor):
|
||||
with pytest.raises(VLLMValidationError) as exc_info:
|
||||
processor._lora_request_to_seq([None], num_requests=2)
|
||||
|
||||
assert str(exc_info.value) == (
|
||||
"The lengths of prompts (2) and lora_request (1) must be the same."
|
||||
)
|
||||
assert exc_info.value.parameter is None
|
||||
assert exc_info.value.value is None
|
||||
|
||||
|
||||
def test_rejects_conflicting_pooling_task(processor: PoolingIOProcessor):
|
||||
processor.model_config = SimpleNamespace(is_encoder_decoder=False)
|
||||
processor.renderer = SimpleNamespace(
|
||||
default_cmpl_tok_params=TokenizeParams(max_total_tokens=None)
|
||||
)
|
||||
ctx = OfflineEncodeInputsContext(
|
||||
pooling_task="embed",
|
||||
tokenization_kwargs=None,
|
||||
lora_request=None,
|
||||
priorities=None,
|
||||
prompts=[[1]],
|
||||
pooling_params=PoolingParams(task="classify"),
|
||||
)
|
||||
|
||||
with pytest.raises(VLLMValidationError) as exc_info:
|
||||
processor.get_request_factory_offline(ctx)
|
||||
|
||||
assert str(exc_info.value) == (
|
||||
"You cannot overwrite param.task='classify' with pooling_task='embed'!"
|
||||
)
|
||||
assert exc_info.value.parameter is None
|
||||
assert exc_info.value.value is None
|
||||
|
||||
|
||||
def test_qwen3_reranker_warns_without_chat_template(monkeypatch):
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
from vllm.entrypoints.pooling.scoring.io_processor import CrossEncoderIOProcessor
|
||||
|
||||
monkeypatch.setattr(
|
||||
"vllm.model_executor.model_loader.get_model_cls", lambda *_: MagicMock()
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"vllm.model_executor.models.interfaces.supports_score_template",
|
||||
lambda *_: False,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"vllm.entrypoints.pooling.scoring.io_processor.is_mistral_tokenizer",
|
||||
lambda *_: False,
|
||||
)
|
||||
|
||||
mock_logger = MagicMock()
|
||||
monkeypatch.setattr(
|
||||
"vllm.entrypoints.pooling.scoring.io_processor.logger",
|
||||
mock_logger,
|
||||
)
|
||||
|
||||
model_config = MagicMock()
|
||||
model_config.architecture = "Qwen3ForSequenceClassification"
|
||||
model_config.model = "Qwen/Qwen3-Reranker-0.6B"
|
||||
model_config.hf_config.is_original_qwen3_reranker = True
|
||||
model_config.use_sep_token = False
|
||||
model_config.is_multimodal_model = False
|
||||
model_config.max_model_len = 8192
|
||||
|
||||
vllm_config = MagicMock(model_config=model_config)
|
||||
tokenizer = MagicMock()
|
||||
tokenizer.pad_token_id = 0
|
||||
renderer = MagicMock()
|
||||
renderer.get_tokenizer.return_value = tokenizer
|
||||
chat_template_config = MagicMock(chat_template=None)
|
||||
|
||||
CrossEncoderIOProcessor(
|
||||
vllm_config=vllm_config,
|
||||
renderer=renderer,
|
||||
chat_template_config=chat_template_config,
|
||||
)
|
||||
|
||||
mock_logger.warning.assert_called_once()
|
||||
warning_call = mock_logger.warning.call_args
|
||||
assert "Qwen/Qwen3-Reranker-0.6B" in warning_call[0][1]
|
||||
assert "qwen3_reranker.jinja" in warning_call[0][2]
|
||||
|
||||
|
||||
def test_qwen3_vl_reranker_warns_with_vl_template(monkeypatch):
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
from vllm.entrypoints.pooling.scoring.io_processor import CrossEncoderIOProcessor
|
||||
|
||||
monkeypatch.setattr(
|
||||
"vllm.model_executor.model_loader.get_model_cls", lambda *_: MagicMock()
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"vllm.model_executor.models.interfaces.supports_score_template",
|
||||
lambda *_: False,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"vllm.entrypoints.pooling.scoring.io_processor.is_mistral_tokenizer",
|
||||
lambda *_: False,
|
||||
)
|
||||
|
||||
mock_logger = MagicMock()
|
||||
monkeypatch.setattr(
|
||||
"vllm.entrypoints.pooling.scoring.io_processor.logger",
|
||||
mock_logger,
|
||||
)
|
||||
|
||||
model_config = MagicMock()
|
||||
model_config.architecture = "Qwen3VLForSequenceClassification"
|
||||
model_config.model = "Qwen/Qwen3-VL-Reranker-2B"
|
||||
model_config.hf_config.is_original_qwen3_reranker = True
|
||||
model_config.use_sep_token = False
|
||||
model_config.is_multimodal_model = True
|
||||
model_config.max_model_len = 8192
|
||||
|
||||
vllm_config = MagicMock(model_config=model_config)
|
||||
tokenizer = MagicMock()
|
||||
tokenizer.pad_token_id = 0
|
||||
renderer = MagicMock()
|
||||
renderer.get_tokenizer.return_value = tokenizer
|
||||
chat_template_config = MagicMock(chat_template=None)
|
||||
|
||||
CrossEncoderIOProcessor(
|
||||
vllm_config=vllm_config,
|
||||
renderer=renderer,
|
||||
chat_template_config=chat_template_config,
|
||||
)
|
||||
|
||||
mock_logger.warning.assert_called_once()
|
||||
warning_call = mock_logger.warning.call_args
|
||||
assert "Qwen/Qwen3-VL-Reranker-2B" in warning_call[0][1]
|
||||
assert "qwen3_vl_reranker.jinja" in warning_call[0][2]
|
||||
|
||||
|
||||
def test_qwen3_reranker_no_warning_when_template_provided(monkeypatch):
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
from vllm.entrypoints.pooling.scoring.io_processor import CrossEncoderIOProcessor
|
||||
|
||||
monkeypatch.setattr(
|
||||
"vllm.model_executor.model_loader.get_model_cls", lambda *_: MagicMock()
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"vllm.model_executor.models.interfaces.supports_score_template",
|
||||
lambda *_: False,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"vllm.entrypoints.pooling.scoring.io_processor.is_mistral_tokenizer",
|
||||
lambda *_: False,
|
||||
)
|
||||
|
||||
mock_logger = MagicMock()
|
||||
monkeypatch.setattr(
|
||||
"vllm.entrypoints.pooling.scoring.io_processor.logger",
|
||||
mock_logger,
|
||||
)
|
||||
|
||||
model_config = MagicMock()
|
||||
model_config.architecture = "Qwen3ForSequenceClassification"
|
||||
model_config.model = "Qwen/Qwen3-Reranker-0.6B"
|
||||
model_config.hf_config.is_original_qwen3_reranker = True
|
||||
model_config.use_sep_token = False
|
||||
model_config.is_multimodal_model = False
|
||||
model_config.max_model_len = 8192
|
||||
|
||||
vllm_config = MagicMock(model_config=model_config)
|
||||
tokenizer = MagicMock()
|
||||
tokenizer.pad_token_id = 0
|
||||
renderer = MagicMock()
|
||||
renderer.get_tokenizer.return_value = tokenizer
|
||||
chat_template_config = MagicMock(chat_template="custom template")
|
||||
|
||||
CrossEncoderIOProcessor(
|
||||
vllm_config=vllm_config,
|
||||
renderer=renderer,
|
||||
chat_template_config=chat_template_config,
|
||||
)
|
||||
|
||||
mock_logger.warning.assert_not_called()
|
||||
@@ -0,0 +1,59 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import importlib
|
||||
import importlib.util
|
||||
import json
|
||||
import warnings
|
||||
from types import SimpleNamespace
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.entrypoints.pooling.utils import encode_pooling_output_float_or_ndarray
|
||||
|
||||
|
||||
def _pooling_output(data):
|
||||
return SimpleNamespace(outputs=SimpleNamespace(data=data))
|
||||
|
||||
|
||||
def test_encode_pooling_output_float_or_ndarray_returns_numpy_array():
|
||||
output = _pooling_output(torch.tensor([1.0, 2.0, 3.0], dtype=torch.float32))
|
||||
|
||||
encoded = encode_pooling_output_float_or_ndarray(output)
|
||||
|
||||
assert isinstance(encoded, np.ndarray)
|
||||
np.testing.assert_allclose(encoded, [1.0, 2.0, 3.0])
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
importlib.util.find_spec("orjson") is None,
|
||||
reason="orjson is not installed",
|
||||
)
|
||||
def test_orjson_serializes_numpy_array():
|
||||
from fastapi.responses import ORJSONResponse
|
||||
|
||||
output = _pooling_output(torch.tensor([1.0, 2.0, 3.0], dtype=torch.float32))
|
||||
encoded = encode_pooling_output_float_or_ndarray(output)
|
||||
|
||||
with warnings.catch_warnings():
|
||||
warnings.simplefilter("ignore", DeprecationWarning)
|
||||
response = ORJSONResponse(content={"embedding": encoded})
|
||||
assert json.loads(response.body)["embedding"] == pytest.approx([1.0, 2.0, 3.0])
|
||||
|
||||
|
||||
def test_encode_pooling_output_float_or_ndarray_falls_back_to_list():
|
||||
class DataWithUnsupportedNumpy:
|
||||
def is_contiguous(self):
|
||||
return True
|
||||
|
||||
def numpy(self):
|
||||
raise TypeError("unsupported dtype")
|
||||
|
||||
def tolist(self):
|
||||
return [1.0, 2.0, 3.0]
|
||||
|
||||
output = _pooling_output(DataWithUnsupportedNumpy())
|
||||
|
||||
assert encode_pooling_output_float_or_ndarray(output) == [1.0, 2.0, 3.0]
|
||||
Reference in New Issue
Block a user