[Bugfix][Scoring] Warn when serving original Qwen3 reranker without chat template (#56017)

Signed-off-by: 김재억 <gadian88@gmail.com>
Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>

tests/entrypoints/pooling/test_io_processor.py

@@ -74,3 +74,150 @@ def test_rejects_conflicting_pooling_task(processor: PoolingIOProcessor):
     )
     assert exc_info.value.parameter is None
     assert exc_info.value.value is None
+
+
+def test_qwen3_reranker_warns_without_chat_template(monkeypatch):
+    from unittest.mock import MagicMock
+
+    from vllm.entrypoints.pooling.scoring.io_processor import CrossEncoderIOProcessor
+
+    monkeypatch.setattr(
+        "vllm.model_executor.model_loader.get_model_cls", lambda *_: MagicMock()
+    )
+    monkeypatch.setattr(
+        "vllm.model_executor.models.interfaces.supports_score_template",
+        lambda *_: False,
+    )
+    monkeypatch.setattr(
+        "vllm.entrypoints.pooling.scoring.io_processor.is_mistral_tokenizer",
+        lambda *_: False,
+    )
+
+    mock_logger = MagicMock()
+    monkeypatch.setattr(
+        "vllm.entrypoints.pooling.scoring.io_processor.logger",
+        mock_logger,
+    )
+
+    model_config = MagicMock()
+    model_config.architecture = "Qwen3ForSequenceClassification"
+    model_config.model = "Qwen/Qwen3-Reranker-0.6B"
+    model_config.hf_config.is_original_qwen3_reranker = True
+    model_config.use_sep_token = False
+    model_config.is_multimodal_model = False
+    model_config.max_model_len = 8192
+
+    vllm_config = MagicMock(model_config=model_config)
+    tokenizer = MagicMock()
+    tokenizer.pad_token_id = 0
+    renderer = MagicMock()
+    renderer.get_tokenizer.return_value = tokenizer
+    chat_template_config = MagicMock(chat_template=None)
+
+    CrossEncoderIOProcessor(
+        vllm_config=vllm_config,
+        renderer=renderer,
+        chat_template_config=chat_template_config,
+    )
+
+    mock_logger.warning.assert_called_once()
+    warning_call = mock_logger.warning.call_args
+    assert "Qwen/Qwen3-Reranker-0.6B" in warning_call[0][1]
+    assert "qwen3_reranker.jinja" in warning_call[0][2]
+
+
+def test_qwen3_vl_reranker_warns_with_vl_template(monkeypatch):
+    from unittest.mock import MagicMock
+
+    from vllm.entrypoints.pooling.scoring.io_processor import CrossEncoderIOProcessor
+
+    monkeypatch.setattr(
+        "vllm.model_executor.model_loader.get_model_cls", lambda *_: MagicMock()
+    )
+    monkeypatch.setattr(
+        "vllm.model_executor.models.interfaces.supports_score_template",
+        lambda *_: False,
+    )
+    monkeypatch.setattr(
+        "vllm.entrypoints.pooling.scoring.io_processor.is_mistral_tokenizer",
+        lambda *_: False,
+    )
+
+    mock_logger = MagicMock()
+    monkeypatch.setattr(
+        "vllm.entrypoints.pooling.scoring.io_processor.logger",
+        mock_logger,
+    )
+
+    model_config = MagicMock()
+    model_config.architecture = "Qwen3VLForSequenceClassification"
+    model_config.model = "Qwen/Qwen3-VL-Reranker-2B"
+    model_config.hf_config.is_original_qwen3_reranker = True
+    model_config.use_sep_token = False
+    model_config.is_multimodal_model = True
+    model_config.max_model_len = 8192
+
+    vllm_config = MagicMock(model_config=model_config)
+    tokenizer = MagicMock()
+    tokenizer.pad_token_id = 0
+    renderer = MagicMock()
+    renderer.get_tokenizer.return_value = tokenizer
+    chat_template_config = MagicMock(chat_template=None)
+
+    CrossEncoderIOProcessor(
+        vllm_config=vllm_config,
+        renderer=renderer,
+        chat_template_config=chat_template_config,
+    )
+
+    mock_logger.warning.assert_called_once()
+    warning_call = mock_logger.warning.call_args
+    assert "Qwen/Qwen3-VL-Reranker-2B" in warning_call[0][1]
+    assert "qwen3_vl_reranker.jinja" in warning_call[0][2]
+
+
+def test_qwen3_reranker_no_warning_when_template_provided(monkeypatch):
+    from unittest.mock import MagicMock
+
+    from vllm.entrypoints.pooling.scoring.io_processor import CrossEncoderIOProcessor
+
+    monkeypatch.setattr(
+        "vllm.model_executor.model_loader.get_model_cls", lambda *_: MagicMock()
+    )
+    monkeypatch.setattr(
+        "vllm.model_executor.models.interfaces.supports_score_template",
+        lambda *_: False,
+    )
+    monkeypatch.setattr(
+        "vllm.entrypoints.pooling.scoring.io_processor.is_mistral_tokenizer",
+        lambda *_: False,
+    )
+
+    mock_logger = MagicMock()
+    monkeypatch.setattr(
+        "vllm.entrypoints.pooling.scoring.io_processor.logger",
+        mock_logger,
+    )
+
+    model_config = MagicMock()
+    model_config.architecture = "Qwen3ForSequenceClassification"
+    model_config.model = "Qwen/Qwen3-Reranker-0.6B"
+    model_config.hf_config.is_original_qwen3_reranker = True
+    model_config.use_sep_token = False
+    model_config.is_multimodal_model = False
+    model_config.max_model_len = 8192
+
+    vllm_config = MagicMock(model_config=model_config)
+    tokenizer = MagicMock()
+    tokenizer.pad_token_id = 0
+    renderer = MagicMock()
+    renderer.get_tokenizer.return_value = tokenizer
+    chat_template_config = MagicMock(chat_template="custom template")
+
+    CrossEncoderIOProcessor(
+        vllm_config=vllm_config,
+        renderer=renderer,
+        chat_template_config=chat_template_config,
+    )
+
+    mock_logger.warning.assert_not_called()

vllm/entrypoints/pooling/scoring/io_processor.py

@@ -7,6 +7,7 @@ from typing import Any, TypeAlias, cast
 import torch.nn.functional as F
 
 from vllm import PoolingParams, PoolingRequestOutput, TokensPrompt
+from vllm.logger import init_logger
 from vllm.renderers import TokenizeParams
 from vllm.renderers.hf import safe_apply_chat_template
 from vllm.renderers.inputs.preprocess import (
@@ -47,6 +48,8 @@ from .utils import (
     validate_score_input,
 )
 
+logger = init_logger(__name__)
+
 ScoringServeContext: TypeAlias = PoolingServeContext[ScoringRequest]
 
 
@@ -425,6 +428,24 @@ class CrossEncoderIOProcessor(ScoringIOProcessor):
         self.model = model if self.supports_score_template else None
         self.use_sep_token = self.model_config.use_sep_token
 
+        if (
+            getattr(self.model_config.hf_config, "is_original_qwen3_reranker", False)
+            and self.chat_template is None
+        ):
+            suggested_template = (
+                "examples/pooling/score/template/qwen3_vl_reranker.jinja"
+                if self.model_config.is_multimodal_model
+                else "examples/pooling/score/template/qwen3_reranker.jinja"
+            )
+            logger.warning(
+                "Serving an original Qwen3 reranker (%s) without a "
+                "--chat-template. The model was trained with a specific prompt "
+                "template; running without it may produce inaccurate relevance "
+                "scores. Consider specifying `--chat-template %s`.",
+                self.model_config.model,
+                suggested_template,
+            )
+
     #######################################
     # online APIs