From 27d7061ef62b3d853b4d335baf8ea9b3c56d9bf3 Mon Sep 17 00:00:00 2001 From: Umut Polat <52835619+umut-polat@users.noreply.github.com> Date: Mon, 27 Jul 2026 17:52:59 +0300 Subject: [PATCH] [Bugfix] Restore truncate_prompt_tokens for Jina rerank/score online (#49963) Signed-off-by: Umut Polat <52835619+umut-polat@users.noreply.github.com> --- .../test_jina_ranking_io_processor_unit.py | 59 +++++++++++++++++++ .../pooling/scoring/io_processor.py | 11 +++- 2 files changed, 69 insertions(+), 1 deletion(-) create mode 100644 tests/entrypoints/pooling/scoring/test_jina_ranking_io_processor_unit.py diff --git a/tests/entrypoints/pooling/scoring/test_jina_ranking_io_processor_unit.py b/tests/entrypoints/pooling/scoring/test_jina_ranking_io_processor_unit.py new file mode 100644 index 0000000000..c2fe62f609 --- /dev/null +++ b/tests/entrypoints/pooling/scoring/test_jina_ranking_io_processor_unit.py @@ -0,0 +1,59 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Unit tests for JinaRankingIOProcessor online request building.""" + +from unittest.mock import MagicMock + +import pytest + +from vllm.entrypoints.pooling.base.io_processor import PoolingIOProcessor +from vllm.entrypoints.pooling.scoring.io_processor import JinaRankingIOProcessor +from vllm.entrypoints.pooling.scoring.protocol import RerankRequest +from vllm.entrypoints.pooling.scoring.typing import ScoringData + +pytestmark = pytest.mark.skip_global_cleanup + + +def test_online_forwards_truncate_prompt_tokens_to_proxy(monkeypatch): + """The proxy request handed to the base factory must carry + truncate_prompt_tokens/truncation_side from the real request. + + JinaRankingIOProcessor swaps ctx.request for a proxy + PoolingCompletionRequest before delegating to the base factory, which + reads truncation off ctx.request. Dropping the fields on the proxy + silently disables truncate_prompt_tokens for Jina rerank/score. + """ + proc = JinaRankingIOProcessor.__new__(JinaRankingIOProcessor) + proc.valid_inputs_online = MagicMock( + return_value=ScoringData(data_1=["query"], data_2=["doc"]) + ) + proc._get_token_limits = MagicMock(return_value=(0, 0)) + proc.ensure_str = MagicMock(side_effect=lambda data: list(data)) + proc.format_docs_prompts_func = MagicMock(return_value="formatted prompt") + + captured: dict[str, object] = {} + + def _spy_base(self, ctx): + captured["truncate_prompt_tokens"] = ctx.request.truncate_prompt_tokens + captured["truncation_side"] = ctx.request.truncation_side + return [] + + monkeypatch.setattr(PoolingIOProcessor, "get_request_factory_online", _spy_base) + + request = RerankRequest( + model="m", + query="query", + documents=["doc"], + truncate_prompt_tokens=512, + truncation_side="left", + ) + ctx = MagicMock() + ctx.request = request + ctx.prompt_extras = None + + proc.get_request_factory_online(ctx) + + assert captured["truncate_prompt_tokens"] == 512 + assert captured["truncation_side"] == "left" + # The real request is restored after delegating. + assert ctx.request is request diff --git a/vllm/entrypoints/pooling/scoring/io_processor.py b/vllm/entrypoints/pooling/scoring/io_processor.py index 8ebabca57c..5dfe8a3a13 100644 --- a/vllm/entrypoints/pooling/scoring/io_processor.py +++ b/vllm/entrypoints/pooling/scoring/io_processor.py @@ -820,7 +820,16 @@ class JinaRankingIOProcessor(LateInteractionIOProcessor, JinaRankingIOProcessorM for q, d in zip(queries, docs) ] - ctx.request = PoolingCompletionRequest(task="token_embed", input=prompts) + # Forward truncation from the real request: the base factory reads + # these off ctx.request, so omitting them here silently drops + # truncate_prompt_tokens for Jina rerank/score (unlike the embed and + # bi/cross-encoder paths, which read them from the real request). + ctx.request = PoolingCompletionRequest( + task="token_embed", + input=prompts, + truncate_prompt_tokens=request.truncate_prompt_tokens, + truncation_side=request.truncation_side, + ) requests = PoolingIOProcessor.get_request_factory_online(self, ctx) ctx.request = request return requests