[Bugfix] Restore truncate_prompt_tokens for Jina rerank/score online (#49963)
Signed-off-by: Umut Polat <52835619+umut-polat@users.noreply.github.com>
This commit is contained in:
@@ -0,0 +1,59 @@
|
|||||||
|
# SPDX-License-Identifier: Apache-2.0
|
||||||
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||||
|
"""Unit tests for JinaRankingIOProcessor online request building."""
|
||||||
|
|
||||||
|
from unittest.mock import MagicMock
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
from vllm.entrypoints.pooling.base.io_processor import PoolingIOProcessor
|
||||||
|
from vllm.entrypoints.pooling.scoring.io_processor import JinaRankingIOProcessor
|
||||||
|
from vllm.entrypoints.pooling.scoring.protocol import RerankRequest
|
||||||
|
from vllm.entrypoints.pooling.scoring.typing import ScoringData
|
||||||
|
|
||||||
|
pytestmark = pytest.mark.skip_global_cleanup
|
||||||
|
|
||||||
|
|
||||||
|
def test_online_forwards_truncate_prompt_tokens_to_proxy(monkeypatch):
|
||||||
|
"""The proxy request handed to the base factory must carry
|
||||||
|
truncate_prompt_tokens/truncation_side from the real request.
|
||||||
|
|
||||||
|
JinaRankingIOProcessor swaps ctx.request for a proxy
|
||||||
|
PoolingCompletionRequest before delegating to the base factory, which
|
||||||
|
reads truncation off ctx.request. Dropping the fields on the proxy
|
||||||
|
silently disables truncate_prompt_tokens for Jina rerank/score.
|
||||||
|
"""
|
||||||
|
proc = JinaRankingIOProcessor.__new__(JinaRankingIOProcessor)
|
||||||
|
proc.valid_inputs_online = MagicMock(
|
||||||
|
return_value=ScoringData(data_1=["query"], data_2=["doc"])
|
||||||
|
)
|
||||||
|
proc._get_token_limits = MagicMock(return_value=(0, 0))
|
||||||
|
proc.ensure_str = MagicMock(side_effect=lambda data: list(data))
|
||||||
|
proc.format_docs_prompts_func = MagicMock(return_value="formatted prompt")
|
||||||
|
|
||||||
|
captured: dict[str, object] = {}
|
||||||
|
|
||||||
|
def _spy_base(self, ctx):
|
||||||
|
captured["truncate_prompt_tokens"] = ctx.request.truncate_prompt_tokens
|
||||||
|
captured["truncation_side"] = ctx.request.truncation_side
|
||||||
|
return []
|
||||||
|
|
||||||
|
monkeypatch.setattr(PoolingIOProcessor, "get_request_factory_online", _spy_base)
|
||||||
|
|
||||||
|
request = RerankRequest(
|
||||||
|
model="m",
|
||||||
|
query="query",
|
||||||
|
documents=["doc"],
|
||||||
|
truncate_prompt_tokens=512,
|
||||||
|
truncation_side="left",
|
||||||
|
)
|
||||||
|
ctx = MagicMock()
|
||||||
|
ctx.request = request
|
||||||
|
ctx.prompt_extras = None
|
||||||
|
|
||||||
|
proc.get_request_factory_online(ctx)
|
||||||
|
|
||||||
|
assert captured["truncate_prompt_tokens"] == 512
|
||||||
|
assert captured["truncation_side"] == "left"
|
||||||
|
# The real request is restored after delegating.
|
||||||
|
assert ctx.request is request
|
||||||
@@ -820,7 +820,16 @@ class JinaRankingIOProcessor(LateInteractionIOProcessor, JinaRankingIOProcessorM
|
|||||||
for q, d in zip(queries, docs)
|
for q, d in zip(queries, docs)
|
||||||
]
|
]
|
||||||
|
|
||||||
ctx.request = PoolingCompletionRequest(task="token_embed", input=prompts)
|
# Forward truncation from the real request: the base factory reads
|
||||||
|
# these off ctx.request, so omitting them here silently drops
|
||||||
|
# truncate_prompt_tokens for Jina rerank/score (unlike the embed and
|
||||||
|
# bi/cross-encoder paths, which read them from the real request).
|
||||||
|
ctx.request = PoolingCompletionRequest(
|
||||||
|
task="token_embed",
|
||||||
|
input=prompts,
|
||||||
|
truncate_prompt_tokens=request.truncate_prompt_tokens,
|
||||||
|
truncation_side=request.truncation_side,
|
||||||
|
)
|
||||||
requests = PoolingIOProcessor.get_request_factory_online(self, ctx)
|
requests = PoolingIOProcessor.get_request_factory_online(self, ctx)
|
||||||
ctx.request = request
|
ctx.request = request
|
||||||
return requests
|
return requests
|
||||||
|
|||||||
Reference in New Issue
Block a user