feat: add max_tokens_per_doc in rerank request. (#38827)
Signed-off-by: Jesus Federico <jefp@amazon.com> Signed-off-by: wang.yuqi <yuqi.wang@daocloud.io> Co-authored-by: Claude Opus 4.6 (1M context) <noreply@anthropic.com> Co-authored-by: wang.yuqi <yuqi.wang@daocloud.io>
This commit is contained in:
co-authored by
Claude Opus 4.6
wang.yuqi
parent
0e39202ca9
commit
edbc1abd1c
@@ -112,6 +112,35 @@ def test_classify(llm):
|
||||
assert len(outputs[0].outputs.data) == 1
|
||||
|
||||
|
||||
@pytest.mark.skip_global_cleanup
|
||||
def test_max_tokens_per_doc(llm: LLM):
|
||||
"""Test max_tokens_per_doc via PoolingParams.extra_kwargs (offline)."""
|
||||
long_doc = "The capital of France is Paris. " * 20
|
||||
|
||||
# Without truncation
|
||||
outputs_no_limit = llm.score(
|
||||
TEXTS_1[0],
|
||||
long_doc,
|
||||
use_tqdm=False,
|
||||
)
|
||||
|
||||
# With truncation via extra_kwargs
|
||||
outputs_with_limit = llm.score(
|
||||
TEXTS_1[0],
|
||||
long_doc,
|
||||
pooling_params=PoolingParams(extra_kwargs={"max_tokens_per_doc": 10}),
|
||||
use_tqdm=False,
|
||||
)
|
||||
|
||||
assert len(outputs_no_limit) == 1
|
||||
assert len(outputs_with_limit) == 1
|
||||
|
||||
# Truncated version should have fewer prompt tokens
|
||||
no_limit_tokens = len(outputs_no_limit[0].prompt_token_ids)
|
||||
with_limit_tokens = len(outputs_with_limit[0].prompt_token_ids)
|
||||
assert with_limit_tokens < no_limit_tokens
|
||||
|
||||
|
||||
def test_pooling_params(llm: LLM):
|
||||
def get_outputs(use_activation):
|
||||
outputs = llm.score(
|
||||
|
||||
@@ -471,6 +471,78 @@ async def test_pooling_token_classify(server: RemoteOpenAIServer):
|
||||
assert len(poolings.data[0].data[0]) == 1
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_rerank_max_tokens_per_doc(
|
||||
server: RemoteOpenAIServer,
|
||||
):
|
||||
"""Test that max_tokens_per_doc actually reduces the token count."""
|
||||
query = "What is the capital of France?"
|
||||
# Use a doc that fits within max_model_len=100 (query ~8 tokens + 4 special)
|
||||
long_doc = "The capital of France is Paris. " * 10 # ~70 tokens
|
||||
|
||||
# Without max_tokens_per_doc
|
||||
response_no_limit = requests.post(
|
||||
server.url_for("rerank"),
|
||||
json={
|
||||
"model": MODEL_NAME,
|
||||
"query": query,
|
||||
"documents": [long_doc],
|
||||
"truncate_prompt_tokens": 99,
|
||||
},
|
||||
)
|
||||
response_no_limit.raise_for_status()
|
||||
rerank_no_limit = RerankResponse.model_validate(response_no_limit.json())
|
||||
|
||||
# With max_tokens_per_doc
|
||||
response_with_limit = requests.post(
|
||||
server.url_for("rerank"),
|
||||
json={
|
||||
"model": MODEL_NAME,
|
||||
"query": query,
|
||||
"documents": [long_doc],
|
||||
"max_tokens_per_doc": 10,
|
||||
},
|
||||
)
|
||||
response_with_limit.raise_for_status()
|
||||
rerank_with_limit = RerankResponse.model_validate(response_with_limit.json())
|
||||
|
||||
assert rerank_with_limit.usage.prompt_tokens < rerank_no_limit.usage.prompt_tokens
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_rerank_max_tokens_per_doc_validation(
|
||||
server: RemoteOpenAIServer,
|
||||
):
|
||||
"""Test that max_tokens_per_doc validation works correctly."""
|
||||
query = "What is the capital of France?"
|
||||
documents = ["The capital of France is Paris."]
|
||||
|
||||
# Test with max_tokens_per_doc=0 (should succeed — means no truncation)
|
||||
response = requests.post(
|
||||
server.url_for("rerank"),
|
||||
json={
|
||||
"model": MODEL_NAME,
|
||||
"query": query,
|
||||
"documents": documents,
|
||||
"max_tokens_per_doc": 0,
|
||||
},
|
||||
)
|
||||
response.raise_for_status()
|
||||
|
||||
# Test with invalid max_tokens_per_doc (negative)
|
||||
response = requests.post(
|
||||
server.url_for("rerank"),
|
||||
json={
|
||||
"model": MODEL_NAME,
|
||||
"query": query,
|
||||
"documents": documents,
|
||||
"max_tokens_per_doc": -5,
|
||||
},
|
||||
)
|
||||
assert response.status_code == 400
|
||||
assert "max_tokens_per_doc must be a non-negative integer" in response.text
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("task", ["embed", "token_embed", "plugin"])
|
||||
async def test_pooling_not_supported(server: RemoteOpenAIServer, task: str):
|
||||
|
||||
@@ -0,0 +1,209 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
Tests for max_tokens_per_doc and max_tokens_per_query.
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
from dataclasses import dataclass
|
||||
|
||||
import pytest
|
||||
import requests
|
||||
|
||||
from tests.utils import VLLM_PATH, RemoteOpenAIServer
|
||||
from vllm.entrypoints.pooling.scoring.protocol import RerankResponse
|
||||
|
||||
os.environ["VLLM_LOGGING_LEVEL"] = "WARNING"
|
||||
|
||||
TEMPLATE_DIR = str(VLLM_PATH / "examples/pooling/score/template")
|
||||
|
||||
long_query = "What is the capital of France?" * 20
|
||||
long_doc = "The capital of France is Paris. " * 20
|
||||
|
||||
|
||||
@dataclass
|
||||
class TestConfig:
|
||||
model: str
|
||||
args: list[str]
|
||||
without_truncated_prompt_tokens: int
|
||||
with_max_tokens_per_query_prompt_tokens: int
|
||||
with_max_tokens_per_doc_prompt_tokens: int
|
||||
with_max_tokens_per_query_and_doc_prompt_tokens: int
|
||||
|
||||
|
||||
RERANK_CONFIGS = [
|
||||
# 1. cross-encoder
|
||||
TestConfig(
|
||||
model="jinaai/jina-reranker-v2-base-multilingual",
|
||||
args=[
|
||||
"--enforce-eager",
|
||||
"--max-model-len",
|
||||
"1024",
|
||||
"--trust-remote-code",
|
||||
],
|
||||
without_truncated_prompt_tokens=284,
|
||||
with_max_tokens_per_query_prompt_tokens=154,
|
||||
with_max_tokens_per_doc_prompt_tokens=154,
|
||||
with_max_tokens_per_query_and_doc_prompt_tokens=24,
|
||||
),
|
||||
# 2. cross-encoder + score template
|
||||
TestConfig(
|
||||
model="Qwen/Qwen3-Reranker-0.6B",
|
||||
args=[
|
||||
"--enforce-eager",
|
||||
"--max-model-len",
|
||||
"1024",
|
||||
"--hf-overrides",
|
||||
json.dumps(
|
||||
{
|
||||
"architectures": ["Qwen3ForSequenceClassification"],
|
||||
"classifier_from_token": ["no", "yes"],
|
||||
"is_original_qwen3_reranker": True,
|
||||
}
|
||||
),
|
||||
"--chat-template",
|
||||
os.path.join(TEMPLATE_DIR, "qwen3_reranker.jinja"),
|
||||
],
|
||||
without_truncated_prompt_tokens=352,
|
||||
with_max_tokens_per_query_prompt_tokens=223,
|
||||
with_max_tokens_per_doc_prompt_tokens=221,
|
||||
with_max_tokens_per_query_and_doc_prompt_tokens=92,
|
||||
),
|
||||
# 3. bi-encoder
|
||||
TestConfig(
|
||||
model="intfloat/multilingual-e5-small",
|
||||
args=[
|
||||
"--enforce-eager",
|
||||
"--max-model-len",
|
||||
"512",
|
||||
"--trust-remote-code",
|
||||
],
|
||||
without_truncated_prompt_tokens=286,
|
||||
with_max_tokens_per_query_prompt_tokens=156,
|
||||
with_max_tokens_per_doc_prompt_tokens=155,
|
||||
with_max_tokens_per_query_and_doc_prompt_tokens=25,
|
||||
),
|
||||
# 4. late-interaction
|
||||
TestConfig(
|
||||
model="answerdotai/answerai-colbert-small-v1",
|
||||
args=[
|
||||
"--enforce-eager",
|
||||
"--max-model-len",
|
||||
"512",
|
||||
"--trust-remote-code",
|
||||
],
|
||||
without_truncated_prompt_tokens=285,
|
||||
with_max_tokens_per_query_prompt_tokens=155,
|
||||
with_max_tokens_per_doc_prompt_tokens=155,
|
||||
with_max_tokens_per_query_and_doc_prompt_tokens=25,
|
||||
),
|
||||
# 5. jinaai/jina-reranker-v3
|
||||
TestConfig(
|
||||
model="jinaai/jina-reranker-v3",
|
||||
args=[
|
||||
"--enforce-eager",
|
||||
"--max-model-len",
|
||||
"1024",
|
||||
"--trust-remote-code",
|
||||
],
|
||||
without_truncated_prompt_tokens=567,
|
||||
with_max_tokens_per_query_prompt_tokens=308,
|
||||
with_max_tokens_per_doc_prompt_tokens=436,
|
||||
with_max_tokens_per_query_and_doc_prompt_tokens=177,
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
@pytest.fixture(scope="module", params=RERANK_CONFIGS, ids=lambda c: c.model)
|
||||
def server(request):
|
||||
config: TestConfig = request.param
|
||||
with RemoteOpenAIServer(config.model, config.args) as remote_server:
|
||||
yield config, remote_server
|
||||
|
||||
|
||||
def test_without_truncated(server):
|
||||
"""Test that max_tokens_per_doc truncates documents correctly."""
|
||||
config, remote_server = server
|
||||
|
||||
response = requests.post(
|
||||
remote_server.url_for("rerank"),
|
||||
json={"model": config.model, "query": long_query, "documents": [long_doc]},
|
||||
)
|
||||
response.raise_for_status()
|
||||
rerank = RerankResponse.model_validate(response.json())
|
||||
|
||||
assert rerank.id is not None
|
||||
assert rerank.results is not None
|
||||
assert len(rerank.results) == 1
|
||||
assert rerank.usage.prompt_tokens == config.without_truncated_prompt_tokens
|
||||
|
||||
|
||||
def test_max_tokens_per_query(server):
|
||||
"""Test that max_tokens_per_doc truncates documents correctly."""
|
||||
config, remote_server = server
|
||||
|
||||
response = requests.post(
|
||||
remote_server.url_for("rerank"),
|
||||
json={
|
||||
"model": config.model,
|
||||
"query": long_query,
|
||||
"documents": [long_doc],
|
||||
"max_tokens_per_query": 10,
|
||||
},
|
||||
)
|
||||
response.raise_for_status()
|
||||
rerank = RerankResponse.model_validate(response.json())
|
||||
|
||||
assert rerank.id is not None
|
||||
assert rerank.results is not None
|
||||
assert len(rerank.results) == 1
|
||||
assert rerank.usage.prompt_tokens == config.with_max_tokens_per_query_prompt_tokens
|
||||
|
||||
|
||||
def test_max_tokens_per_doc(server):
|
||||
"""Test that max_tokens_per_doc truncates documents correctly."""
|
||||
config, remote_server = server
|
||||
|
||||
response = requests.post(
|
||||
remote_server.url_for("rerank"),
|
||||
json={
|
||||
"model": config.model,
|
||||
"query": long_query,
|
||||
"documents": [long_doc],
|
||||
"max_tokens_per_doc": 10,
|
||||
},
|
||||
)
|
||||
response.raise_for_status()
|
||||
rerank = RerankResponse.model_validate(response.json())
|
||||
|
||||
assert rerank.id is not None
|
||||
assert rerank.results is not None
|
||||
assert len(rerank.results) == 1
|
||||
assert rerank.usage.prompt_tokens == config.with_max_tokens_per_doc_prompt_tokens
|
||||
|
||||
|
||||
def test_max_tokens_per_query_and_doc(server):
|
||||
"""Test that max_tokens_per_doc truncates documents correctly."""
|
||||
config, remote_server = server
|
||||
|
||||
response = requests.post(
|
||||
remote_server.url_for("rerank"),
|
||||
json={
|
||||
"model": config.model,
|
||||
"query": long_query,
|
||||
"documents": [long_doc],
|
||||
"max_tokens_per_query": 10,
|
||||
"max_tokens_per_doc": 10,
|
||||
},
|
||||
)
|
||||
response.raise_for_status()
|
||||
rerank = RerankResponse.model_validate(response.json())
|
||||
|
||||
assert rerank.id is not None
|
||||
assert rerank.results is not None
|
||||
assert len(rerank.results) == 1
|
||||
assert (
|
||||
rerank.usage.prompt_tokens
|
||||
== config.with_max_tokens_per_query_and_doc_prompt_tokens
|
||||
)
|
||||
Reference in New Issue
Block a user