feat: add max_tokens_per_doc in rerank request. (#38827)
Signed-off-by: Jesus Federico <jefp@amazon.com> Signed-off-by: wang.yuqi <yuqi.wang@daocloud.io> Co-authored-by: Claude Opus 4.6 (1M context) <noreply@anthropic.com> Co-authored-by: wang.yuqi <yuqi.wang@daocloud.io>
This commit is contained in:
co-authored by
Claude Opus 4.6
wang.yuqi
parent
0e39202ca9
commit
edbc1abd1c
@@ -112,6 +112,35 @@ def test_classify(llm):
|
||||
assert len(outputs[0].outputs.data) == 1
|
||||
|
||||
|
||||
@pytest.mark.skip_global_cleanup
|
||||
def test_max_tokens_per_doc(llm: LLM):
|
||||
"""Test max_tokens_per_doc via PoolingParams.extra_kwargs (offline)."""
|
||||
long_doc = "The capital of France is Paris. " * 20
|
||||
|
||||
# Without truncation
|
||||
outputs_no_limit = llm.score(
|
||||
TEXTS_1[0],
|
||||
long_doc,
|
||||
use_tqdm=False,
|
||||
)
|
||||
|
||||
# With truncation via extra_kwargs
|
||||
outputs_with_limit = llm.score(
|
||||
TEXTS_1[0],
|
||||
long_doc,
|
||||
pooling_params=PoolingParams(extra_kwargs={"max_tokens_per_doc": 10}),
|
||||
use_tqdm=False,
|
||||
)
|
||||
|
||||
assert len(outputs_no_limit) == 1
|
||||
assert len(outputs_with_limit) == 1
|
||||
|
||||
# Truncated version should have fewer prompt tokens
|
||||
no_limit_tokens = len(outputs_no_limit[0].prompt_token_ids)
|
||||
with_limit_tokens = len(outputs_with_limit[0].prompt_token_ids)
|
||||
assert with_limit_tokens < no_limit_tokens
|
||||
|
||||
|
||||
def test_pooling_params(llm: LLM):
|
||||
def get_outputs(use_activation):
|
||||
outputs = llm.score(
|
||||
|
||||
@@ -471,6 +471,78 @@ async def test_pooling_token_classify(server: RemoteOpenAIServer):
|
||||
assert len(poolings.data[0].data[0]) == 1
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_rerank_max_tokens_per_doc(
|
||||
server: RemoteOpenAIServer,
|
||||
):
|
||||
"""Test that max_tokens_per_doc actually reduces the token count."""
|
||||
query = "What is the capital of France?"
|
||||
# Use a doc that fits within max_model_len=100 (query ~8 tokens + 4 special)
|
||||
long_doc = "The capital of France is Paris. " * 10 # ~70 tokens
|
||||
|
||||
# Without max_tokens_per_doc
|
||||
response_no_limit = requests.post(
|
||||
server.url_for("rerank"),
|
||||
json={
|
||||
"model": MODEL_NAME,
|
||||
"query": query,
|
||||
"documents": [long_doc],
|
||||
"truncate_prompt_tokens": 99,
|
||||
},
|
||||
)
|
||||
response_no_limit.raise_for_status()
|
||||
rerank_no_limit = RerankResponse.model_validate(response_no_limit.json())
|
||||
|
||||
# With max_tokens_per_doc
|
||||
response_with_limit = requests.post(
|
||||
server.url_for("rerank"),
|
||||
json={
|
||||
"model": MODEL_NAME,
|
||||
"query": query,
|
||||
"documents": [long_doc],
|
||||
"max_tokens_per_doc": 10,
|
||||
},
|
||||
)
|
||||
response_with_limit.raise_for_status()
|
||||
rerank_with_limit = RerankResponse.model_validate(response_with_limit.json())
|
||||
|
||||
assert rerank_with_limit.usage.prompt_tokens < rerank_no_limit.usage.prompt_tokens
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_rerank_max_tokens_per_doc_validation(
|
||||
server: RemoteOpenAIServer,
|
||||
):
|
||||
"""Test that max_tokens_per_doc validation works correctly."""
|
||||
query = "What is the capital of France?"
|
||||
documents = ["The capital of France is Paris."]
|
||||
|
||||
# Test with max_tokens_per_doc=0 (should succeed — means no truncation)
|
||||
response = requests.post(
|
||||
server.url_for("rerank"),
|
||||
json={
|
||||
"model": MODEL_NAME,
|
||||
"query": query,
|
||||
"documents": documents,
|
||||
"max_tokens_per_doc": 0,
|
||||
},
|
||||
)
|
||||
response.raise_for_status()
|
||||
|
||||
# Test with invalid max_tokens_per_doc (negative)
|
||||
response = requests.post(
|
||||
server.url_for("rerank"),
|
||||
json={
|
||||
"model": MODEL_NAME,
|
||||
"query": query,
|
||||
"documents": documents,
|
||||
"max_tokens_per_doc": -5,
|
||||
},
|
||||
)
|
||||
assert response.status_code == 400
|
||||
assert "max_tokens_per_doc must be a non-negative integer" in response.text
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("task", ["embed", "token_embed", "plugin"])
|
||||
async def test_pooling_not_supported(server: RemoteOpenAIServer, task: str):
|
||||
|
||||
Reference in New Issue
Block a user