[Feature] Add instruction support for score/rerank chat templates (#42412)

Signed-off-by: KrxGu <krishom70@gmail.com>
This commit is contained in:
Krish Gupta
2026-05-14 09:41:22 +08:00
committed by GitHub
parent f51f6844f9
commit 70c00163ff
5 changed files with 182 additions and 12 deletions
@@ -377,3 +377,135 @@ async def test_score_api_queries_list_documents_list(
backend,
"paired[3]_text_vs_text_plus_image",
)
INSTRUCTION = (
"Given a multimodal retrieval query, retrieve candidates that "
"visually or textually match the requested scene, object, or action."
)
@pytest.mark.asyncio
async def test_score_api_instruction_field(
server: tuple[RemoteOpenAIServer, str],
):
remote_server, _ = server
default_response = requests.post(
remote_server.url_for("score"),
json={
"model": MODEL_NAME,
"queries": query,
"documents": document,
},
)
default_response.raise_for_status()
default_score = ScoreResponse.model_validate(default_response.json())
instruction_response = requests.post(
remote_server.url_for("score"),
json={
"model": MODEL_NAME,
"queries": query,
"documents": document,
"instruction": INSTRUCTION,
},
)
instruction_response.raise_for_status()
instruction_score = ScoreResponse.model_validate(instruction_response.json())
assert instruction_score.id is not None
assert instruction_score.data is not None
assert len(instruction_score.data) == 1
assert instruction_score.usage.prompt_tokens > default_score.usage.prompt_tokens
@pytest.mark.asyncio
async def test_rerank_api_instruction_field(
server: tuple[RemoteOpenAIServer, str],
):
remote_server, _ = server
doc_list = [
document,
{"content": [documents[0]]},
{"content": [documents[1]]},
{"content": [documents[0], documents[1]]},
]
default_response = requests.post(
remote_server.url_for("rerank"),
json={
"model": MODEL_NAME,
"query": query,
"documents": doc_list,
},
)
default_response.raise_for_status()
default_rerank = RerankResponse.model_validate(default_response.json())
instruction_response = requests.post(
remote_server.url_for("rerank"),
json={
"model": MODEL_NAME,
"query": query,
"documents": doc_list,
"instruction": INSTRUCTION,
},
)
instruction_response.raise_for_status()
instruction_rerank = RerankResponse.model_validate(instruction_response.json())
assert instruction_rerank.id is not None
assert instruction_rerank.model is not None
assert instruction_rerank.usage is not None
assert len(instruction_rerank.results) == len(default_rerank.results)
assert instruction_rerank.usage.prompt_tokens > default_rerank.usage.prompt_tokens
@pytest.mark.asyncio
async def test_rerank_api_instruction_field_matches_chat_template_kwargs(
server: tuple[RemoteOpenAIServer, str],
):
remote_server, _ = server
doc_list = [
document,
{"content": [documents[0]]},
{"content": [documents[1]]},
{"content": [documents[0], documents[1]]},
]
field_response = requests.post(
remote_server.url_for("rerank"),
json={
"model": MODEL_NAME,
"query": query,
"documents": doc_list,
"instruction": INSTRUCTION,
},
)
field_response.raise_for_status()
field_rerank = RerankResponse.model_validate(field_response.json())
kwargs_response = requests.post(
remote_server.url_for("rerank"),
json={
"model": MODEL_NAME,
"query": query,
"documents": doc_list,
"chat_template_kwargs": {"instruction": INSTRUCTION},
},
)
kwargs_response.raise_for_status()
kwargs_rerank = RerankResponse.model_validate(kwargs_response.json())
assert kwargs_rerank.usage.prompt_tokens == field_rerank.usage.prompt_tokens
field_scores = [
r.relevance_score for r in sorted(field_rerank.results, key=lambda x: x.index)
]
kwargs_scores = [
r.relevance_score for r in sorted(kwargs_rerank.results, key=lambda x: x.index)
]
assert field_scores == pytest.approx(kwargs_scores)