[Frontend] Support using chat template as custom score template for reranking models (#30550)

Signed-off-by: Jakub Zakrzewski <jzakrzewski@nvidia.com>
Signed-off-by: wang.yuqi <yuqi.wang@daocloud.io>
Signed-off-by: wang.yuqi <noooop@126.com>
Co-authored-by: wang.yuqi <yuqi.wang@daocloud.io>
This commit is contained in:
Jakub Zakrzewski
2025-12-23 11:19:16 +00:00
committed by GitHub
co-authored by wang.yuqi
parent 27c6c2f98c
commit 23daef548d
19 changed files with 663 additions and 46 deletions
@@ -52,6 +52,7 @@ class ServingScores(OpenAIServing):
models: OpenAIServingModels,
*,
request_logger: RequestLogger | None,
score_template: str | None = None,
log_error_stack: bool = False,
) -> None:
super().__init__(
@@ -60,6 +61,7 @@ class ServingScores(OpenAIServing):
request_logger=request_logger,
log_error_stack=log_error_stack,
)
self.score_template = score_template
async def _embedding_score(
self,
@@ -169,6 +171,7 @@ class ServingScores(OpenAIServing):
data_2=data_2,
tokenizer=tokenizer,
tokenization_kwargs=tokenization_kwargs,
score_template=self.score_template,
)
self._validate_input(request, engine_prompt["prompt_token_ids"], full_prompt)
if request.mm_processor_kwargs is not None: