[Feature] Add instruction support for score/rerank chat templates (#42412)
Signed-off-by: KrxGu <krishom70@gmail.com>
This commit is contained in:
@@ -157,7 +157,7 @@ class BiEncoderIOProcessor(ScoringIOProcessor):
|
||||
tok_params,
|
||||
prompt_extras={
|
||||
k: v
|
||||
for k in ("mm_processor_kwargs", "cache_salt")
|
||||
for k in ("mm_processor_kwargs", "cache_salt", "chat_template_kwargs")
|
||||
if (v := getattr(request, k, None)) is not None
|
||||
},
|
||||
)
|
||||
@@ -384,7 +384,7 @@ class CrossEncoderIOProcessor(ScoringIOProcessor):
|
||||
max_tokens_per_doc=max_tokens_per_doc,
|
||||
prompt_extras={
|
||||
k: v
|
||||
for k in ("mm_processor_kwargs", "cache_salt")
|
||||
for k in ("mm_processor_kwargs", "cache_salt", "chat_template_kwargs")
|
||||
if (v := getattr(request, k, None)) is not None
|
||||
},
|
||||
)
|
||||
@@ -407,6 +407,7 @@ class CrossEncoderIOProcessor(ScoringIOProcessor):
|
||||
pooling_params=ctx.pooling_params
|
||||
)
|
||||
|
||||
prompt_extras = ctx.pooling_params.extra_kwargs if ctx.pooling_params else None
|
||||
engine_inputs, pooling_params_list = self._pre_process(
|
||||
ctx.prompts,
|
||||
tok_params,
|
||||
@@ -414,6 +415,7 @@ class CrossEncoderIOProcessor(ScoringIOProcessor):
|
||||
ctx.chat_template,
|
||||
max_tokens_per_query=max_tokens_per_query,
|
||||
max_tokens_per_doc=max_tokens_per_doc,
|
||||
prompt_extras=prompt_extras,
|
||||
)
|
||||
ctx.pooling_params = pooling_params_list
|
||||
return engine_inputs
|
||||
@@ -453,6 +455,9 @@ class CrossEncoderIOProcessor(ScoringIOProcessor):
|
||||
chat_template=chat_template,
|
||||
max_tokens_per_query=max_tokens_per_query,
|
||||
max_tokens_per_doc=max_tokens_per_doc,
|
||||
chat_template_kwargs=prompt_extras.get("chat_template_kwargs")
|
||||
if prompt_extras
|
||||
else None,
|
||||
)
|
||||
|
||||
if token_type_ids := engine_prompt.pop("token_type_ids", None):
|
||||
@@ -477,6 +482,7 @@ class CrossEncoderIOProcessor(ScoringIOProcessor):
|
||||
chat_template: str | None = None,
|
||||
max_tokens_per_query: int = 0,
|
||||
max_tokens_per_doc: int = 0,
|
||||
chat_template_kwargs: dict[str, Any] | None = None,
|
||||
):
|
||||
model_config = self.model_config
|
||||
tokenizer = self.tokenizer
|
||||
@@ -556,6 +562,14 @@ class CrossEncoderIOProcessor(ScoringIOProcessor):
|
||||
# If that fails because there is no such template,
|
||||
# fall back to the default implementation.
|
||||
try:
|
||||
_safe_kwargs = chat_template_kwargs or {}
|
||||
_reserved = {"chat_template", "tools", "tokenize"}
|
||||
_unexpected = _reserved & _safe_kwargs.keys()
|
||||
if _unexpected:
|
||||
raise ValueError(
|
||||
"chat_template_kwargs contains reserved keys that "
|
||||
f"conflict with fixed scorer arguments: {_unexpected}"
|
||||
)
|
||||
full_prompt = safe_apply_chat_template(
|
||||
model_config,
|
||||
tokenizer,
|
||||
@@ -566,6 +580,7 @@ class CrossEncoderIOProcessor(ScoringIOProcessor):
|
||||
chat_template=chat_template,
|
||||
tools=None,
|
||||
tokenize=False,
|
||||
**_safe_kwargs,
|
||||
)
|
||||
prompt_inputs = tokenizer(full_prompt, **encode_kwargs)
|
||||
except ChatTemplateResolutionError:
|
||||
|
||||
@@ -1,9 +1,9 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import time
|
||||
from typing import TypeAlias
|
||||
from typing import Any, TypeAlias
|
||||
|
||||
from pydantic import BaseModel, Field
|
||||
from pydantic import BaseModel, Field, model_validator
|
||||
|
||||
from vllm import PoolingParams
|
||||
from vllm.config import ModelConfig
|
||||
@@ -35,8 +35,37 @@ class ScoringRequestMixin(PoolingBasicRequestMixin, ClassifyRequestMixin):
|
||||
"applies to the combined query+document)."
|
||||
),
|
||||
)
|
||||
instruction: str | None = Field(
|
||||
default=None,
|
||||
description=(
|
||||
"Task instruction prepended to each scored pair via the chat "
|
||||
"template. Equivalent to passing "
|
||||
"chat_template_kwargs={'instruction': ...}."
|
||||
),
|
||||
)
|
||||
chat_template_kwargs: dict[str, Any] | None = Field(
|
||||
default=None,
|
||||
description=(
|
||||
"Additional keyword args to pass to the chat template renderer. "
|
||||
"Will be accessible by the score/rerank chat template."
|
||||
),
|
||||
)
|
||||
# --8<-- [end:scoring-common-params]
|
||||
|
||||
@model_validator(mode="after")
|
||||
def _merge_instruction_into_kwargs(self) -> "ScoringRequestMixin":
|
||||
"""Fold the top-level `instruction` field into `chat_template_kwargs`.
|
||||
|
||||
This allows callers to use either the convenience field or the generic
|
||||
dict. Explicit keys inside `chat_template_kwargs` take precedence over
|
||||
the top-level `instruction` field.
|
||||
"""
|
||||
if self.instruction is not None:
|
||||
merged = dict(self.chat_template_kwargs or {})
|
||||
merged.setdefault("instruction", self.instruction)
|
||||
self.chat_template_kwargs = merged
|
||||
return self
|
||||
|
||||
def build_tok_params(self, model_config: ModelConfig) -> TokenizeParams:
|
||||
return self._build_pooling_tok_params(
|
||||
model_config,
|
||||
|
||||
Reference in New Issue
Block a user