feat: add max_tokens_per_doc in rerank request. (#38827)
Signed-off-by: Jesus Federico <jefp@amazon.com> Signed-off-by: wang.yuqi <yuqi.wang@daocloud.io> Co-authored-by: Claude Opus 4.6 (1M context) <noreply@anthropic.com> Co-authored-by: wang.yuqi <yuqi.wang@daocloud.io>
This commit is contained in:
co-authored by
Claude Opus 4.6
wang.yuqi
parent
0e39202ca9
commit
edbc1abd1c
@@ -20,6 +20,24 @@ from .typing import ScoreContentPartParam, ScoreInput
|
||||
|
||||
|
||||
class ScoreRequestMixin(PoolingBasicRequestMixin, ClassifyRequestMixin):
|
||||
max_tokens_per_query: int = Field(
|
||||
default=0,
|
||||
description=(
|
||||
"Maximum number of tokens per query. Queries longer than "
|
||||
"this will be truncated to this length. 0 means no "
|
||||
"query-level truncation is applied."
|
||||
),
|
||||
)
|
||||
max_tokens_per_doc: int = Field(
|
||||
default=0,
|
||||
description=(
|
||||
"Maximum number of tokens per document. Documents longer than "
|
||||
"this will be truncated to this length. 0 means no "
|
||||
"document-level truncation is applied (only truncate_prompt_tokens "
|
||||
"applies to the combined query+document)."
|
||||
),
|
||||
)
|
||||
|
||||
def build_tok_params(self, model_config: ModelConfig) -> TokenizeParams:
|
||||
encoder_config = model_config.encoder_config or {}
|
||||
|
||||
@@ -91,29 +109,11 @@ ScoreRequest: TypeAlias = (
|
||||
)
|
||||
|
||||
|
||||
class RerankRequest(PoolingBasicRequestMixin, ClassifyRequestMixin):
|
||||
class RerankRequest(ScoreRequestMixin):
|
||||
query: ScoreInput
|
||||
documents: ScoreInput | list[ScoreInput]
|
||||
top_n: int = Field(default_factory=lambda: 0)
|
||||
|
||||
def build_tok_params(self, model_config: ModelConfig) -> TokenizeParams:
|
||||
encoder_config = model_config.encoder_config or {}
|
||||
|
||||
return TokenizeParams(
|
||||
max_total_tokens=model_config.max_model_len,
|
||||
max_output_tokens=0,
|
||||
truncate_prompt_tokens=self.truncate_prompt_tokens,
|
||||
truncation_side=self.truncation_side,
|
||||
do_lower_case=encoder_config.get("do_lower_case", False),
|
||||
max_total_tokens_param="max_model_len",
|
||||
)
|
||||
|
||||
def to_pooling_params(self, task: PoolingTask = "classify"):
|
||||
return PoolingParams(
|
||||
task=task,
|
||||
use_activation=self.use_activation,
|
||||
)
|
||||
|
||||
|
||||
ScoringRequest: TypeAlias = ScoreRequest | RerankRequest
|
||||
|
||||
|
||||
Reference in New Issue
Block a user