[Feature] Add fixed-token prefill scoring (#54335)
Signed-off-by: aoshen02 <aoshen@inferact.ai> Signed-off-by: aoshen02 <aoshen02@users.noreply.github.com> Co-authored-by: aoshen02 <aoshen02@users.noreply.github.com> Co-authored-by: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
aoshen02
Claude Opus 5
parent
a7ff44355c
commit
b721a4c709
@@ -381,6 +381,7 @@ class GenerateResponse(BaseModel):
|
||||
choices: list[GenerateResponseChoice]
|
||||
usage: UsageInfo | None = Field(default=None)
|
||||
prompt_logprobs: list[dict[int, Logprob] | None] | None = None
|
||||
prompt_token_id_logprobs: str | None = None
|
||||
prompt_token_ids: list[int] | None = None
|
||||
mm_placeholders: dict[str, list[PlaceholderRangeInfo]] | None = None
|
||||
metrics: PerRequestMetrics | None = None
|
||||
|
||||
@@ -138,6 +138,11 @@ class ServingTokens(GenerateBaseServing):
|
||||
f"sampling_params.n must be at most the server's max_num_seqs "
|
||||
f"({max_num_seqs}), got {sampling_params.n}."
|
||||
)
|
||||
# The stream schema has no field for the scores.
|
||||
if request.stream and sampling_params.prompt_logprob_token_ids is not None:
|
||||
return self.create_error_response(
|
||||
"prompt_logprob_token_ids are not available when stream=true."
|
||||
)
|
||||
if self.force_no_detokenize and sampling_params.stop:
|
||||
# SamplingParams rejects stop with detokenize=False at request
|
||||
# validation, but this server forces detokenize=False afterwards,
|
||||
@@ -389,6 +394,11 @@ class ServingTokens(GenerateBaseServing):
|
||||
choices=choices,
|
||||
usage=usage,
|
||||
prompt_logprobs=clamp_prompt_logprobs(final_res.prompt_logprobs),
|
||||
prompt_token_id_logprobs=(
|
||||
numpy2base64(final_res.prompt_token_id_logprobs)
|
||||
if final_res.prompt_token_id_logprobs is not None
|
||||
else None
|
||||
),
|
||||
prompt_token_ids=(
|
||||
final_res.prompt_token_ids if request.return_token_ids else None
|
||||
),
|
||||
|
||||
Reference in New Issue
Block a user