[Feature] Add fixed-token prefill scoring (#54335)
Signed-off-by: aoshen02 <aoshen@inferact.ai> Signed-off-by: aoshen02 <aoshen02@users.noreply.github.com> Co-authored-by: aoshen02 <aoshen02@users.noreply.github.com> Co-authored-by: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
aoshen02
Claude Opus 5
parent
a7ff44355c
commit
b721a4c709
@@ -390,8 +390,12 @@ class RequestState:
|
||||
if self.output_kind == RequestOutputKind.DELTA:
|
||||
# Side effect: logprobs processor forgets prompt logprobs
|
||||
prompt_logprobs = self.logprobs_processor.pop_prompt_logprobs()
|
||||
prompt_token_id_logprobs = (
|
||||
self.logprobs_processor.pop_prompt_token_id_logprobs()
|
||||
)
|
||||
else:
|
||||
prompt_logprobs = self.logprobs_processor.prompt_logprobs
|
||||
prompt_token_id_logprobs = self.logprobs_processor.prompt_token_id_logprobs
|
||||
|
||||
return RequestOutput(
|
||||
request_id=external_req_id, # request_id is what was provided externally
|
||||
@@ -399,6 +403,7 @@ class RequestState:
|
||||
prompt=self.prompt,
|
||||
prompt_token_ids=prompt_token_ids,
|
||||
prompt_logprobs=prompt_logprobs,
|
||||
prompt_token_id_logprobs=prompt_token_id_logprobs,
|
||||
outputs=cast(list[CompletionOutput], outputs),
|
||||
finished=finished,
|
||||
kv_transfer_params=kv_transfer_params,
|
||||
|
||||
Reference in New Issue
Block a user