[Feature] Add fixed-token prefill scoring (#54335)

Signed-off-by: aoshen02 <aoshen@inferact.ai>
Signed-off-by: aoshen02 <aoshen02@users.noreply.github.com>
Co-authored-by: aoshen02 <aoshen02@users.noreply.github.com>
Co-authored-by: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
aoshen02
2026-09-28 07:50:09 -07:00
committed by GitHub
co-authored by aoshen02 Claude Opus 5
parent a7ff44355c
commit b721a4c709
36 changed files with 768 additions and 8 deletions
+7
View File
@@ -156,6 +156,10 @@ class AsyncOutput(AsyncModelRunnerOutput):
k: v.to_cpu_nonblocking() if v is not None else None
for k, v in self.model_runner_output.prompt_logprobs_dict.items()
}
token_id_logprobs = self.model_runner_output.prompt_token_id_logprobs_dict
self.prompt_token_id_logprobs_dict = {
k: v.to("cpu", non_blocking=True) for k, v in token_id_logprobs.items()
}
if self.pending_aux_output is not None:
self.pending_aux_output.enqueue_cpu_copy(
num_sampled=self.num_sampled_tokens_np,
@@ -194,6 +198,9 @@ class AsyncOutput(AsyncModelRunnerOutput):
if self.logprobs_tensors is not None:
self.model_runner_output.logprobs = self.logprobs_tensors.tolists()
self.model_runner_output.prompt_logprobs_dict = self.prompt_logprobs_dict
self.model_runner_output.prompt_token_id_logprobs_dict = (
self.prompt_token_id_logprobs_dict
)
if self.pending_aux_output is not None:
self.model_runner_output.aux_output_connector_output = (
self.pending_aux_output.process_output()