Generative Scoring (#34539)
Signed-off-by: Vedant Jhaveri <vjhaveri@linkedin.com> Co-authored-by: Vedant Jhaveri <vjhaveri@linkedin.com> Co-authored-by: Claude Opus 4.6 (1M context) <noreply@anthropic.com> Co-authored-by: Cyrus Leung <tlleungac@connect.ust.hk>
This commit is contained in:
co-authored by
Vedant Jhaveri
Claude Opus 4.6
Cyrus Leung
parent
36f1dc19ae
commit
2e56975657
@@ -235,6 +235,10 @@ class InputBatch:
|
||||
|
||||
self.num_logprobs: dict[str, int] = {}
|
||||
|
||||
# req_id -> list of specific token IDs to compute logprobs for
|
||||
# More efficient than num_logprobs=-1 when only a few tokens are needed
|
||||
self.logprob_token_ids: dict[str, list[int]] = {}
|
||||
|
||||
# To accumulate prompt logprobs tensor chunks across prefill steps.
|
||||
self.in_progress_prompt_logprobs_cpu: dict[str, LogprobsTensors] = {}
|
||||
|
||||
@@ -395,6 +399,10 @@ class InputBatch:
|
||||
else sampling_params.logprobs
|
||||
)
|
||||
|
||||
# Store specific token IDs to compute logprobs for (more efficient)
|
||||
if sampling_params.logprob_token_ids is not None:
|
||||
self.logprob_token_ids[req_id] = sampling_params.logprob_token_ids
|
||||
|
||||
if sampling_params.allowed_token_ids:
|
||||
self.has_allowed_token_ids.add(req_id)
|
||||
if self.allowed_token_ids_mask_cpu_tensor is None:
|
||||
@@ -522,6 +530,7 @@ class InputBatch:
|
||||
self.repetition_penalties_reqs.discard(req_id)
|
||||
self.generators.pop(req_index, None)
|
||||
self.num_logprobs.pop(req_id, None)
|
||||
self.logprob_token_ids.pop(req_id, None)
|
||||
self.in_progress_prompt_logprobs_cpu.pop(req_id, None)
|
||||
if self.prev_req_id_to_index is not None:
|
||||
self.prev_req_id_to_index.pop(req_id, None)
|
||||
@@ -865,6 +874,15 @@ class InputBatch:
|
||||
)
|
||||
allowed_token_ids_mask = self.allowed_token_ids_mask[:num_reqs]
|
||||
|
||||
# Build per-request logprob_token_ids mapping: req_index -> token_ids
|
||||
logprob_token_ids_by_index: dict[int, list[int]] | None = None
|
||||
if self.logprob_token_ids:
|
||||
logprob_token_ids_by_index = {}
|
||||
for req_id, token_ids in self.logprob_token_ids.items():
|
||||
if req_id in self.req_id_to_index:
|
||||
req_index = self.req_id_to_index[req_id]
|
||||
logprob_token_ids_by_index[req_index] = token_ids
|
||||
|
||||
return SamplingMetadata(
|
||||
temperature=temperature,
|
||||
all_greedy=self.all_greedy,
|
||||
@@ -873,6 +891,7 @@ class InputBatch:
|
||||
top_k=None if self.no_top_k else self.top_k[:num_reqs],
|
||||
generators=self.generators,
|
||||
max_num_logprobs=self.max_num_logprobs,
|
||||
logprob_token_ids=logprob_token_ids_by_index,
|
||||
prompt_token_ids=prompt_token_ids,
|
||||
frequency_penalties=self.frequency_penalties[:num_reqs],
|
||||
presence_penalties=self.presence_penalties[:num_reqs],
|
||||
|
||||
Reference in New Issue
Block a user