[Bugfix] spec decode handle None entries in topk args in create_sequence_group_output (#7232)
Signed-off-by: Travis Johnson <tsjohnso@us.ibm.com>
This commit is contained in:
@@ -64,23 +64,25 @@ def create_sequence_group_output(
|
||||
token_id_logprob_rank (int): The logprob rank of the sampled token.
|
||||
token_id_logprob (float): The logprob value of the sampled token.
|
||||
seq_id (int): The sequence id.
|
||||
topk_token_ids (List[int]): The list of top-k token ids.
|
||||
topk_logprobs (List[float]): The list of top-k logprobs.
|
||||
topk_token_ids (List[Optional[int]]): The list of top-k token ids.
|
||||
topk_logprobs (List[Optional[float]]): The list of top-k logprobs.
|
||||
"""
|
||||
# vLLM logprobs always include the sampled token. In addition, the user may
|
||||
# request topk-logprobs (where top-k varies per user up to max_logprobs).
|
||||
logprobs: Dict[Optional[int], Logprob] = {
|
||||
logprobs: Dict[int, Logprob] = {
|
||||
token_id: Logprob(
|
||||
logprob=token_id_logprob,
|
||||
rank=token_id_logprob_rank,
|
||||
),
|
||||
}
|
||||
logprobs.update({
|
||||
topk_token_ids[topk_logprob_index]: Logprob(
|
||||
logprob=topk_logprobs[topk_logprob_index],
|
||||
rank=topk_logprob_index + 1,
|
||||
topk_token_id: Logprob(
|
||||
logprob=topk_logprob if topk_logprob is not None else 0.0,
|
||||
rank=topk_index + 1,
|
||||
)
|
||||
for topk_logprob_index, _ in enumerate(topk_token_ids)
|
||||
for topk_index, (topk_token_id, topk_logprob) \
|
||||
in enumerate(zip(topk_token_ids, topk_logprobs)) \
|
||||
if topk_token_id is not None
|
||||
})
|
||||
|
||||
return CompletionSequenceGroupOutput(
|
||||
|
||||
Reference in New Issue
Block a user