[Core][Bugfix] Support prompt_logprobs returned with speculative decoding (#8047)
Signed-off-by: Travis Johnson <tsjohnso@us.ibm.com>
This commit is contained in:
@@ -6,7 +6,8 @@ import torch
|
||||
|
||||
from vllm.model_executor.layers.sampler import SamplerOutput
|
||||
from vllm.sequence import (CompletionSequenceGroupOutput, Logprob,
|
||||
SequenceGroupMetadata, SequenceOutput)
|
||||
PromptLogprobs, SequenceGroupMetadata,
|
||||
SequenceOutput)
|
||||
|
||||
SeqId = int
|
||||
|
||||
@@ -49,21 +50,19 @@ def get_sampled_token_logprobs(
|
||||
return sampled_token_ids_ranks, selected_logprobs
|
||||
|
||||
|
||||
def create_sequence_group_output(
|
||||
def create_logprobs_output(
|
||||
token_id: int,
|
||||
token_id_logprob_rank: int,
|
||||
token_id_logprob: float,
|
||||
seq_id: SeqId,
|
||||
topk_token_ids: List[Optional[int]],
|
||||
topk_logprobs: List[Optional[float]],
|
||||
) -> CompletionSequenceGroupOutput:
|
||||
"""Create a SequenceGroupOutput given the sampling results.
|
||||
) -> Dict[int, Logprob]:
|
||||
"""Create a Logprob Dict for a token given the sampling results.
|
||||
|
||||
Args:
|
||||
token_id (int): The sampled token for the sequence.
|
||||
token_id_logprob_rank (int): The logprob rank of the sampled token.
|
||||
token_id_logprob (float): The logprob value of the sampled token.
|
||||
seq_id (int): The sequence id.
|
||||
topk_token_ids (List[Optional[int]]): The list of top-k token ids.
|
||||
topk_logprobs (List[Optional[float]]): The list of top-k logprobs.
|
||||
"""
|
||||
@@ -85,14 +84,44 @@ def create_sequence_group_output(
|
||||
if topk_token_id is not None
|
||||
})
|
||||
|
||||
return logprobs
|
||||
|
||||
|
||||
def create_sequence_group_output(
|
||||
token_id: int,
|
||||
token_id_logprob_rank: int,
|
||||
token_id_logprob: float,
|
||||
seq_id: SeqId,
|
||||
topk_token_ids: List[Optional[int]],
|
||||
topk_logprobs: List[Optional[float]],
|
||||
prompt_logprobs: Optional[PromptLogprobs] = None,
|
||||
) -> CompletionSequenceGroupOutput:
|
||||
"""Create a SequenceGroupOutput given the sampling results.
|
||||
|
||||
Args:
|
||||
token_id (int): The sampled token for the sequence.
|
||||
token_id_logprob_rank (int): The logprob rank of the sampled token.
|
||||
token_id_logprob (float): The logprob value of the sampled token.
|
||||
seq_id (int): The sequence id.
|
||||
topk_token_ids (List[Optional[int]]): The list of top-k token ids.
|
||||
topk_logprobs (List[Optional[float]]): The list of top-k logprobs.
|
||||
"""
|
||||
|
||||
logprobs = create_logprobs_output(
|
||||
token_id,
|
||||
token_id_logprob_rank,
|
||||
token_id_logprob,
|
||||
topk_token_ids,
|
||||
topk_logprobs,
|
||||
)
|
||||
|
||||
return CompletionSequenceGroupOutput(
|
||||
samples=[
|
||||
SequenceOutput(parent_seq_id=seq_id,
|
||||
output_token=token_id,
|
||||
logprobs=logprobs)
|
||||
],
|
||||
# TODO add prompt logprobs support.
|
||||
prompt_logprobs=None,
|
||||
prompt_logprobs=prompt_logprobs,
|
||||
)
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user