Implement stop strings and best_of (#114)
This commit is contained in:
+23
-21
@@ -1,6 +1,4 @@
|
||||
from typing import Dict, List, Union
|
||||
|
||||
from transformers import PreTrainedTokenizer, PreTrainedTokenizerFast
|
||||
from typing import Dict, List
|
||||
|
||||
from cacheflow.sequence import SequenceGroup
|
||||
|
||||
@@ -9,20 +7,23 @@ class CompletionOutput:
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
index: int,
|
||||
text: str,
|
||||
token_ids: List[int],
|
||||
cumulative_logprobs: float,
|
||||
cumulative_logprob: float,
|
||||
logprobs: List[Dict[int, float]],
|
||||
) -> None:
|
||||
self.index = index
|
||||
self.text = text
|
||||
self.token_ids = token_ids
|
||||
self.cumulative_logprobs = cumulative_logprobs
|
||||
self.cumulative_logprob = cumulative_logprob
|
||||
self.logprobs = logprobs
|
||||
|
||||
def __repr__(self) -> str:
|
||||
return (f"CompletionOutput(output={self.text!r}, "
|
||||
return (f"CompletionOutput(index={self.index}, "
|
||||
f"text={self.text!r}, "
|
||||
f"token_ids={self.token_ids}, "
|
||||
f"cumulative_logprobs={self.cumulative_logprobs}, "
|
||||
f"cumulative_logprob={self.cumulative_logprob}, "
|
||||
f"logprobs={self.logprobs})")
|
||||
|
||||
|
||||
@@ -43,31 +44,32 @@ class RequestOutput:
|
||||
self.done = done
|
||||
|
||||
@staticmethod
|
||||
def from_seq_group(
|
||||
seq_group: SequenceGroup,
|
||||
tokenizer: Union[PreTrainedTokenizer, PreTrainedTokenizerFast],
|
||||
) -> "RequestOutput":
|
||||
outputs: List[CompletionOutput] = []
|
||||
def from_seq_group(seq_group: SequenceGroup) -> "RequestOutput":
|
||||
# Get the top-n sequences.
|
||||
n = seq_group.sampling_params.n
|
||||
seqs = seq_group.get_seqs()
|
||||
for seq in seqs:
|
||||
output_token_ids = seq.data.output_token_ids
|
||||
output_str = tokenizer.decode(output_token_ids,
|
||||
skip_special_tokens=True)
|
||||
seq_logprobs = seq.data.cumulative_logprobs
|
||||
assert n <= len(seqs)
|
||||
sorted_seqs = sorted(
|
||||
seqs, key=lambda seq: seq.get_cumulative_logprob(), reverse=True)
|
||||
top_n_seqs = sorted_seqs[:n]
|
||||
|
||||
# Create the outputs.
|
||||
outputs: List[CompletionOutput] = []
|
||||
for seq in top_n_seqs:
|
||||
logprobs = seq.output_logprobs
|
||||
if seq_group.sampling_params.logprobs == 0:
|
||||
# NOTE: We need to take care of this case because the sequence
|
||||
# always has the logprobs of the sampled tokens even if the
|
||||
# logprobs are not requested.
|
||||
logprobs = {}
|
||||
output = CompletionOutput(output_str, output_token_ids,
|
||||
seq_logprobs, logprobs)
|
||||
output = CompletionOutput(seqs.index(seq), seq.output_text,
|
||||
seq.get_output_token_ids(),
|
||||
seq.get_cumulative_logprob(), logprobs)
|
||||
outputs.append(output)
|
||||
|
||||
# Every sequence in the sequence group should have the same prompt.
|
||||
prompt = seqs[0].prompt
|
||||
prompt_token_ids = seqs[0].data.prompt_token_ids
|
||||
prompt = top_n_seqs[0].prompt
|
||||
prompt_token_ids = top_n_seqs[0].data.prompt_token_ids
|
||||
return RequestOutput(seq_group.request_id, prompt, prompt_token_ids,
|
||||
outputs, seq_group.is_finished())
|
||||
|
||||
|
||||
Reference in New Issue
Block a user