re-implement beam search on top of vllm core (#8726)
Co-authored-by: Brendan Wong <bjwpokemon@gmail.com>
This commit is contained in:
co-authored by
Brendan Wong
parent
88577ac928
commit
0250dd68c5
@@ -798,6 +798,20 @@ class VllmRunner:
|
||||
outputs = self.generate(prompts, beam_search_params)
|
||||
return outputs
|
||||
|
||||
def generate_beam_search_new(
|
||||
self,
|
||||
prompts: Union[List[str], List[List[int]]],
|
||||
beam_width: int,
|
||||
max_tokens: int,
|
||||
) -> List[Tuple[List[List[int]], List[str]]]:
|
||||
outputs = self.model.beam_search(prompts, beam_width, max_tokens)
|
||||
returned_outputs = []
|
||||
for output in outputs:
|
||||
token_ids = [x.tokens for x in output.sequences]
|
||||
texts = [x.text for x in output.sequences]
|
||||
returned_outputs.append((token_ids, texts))
|
||||
return returned_outputs
|
||||
|
||||
def encode(self, prompts: List[str]) -> List[List[float]]:
|
||||
req_outputs = self.model.encode(prompts)
|
||||
outputs = []
|
||||
|
||||
Reference in New Issue
Block a user