Use queue for finished requests (#957)

This commit is contained in:
Antoni Baum
2023-09-05 19:27:23 -07:00
committed by GitHub
parent fbd80ad409
commit c9927c1a6a
2 changed files with 13 additions and 9 deletions
+2 -2
View File
@@ -156,8 +156,8 @@ class VllmRunner:
) -> List[Tuple[List[int], str]]:
greedy_params = SamplingParams(temperature=0.0, max_tokens=max_tokens)
outputs = self.generate(prompts, greedy_params)
return [(output_ids[0], output_str[0]) for output_ids, output_str in
outputs]
return [(output_ids[0], output_str[0])
for output_ids, output_str in outputs]
def generate_beam_search(
self,