Reinstate best_of for V0 (#14356)

Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
This commit is contained in:
Harry Mellor
2025-03-06 08:34:22 -08:00
committed by GitHub
parent 151b08e0fe
commit bf0560bda9
6 changed files with 50 additions and 3 deletions
@@ -168,8 +168,12 @@ class OpenAIServingCompletion(OpenAIServing):
model_name = self._get_model_name(request.model, lora_request)
num_prompts = len(engine_prompts)
# We do not stream the results when use beam search.
stream = (request.stream and not request.use_beam_search)
# Similar to the OpenAI API, when n != best_of, we do not stream the
# results. Noting that best_of is only supported in V0. In addition,
# we do not stream the results when use beam search.
stream = (request.stream
and (request.best_of is None or request.n == request.best_of)
and not request.use_beam_search)
# Streaming response
if stream: