[Bugfix][Frontend] Return all choices from /inference/v1/generate when n > 1 (#52399)
Signed-off-by: Quentin Gallouédec <quentin.gallouedec@huggingface.co>
This commit is contained in:
@@ -218,8 +218,9 @@ class ServingTokens(GenerateBaseServing):
|
||||
|
||||
if self.force_no_detokenize:
|
||||
sampling_params.detokenize = False
|
||||
if request.stream:
|
||||
sampling_params.output_kind = RequestOutputKind.DELTA
|
||||
sampling_params.output_kind = (
|
||||
RequestOutputKind.DELTA if request.stream else RequestOutputKind.FINAL_ONLY
|
||||
)
|
||||
|
||||
self._log_inputs(
|
||||
request_id,
|
||||
|
||||
Reference in New Issue
Block a user