[Frontend] expose stream_interval as req sampling param (#49754)
Signed-off-by: walterbm <walter.beller.morales@gmail.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
bf4f633b4c
commit
8040ef2426
@@ -224,6 +224,9 @@ class RequestState:
|
||||
if not sampling_params.detokenize:
|
||||
tokenizer = None
|
||||
output_kind = sampling_params.output_kind
|
||||
if sampling_params.stream_interval is not None:
|
||||
# clamp to the engine-level stream interval.
|
||||
stream_interval = max(sampling_params.stream_interval, stream_interval)
|
||||
logprobs_processor = LogprobsProcessor.from_new_request(
|
||||
tokenizer=tokenizer,
|
||||
request=request,
|
||||
|
||||
Reference in New Issue
Block a user