[Frontend] expose stream_interval as req sampling param (#49754)
Signed-off-by: walterbm <walter.beller.morales@gmail.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
bf4f633b4c
commit
8040ef2426
@@ -299,6 +299,11 @@ class SamplingParams(
|
||||
include_stop_str_in_output: bool = False
|
||||
"""Whether to include the stop strings in output text."""
|
||||
output_kind: RequestOutputKind = RequestOutputKind.CUMULATIVE
|
||||
stream_interval: int | None = None
|
||||
"""Number of newly generated tokens to batch into each streamed
|
||||
`RequestOutput`. Raises the interval above the engine-level
|
||||
`--stream-interval`. Values below engine setting are clamped up to it.
|
||||
The first and final outputs are always emitted immediately."""
|
||||
skip_clone: bool = False
|
||||
"""Internal flag indicating that this SamplingParams instance is safe to
|
||||
reuse without cloning. When True, clone() will return self without
|
||||
@@ -377,6 +382,7 @@ class SamplingParams(
|
||||
skip_special_tokens: bool = True,
|
||||
spaces_between_special_tokens: bool = True,
|
||||
output_kind: RequestOutputKind = RequestOutputKind.CUMULATIVE,
|
||||
stream_interval: int | None = None,
|
||||
structured_outputs: StructuredOutputsParams | None = None,
|
||||
logit_bias: dict[int, float] | dict[str, float] | None = None,
|
||||
allowed_token_ids: list[int] | None = None,
|
||||
@@ -439,6 +445,7 @@ class SamplingParams(
|
||||
skip_special_tokens=skip_special_tokens,
|
||||
spaces_between_special_tokens=spaces_between_special_tokens,
|
||||
output_kind=output_kind,
|
||||
stream_interval=stream_interval,
|
||||
structured_outputs=structured_outputs,
|
||||
logit_bias=logit_bias,
|
||||
allowed_token_ids=allowed_token_ids,
|
||||
@@ -585,6 +592,12 @@ class SamplingParams(
|
||||
f"min_tokens must be less than or equal to "
|
||||
f"max_tokens={self.max_tokens}, got {self.min_tokens}."
|
||||
)
|
||||
if self.stream_interval is not None and self.stream_interval < 1:
|
||||
raise VLLMValidationError(
|
||||
f"stream_interval must be at least 1, got {self.stream_interval}.",
|
||||
parameter="stream_interval",
|
||||
value=self.stream_interval,
|
||||
)
|
||||
if self.logprobs is not None and self.logprobs != -1 and self.logprobs < 0:
|
||||
raise VLLMValidationError(
|
||||
f"logprobs must be non-negative or -1, got {self.logprobs}.",
|
||||
|
||||
Reference in New Issue
Block a user