[Frontend] expose stream_interval as req sampling param (#49754)

Signed-off-by: walterbm <walter.beller.morales@gmail.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
Walter Beller-Morales
2026-07-27 11:30:37 +08:00
committed by GitHub
co-authored by mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent bf4f633b4c
commit 8040ef2426
5 changed files with 114 additions and 0 deletions
+13
View File
@@ -299,6 +299,11 @@ class SamplingParams(
include_stop_str_in_output: bool = False
"""Whether to include the stop strings in output text."""
output_kind: RequestOutputKind = RequestOutputKind.CUMULATIVE
stream_interval: int | None = None
"""Number of newly generated tokens to batch into each streamed
`RequestOutput`. Raises the interval above the engine-level
`--stream-interval`. Values below engine setting are clamped up to it.
The first and final outputs are always emitted immediately."""
skip_clone: bool = False
"""Internal flag indicating that this SamplingParams instance is safe to
reuse without cloning. When True, clone() will return self without
@@ -377,6 +382,7 @@ class SamplingParams(
skip_special_tokens: bool = True,
spaces_between_special_tokens: bool = True,
output_kind: RequestOutputKind = RequestOutputKind.CUMULATIVE,
stream_interval: int | None = None,
structured_outputs: StructuredOutputsParams | None = None,
logit_bias: dict[int, float] | dict[str, float] | None = None,
allowed_token_ids: list[int] | None = None,
@@ -439,6 +445,7 @@ class SamplingParams(
skip_special_tokens=skip_special_tokens,
spaces_between_special_tokens=spaces_between_special_tokens,
output_kind=output_kind,
stream_interval=stream_interval,
structured_outputs=structured_outputs,
logit_bias=logit_bias,
allowed_token_ids=allowed_token_ids,
@@ -585,6 +592,12 @@ class SamplingParams(
f"min_tokens must be less than or equal to "
f"max_tokens={self.max_tokens}, got {self.min_tokens}."
)
if self.stream_interval is not None and self.stream_interval < 1:
raise VLLMValidationError(
f"stream_interval must be at least 1, got {self.stream_interval}.",
parameter="stream_interval",
value=self.stream_interval,
)
if self.logprobs is not None and self.logprobs != -1 and self.logprobs < 0:
raise VLLMValidationError(
f"logprobs must be non-negative or -1, got {self.logprobs}.",