[Frontend] expose stream_interval as req sampling param (#49754)

Signed-off-by: walterbm <walter.beller.morales@gmail.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
Walter Beller-Morales
2026-07-27 11:30:37 +08:00
committed by GitHub
co-authored by mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent bf4f633b4c
commit 8040ef2426
5 changed files with 114 additions and 0 deletions
+76
View File
@@ -141,6 +141,82 @@ def test_incremental_detokenization(
assert not output_processor.has_unfinished_requests()
def test_request_stream_interval_raises_but_not_below_engine_default(
dummy_test_vectors,
):
"""A per-request stream_interval can raise the interval above the engine
default but not below it (values under the default clamp up), without
altering the generated text."""
engine_stream_interval = 5
# Request 0 (below the default) clamps up to 5; request 1 raises it to 10.
request_stream_intervals = [1, 10]
output_processor = OutputProcessor(
dummy_test_vectors.tokenizer,
log_stats=False,
stream_interval=engine_stream_interval,
)
requests = [
EngineCoreRequest(
request_id=f"request-{idx}-int",
external_req_id=f"request-{idx}",
prompt_token_ids=prompt_tokens,
mm_features=None,
arrival_time=0,
lora_request=None,
cache_salt=None,
data_parallel_rank=None,
sampling_params=SamplingParams(
skip_special_tokens=False,
spaces_between_special_tokens=False,
output_kind=RequestOutputKind.DELTA,
stop=[],
include_stop_str_in_output=False,
stream_interval=request_stream_intervals[idx],
),
pooling_params=None,
)
for idx, prompt_tokens in enumerate(
dummy_test_vectors.prompt_tokens[: len(request_stream_intervals)]
)
]
num_requests = len(requests)
engine_core = MockEngineCore(
tokens_list=dummy_test_vectors.generation_tokens[:num_requests],
prompts_list=dummy_test_vectors.prompt_tokens[:num_requests],
request_ids=[req.request_id for req in requests],
)
for request, prompt in zip(requests, dummy_test_vectors.prompt_strings):
output_processor.add_request(request, prompt)
gen_strings: dict[str, str] = {}
gen_tokens: dict[str, list[int]] = {}
while outputs := engine_core.get_outputs():
for request_output in output_processor.process_outputs(outputs).request_outputs:
request_id = request_output.request_id
new_tokens = request_output.outputs[0].token_ids
if request_id not in gen_strings:
gen_strings[request_id] = request_output.outputs[0].text
gen_tokens[request_id] = list(new_tokens)
assert len(new_tokens) == 1, f"{len(new_tokens)=}"
continue
gen_strings[request_id] += request_output.outputs[0].text
gen_tokens[request_id].extend(new_tokens)
if not request_output.finished:
requested = request_stream_intervals[int(request_id.split("-")[1])]
interval = max(requested, engine_stream_interval)
assert len(new_tokens) == interval, f"{len(new_tokens)=}, {interval=}"
for idx in range(num_requests):
request_id = f"request-{idx}"
assert gen_strings[request_id] == dummy_test_vectors.generation_strings[idx]
assert gen_tokens[request_id] == dummy_test_vectors.generation_tokens[idx]
assert not output_processor.has_unfinished_requests()
def _validate_logprobs(
gen_tokens: dict[str, list[int]],
gen_logprobs: dict[str, SampleLogprobs | None],
@@ -476,6 +476,16 @@ class ChatCompletionRequest(OpenAIBaseModel):
"can detect such behavior and terminate early, saving time and tokens.",
)
stream_interval: Annotated[int, Field(ge=1)] | None = Field(
default=None,
description=(
"Number of tokens to batch into each streamed chunk. Raises the "
"server's `--stream-interval` for this request. Values below the "
"server setting are clamped up to it. The first and last chunks "
"are always sent immediately. Ignored for non-streaming requests."
),
)
# --8<-- [end:chat-completion-extra-params]
@model_validator(mode="before")
@@ -690,6 +700,7 @@ class ChatCompletionRequest(OpenAIBaseModel):
output_kind=(
RequestOutputKind.DELTA if self.stream else RequestOutputKind.FINAL_ONLY
),
stream_interval=self.stream_interval,
structured_outputs=self.extract_structured_outputs(),
logit_bias=self.logit_bias,
bad_words=self.bad_words,
@@ -233,6 +233,16 @@ class CompletionRequest(OpenAIBaseModel):
),
)
stream_interval: Annotated[int, Field(ge=1)] | None = Field(
default=None,
description=(
"Number of tokens to batch into each streamed chunk. Raises the "
"server's `--stream-interval` for this request. Values below the "
"server setting are clamped up to it. The first and last chunks "
"are always sent immediately. Ignored for non-streaming requests."
),
)
# --8<-- [end:completion-extra-params]
def build_tok_params(self, model_config: ModelConfig) -> TokenizeParams:
@@ -365,6 +375,7 @@ class CompletionRequest(OpenAIBaseModel):
output_kind=RequestOutputKind.DELTA
if self.stream
else RequestOutputKind.FINAL_ONLY,
stream_interval=self.stream_interval,
structured_outputs=self.extract_structured_outputs(),
logit_bias=self.logit_bias,
allowed_token_ids=self.allowed_token_ids,
+13
View File
@@ -299,6 +299,11 @@ class SamplingParams(
include_stop_str_in_output: bool = False
"""Whether to include the stop strings in output text."""
output_kind: RequestOutputKind = RequestOutputKind.CUMULATIVE
stream_interval: int | None = None
"""Number of newly generated tokens to batch into each streamed
`RequestOutput`. Raises the interval above the engine-level
`--stream-interval`. Values below engine setting are clamped up to it.
The first and final outputs are always emitted immediately."""
skip_clone: bool = False
"""Internal flag indicating that this SamplingParams instance is safe to
reuse without cloning. When True, clone() will return self without
@@ -377,6 +382,7 @@ class SamplingParams(
skip_special_tokens: bool = True,
spaces_between_special_tokens: bool = True,
output_kind: RequestOutputKind = RequestOutputKind.CUMULATIVE,
stream_interval: int | None = None,
structured_outputs: StructuredOutputsParams | None = None,
logit_bias: dict[int, float] | dict[str, float] | None = None,
allowed_token_ids: list[int] | None = None,
@@ -439,6 +445,7 @@ class SamplingParams(
skip_special_tokens=skip_special_tokens,
spaces_between_special_tokens=spaces_between_special_tokens,
output_kind=output_kind,
stream_interval=stream_interval,
structured_outputs=structured_outputs,
logit_bias=logit_bias,
allowed_token_ids=allowed_token_ids,
@@ -585,6 +592,12 @@ class SamplingParams(
f"min_tokens must be less than or equal to "
f"max_tokens={self.max_tokens}, got {self.min_tokens}."
)
if self.stream_interval is not None and self.stream_interval < 1:
raise VLLMValidationError(
f"stream_interval must be at least 1, got {self.stream_interval}.",
parameter="stream_interval",
value=self.stream_interval,
)
if self.logprobs is not None and self.logprobs != -1 and self.logprobs < 0:
raise VLLMValidationError(
f"logprobs must be non-negative or -1, got {self.logprobs}.",
+3
View File
@@ -224,6 +224,9 @@ class RequestState:
if not sampling_params.detokenize:
tokenizer = None
output_kind = sampling_params.output_kind
if sampling_params.stream_interval is not None:
# clamp to the engine-level stream interval.
stream_interval = max(sampling_params.stream_interval, stream_interval)
logprobs_processor = LogprobsProcessor.from_new_request(
tokenizer=tokenizer,
request=request,