diff --git a/tests/v1/engine/test_output_processor.py b/tests/v1/engine/test_output_processor.py index 1919349790..51dbb9c989 100644 --- a/tests/v1/engine/test_output_processor.py +++ b/tests/v1/engine/test_output_processor.py @@ -141,6 +141,82 @@ def test_incremental_detokenization( assert not output_processor.has_unfinished_requests() +def test_request_stream_interval_raises_but_not_below_engine_default( + dummy_test_vectors, +): + """A per-request stream_interval can raise the interval above the engine + default but not below it (values under the default clamp up), without + altering the generated text.""" + engine_stream_interval = 5 + # Request 0 (below the default) clamps up to 5; request 1 raises it to 10. + request_stream_intervals = [1, 10] + output_processor = OutputProcessor( + dummy_test_vectors.tokenizer, + log_stats=False, + stream_interval=engine_stream_interval, + ) + + requests = [ + EngineCoreRequest( + request_id=f"request-{idx}-int", + external_req_id=f"request-{idx}", + prompt_token_ids=prompt_tokens, + mm_features=None, + arrival_time=0, + lora_request=None, + cache_salt=None, + data_parallel_rank=None, + sampling_params=SamplingParams( + skip_special_tokens=False, + spaces_between_special_tokens=False, + output_kind=RequestOutputKind.DELTA, + stop=[], + include_stop_str_in_output=False, + stream_interval=request_stream_intervals[idx], + ), + pooling_params=None, + ) + for idx, prompt_tokens in enumerate( + dummy_test_vectors.prompt_tokens[: len(request_stream_intervals)] + ) + ] + + num_requests = len(requests) + engine_core = MockEngineCore( + tokens_list=dummy_test_vectors.generation_tokens[:num_requests], + prompts_list=dummy_test_vectors.prompt_tokens[:num_requests], + request_ids=[req.request_id for req in requests], + ) + + for request, prompt in zip(requests, dummy_test_vectors.prompt_strings): + output_processor.add_request(request, prompt) + + gen_strings: dict[str, str] = {} + gen_tokens: dict[str, list[int]] = {} + while outputs := engine_core.get_outputs(): + for request_output in output_processor.process_outputs(outputs).request_outputs: + request_id = request_output.request_id + new_tokens = request_output.outputs[0].token_ids + if request_id not in gen_strings: + gen_strings[request_id] = request_output.outputs[0].text + gen_tokens[request_id] = list(new_tokens) + assert len(new_tokens) == 1, f"{len(new_tokens)=}" + continue + gen_strings[request_id] += request_output.outputs[0].text + gen_tokens[request_id].extend(new_tokens) + if not request_output.finished: + requested = request_stream_intervals[int(request_id.split("-")[1])] + interval = max(requested, engine_stream_interval) + assert len(new_tokens) == interval, f"{len(new_tokens)=}, {interval=}" + + for idx in range(num_requests): + request_id = f"request-{idx}" + assert gen_strings[request_id] == dummy_test_vectors.generation_strings[idx] + assert gen_tokens[request_id] == dummy_test_vectors.generation_tokens[idx] + + assert not output_processor.has_unfinished_requests() + + def _validate_logprobs( gen_tokens: dict[str, list[int]], gen_logprobs: dict[str, SampleLogprobs | None], diff --git a/vllm/entrypoints/openai/chat_completion/protocol.py b/vllm/entrypoints/openai/chat_completion/protocol.py index 3b65faa5aa..505bd01d4c 100644 --- a/vllm/entrypoints/openai/chat_completion/protocol.py +++ b/vllm/entrypoints/openai/chat_completion/protocol.py @@ -476,6 +476,16 @@ class ChatCompletionRequest(OpenAIBaseModel): "can detect such behavior and terminate early, saving time and tokens.", ) + stream_interval: Annotated[int, Field(ge=1)] | None = Field( + default=None, + description=( + "Number of tokens to batch into each streamed chunk. Raises the " + "server's `--stream-interval` for this request. Values below the " + "server setting are clamped up to it. The first and last chunks " + "are always sent immediately. Ignored for non-streaming requests." + ), + ) + # --8<-- [end:chat-completion-extra-params] @model_validator(mode="before") @@ -690,6 +700,7 @@ class ChatCompletionRequest(OpenAIBaseModel): output_kind=( RequestOutputKind.DELTA if self.stream else RequestOutputKind.FINAL_ONLY ), + stream_interval=self.stream_interval, structured_outputs=self.extract_structured_outputs(), logit_bias=self.logit_bias, bad_words=self.bad_words, diff --git a/vllm/entrypoints/openai/completion/protocol.py b/vllm/entrypoints/openai/completion/protocol.py index 6a2bf47bb1..79ad9799e1 100644 --- a/vllm/entrypoints/openai/completion/protocol.py +++ b/vllm/entrypoints/openai/completion/protocol.py @@ -233,6 +233,16 @@ class CompletionRequest(OpenAIBaseModel): ), ) + stream_interval: Annotated[int, Field(ge=1)] | None = Field( + default=None, + description=( + "Number of tokens to batch into each streamed chunk. Raises the " + "server's `--stream-interval` for this request. Values below the " + "server setting are clamped up to it. The first and last chunks " + "are always sent immediately. Ignored for non-streaming requests." + ), + ) + # --8<-- [end:completion-extra-params] def build_tok_params(self, model_config: ModelConfig) -> TokenizeParams: @@ -365,6 +375,7 @@ class CompletionRequest(OpenAIBaseModel): output_kind=RequestOutputKind.DELTA if self.stream else RequestOutputKind.FINAL_ONLY, + stream_interval=self.stream_interval, structured_outputs=self.extract_structured_outputs(), logit_bias=self.logit_bias, allowed_token_ids=self.allowed_token_ids, diff --git a/vllm/sampling_params.py b/vllm/sampling_params.py index 08580f6e8f..25e36ceb56 100644 --- a/vllm/sampling_params.py +++ b/vllm/sampling_params.py @@ -299,6 +299,11 @@ class SamplingParams( include_stop_str_in_output: bool = False """Whether to include the stop strings in output text.""" output_kind: RequestOutputKind = RequestOutputKind.CUMULATIVE + stream_interval: int | None = None + """Number of newly generated tokens to batch into each streamed + `RequestOutput`. Raises the interval above the engine-level + `--stream-interval`. Values below engine setting are clamped up to it. + The first and final outputs are always emitted immediately.""" skip_clone: bool = False """Internal flag indicating that this SamplingParams instance is safe to reuse without cloning. When True, clone() will return self without @@ -377,6 +382,7 @@ class SamplingParams( skip_special_tokens: bool = True, spaces_between_special_tokens: bool = True, output_kind: RequestOutputKind = RequestOutputKind.CUMULATIVE, + stream_interval: int | None = None, structured_outputs: StructuredOutputsParams | None = None, logit_bias: dict[int, float] | dict[str, float] | None = None, allowed_token_ids: list[int] | None = None, @@ -439,6 +445,7 @@ class SamplingParams( skip_special_tokens=skip_special_tokens, spaces_between_special_tokens=spaces_between_special_tokens, output_kind=output_kind, + stream_interval=stream_interval, structured_outputs=structured_outputs, logit_bias=logit_bias, allowed_token_ids=allowed_token_ids, @@ -585,6 +592,12 @@ class SamplingParams( f"min_tokens must be less than or equal to " f"max_tokens={self.max_tokens}, got {self.min_tokens}." ) + if self.stream_interval is not None and self.stream_interval < 1: + raise VLLMValidationError( + f"stream_interval must be at least 1, got {self.stream_interval}.", + parameter="stream_interval", + value=self.stream_interval, + ) if self.logprobs is not None and self.logprobs != -1 and self.logprobs < 0: raise VLLMValidationError( f"logprobs must be non-negative or -1, got {self.logprobs}.", diff --git a/vllm/v1/engine/output_processor.py b/vllm/v1/engine/output_processor.py index 0d7d5fe18c..c467703759 100644 --- a/vllm/v1/engine/output_processor.py +++ b/vllm/v1/engine/output_processor.py @@ -224,6 +224,9 @@ class RequestState: if not sampling_params.detokenize: tokenizer = None output_kind = sampling_params.output_kind + if sampling_params.stream_interval is not None: + # clamp to the engine-level stream interval. + stream_interval = max(sampling_params.stream_interval, stream_interval) logprobs_processor = LogprobsProcessor.from_new_request( tokenizer=tokenizer, request=request,