[Frontend] expose stream_interval as req sampling param (#49754)
Signed-off-by: walterbm <walter.beller.morales@gmail.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
bf4f633b4c
commit
8040ef2426
@@ -141,6 +141,82 @@ def test_incremental_detokenization(
|
||||
assert not output_processor.has_unfinished_requests()
|
||||
|
||||
|
||||
def test_request_stream_interval_raises_but_not_below_engine_default(
|
||||
dummy_test_vectors,
|
||||
):
|
||||
"""A per-request stream_interval can raise the interval above the engine
|
||||
default but not below it (values under the default clamp up), without
|
||||
altering the generated text."""
|
||||
engine_stream_interval = 5
|
||||
# Request 0 (below the default) clamps up to 5; request 1 raises it to 10.
|
||||
request_stream_intervals = [1, 10]
|
||||
output_processor = OutputProcessor(
|
||||
dummy_test_vectors.tokenizer,
|
||||
log_stats=False,
|
||||
stream_interval=engine_stream_interval,
|
||||
)
|
||||
|
||||
requests = [
|
||||
EngineCoreRequest(
|
||||
request_id=f"request-{idx}-int",
|
||||
external_req_id=f"request-{idx}",
|
||||
prompt_token_ids=prompt_tokens,
|
||||
mm_features=None,
|
||||
arrival_time=0,
|
||||
lora_request=None,
|
||||
cache_salt=None,
|
||||
data_parallel_rank=None,
|
||||
sampling_params=SamplingParams(
|
||||
skip_special_tokens=False,
|
||||
spaces_between_special_tokens=False,
|
||||
output_kind=RequestOutputKind.DELTA,
|
||||
stop=[],
|
||||
include_stop_str_in_output=False,
|
||||
stream_interval=request_stream_intervals[idx],
|
||||
),
|
||||
pooling_params=None,
|
||||
)
|
||||
for idx, prompt_tokens in enumerate(
|
||||
dummy_test_vectors.prompt_tokens[: len(request_stream_intervals)]
|
||||
)
|
||||
]
|
||||
|
||||
num_requests = len(requests)
|
||||
engine_core = MockEngineCore(
|
||||
tokens_list=dummy_test_vectors.generation_tokens[:num_requests],
|
||||
prompts_list=dummy_test_vectors.prompt_tokens[:num_requests],
|
||||
request_ids=[req.request_id for req in requests],
|
||||
)
|
||||
|
||||
for request, prompt in zip(requests, dummy_test_vectors.prompt_strings):
|
||||
output_processor.add_request(request, prompt)
|
||||
|
||||
gen_strings: dict[str, str] = {}
|
||||
gen_tokens: dict[str, list[int]] = {}
|
||||
while outputs := engine_core.get_outputs():
|
||||
for request_output in output_processor.process_outputs(outputs).request_outputs:
|
||||
request_id = request_output.request_id
|
||||
new_tokens = request_output.outputs[0].token_ids
|
||||
if request_id not in gen_strings:
|
||||
gen_strings[request_id] = request_output.outputs[0].text
|
||||
gen_tokens[request_id] = list(new_tokens)
|
||||
assert len(new_tokens) == 1, f"{len(new_tokens)=}"
|
||||
continue
|
||||
gen_strings[request_id] += request_output.outputs[0].text
|
||||
gen_tokens[request_id].extend(new_tokens)
|
||||
if not request_output.finished:
|
||||
requested = request_stream_intervals[int(request_id.split("-")[1])]
|
||||
interval = max(requested, engine_stream_interval)
|
||||
assert len(new_tokens) == interval, f"{len(new_tokens)=}, {interval=}"
|
||||
|
||||
for idx in range(num_requests):
|
||||
request_id = f"request-{idx}"
|
||||
assert gen_strings[request_id] == dummy_test_vectors.generation_strings[idx]
|
||||
assert gen_tokens[request_id] == dummy_test_vectors.generation_tokens[idx]
|
||||
|
||||
assert not output_processor.has_unfinished_requests()
|
||||
|
||||
|
||||
def _validate_logprobs(
|
||||
gen_tokens: dict[str, list[int]],
|
||||
gen_logprobs: dict[str, SampleLogprobs | None],
|
||||
|
||||
@@ -476,6 +476,16 @@ class ChatCompletionRequest(OpenAIBaseModel):
|
||||
"can detect such behavior and terminate early, saving time and tokens.",
|
||||
)
|
||||
|
||||
stream_interval: Annotated[int, Field(ge=1)] | None = Field(
|
||||
default=None,
|
||||
description=(
|
||||
"Number of tokens to batch into each streamed chunk. Raises the "
|
||||
"server's `--stream-interval` for this request. Values below the "
|
||||
"server setting are clamped up to it. The first and last chunks "
|
||||
"are always sent immediately. Ignored for non-streaming requests."
|
||||
),
|
||||
)
|
||||
|
||||
# --8<-- [end:chat-completion-extra-params]
|
||||
|
||||
@model_validator(mode="before")
|
||||
@@ -690,6 +700,7 @@ class ChatCompletionRequest(OpenAIBaseModel):
|
||||
output_kind=(
|
||||
RequestOutputKind.DELTA if self.stream else RequestOutputKind.FINAL_ONLY
|
||||
),
|
||||
stream_interval=self.stream_interval,
|
||||
structured_outputs=self.extract_structured_outputs(),
|
||||
logit_bias=self.logit_bias,
|
||||
bad_words=self.bad_words,
|
||||
|
||||
@@ -233,6 +233,16 @@ class CompletionRequest(OpenAIBaseModel):
|
||||
),
|
||||
)
|
||||
|
||||
stream_interval: Annotated[int, Field(ge=1)] | None = Field(
|
||||
default=None,
|
||||
description=(
|
||||
"Number of tokens to batch into each streamed chunk. Raises the "
|
||||
"server's `--stream-interval` for this request. Values below the "
|
||||
"server setting are clamped up to it. The first and last chunks "
|
||||
"are always sent immediately. Ignored for non-streaming requests."
|
||||
),
|
||||
)
|
||||
|
||||
# --8<-- [end:completion-extra-params]
|
||||
|
||||
def build_tok_params(self, model_config: ModelConfig) -> TokenizeParams:
|
||||
@@ -365,6 +375,7 @@ class CompletionRequest(OpenAIBaseModel):
|
||||
output_kind=RequestOutputKind.DELTA
|
||||
if self.stream
|
||||
else RequestOutputKind.FINAL_ONLY,
|
||||
stream_interval=self.stream_interval,
|
||||
structured_outputs=self.extract_structured_outputs(),
|
||||
logit_bias=self.logit_bias,
|
||||
allowed_token_ids=self.allowed_token_ids,
|
||||
|
||||
@@ -299,6 +299,11 @@ class SamplingParams(
|
||||
include_stop_str_in_output: bool = False
|
||||
"""Whether to include the stop strings in output text."""
|
||||
output_kind: RequestOutputKind = RequestOutputKind.CUMULATIVE
|
||||
stream_interval: int | None = None
|
||||
"""Number of newly generated tokens to batch into each streamed
|
||||
`RequestOutput`. Raises the interval above the engine-level
|
||||
`--stream-interval`. Values below engine setting are clamped up to it.
|
||||
The first and final outputs are always emitted immediately."""
|
||||
skip_clone: bool = False
|
||||
"""Internal flag indicating that this SamplingParams instance is safe to
|
||||
reuse without cloning. When True, clone() will return self without
|
||||
@@ -377,6 +382,7 @@ class SamplingParams(
|
||||
skip_special_tokens: bool = True,
|
||||
spaces_between_special_tokens: bool = True,
|
||||
output_kind: RequestOutputKind = RequestOutputKind.CUMULATIVE,
|
||||
stream_interval: int | None = None,
|
||||
structured_outputs: StructuredOutputsParams | None = None,
|
||||
logit_bias: dict[int, float] | dict[str, float] | None = None,
|
||||
allowed_token_ids: list[int] | None = None,
|
||||
@@ -439,6 +445,7 @@ class SamplingParams(
|
||||
skip_special_tokens=skip_special_tokens,
|
||||
spaces_between_special_tokens=spaces_between_special_tokens,
|
||||
output_kind=output_kind,
|
||||
stream_interval=stream_interval,
|
||||
structured_outputs=structured_outputs,
|
||||
logit_bias=logit_bias,
|
||||
allowed_token_ids=allowed_token_ids,
|
||||
@@ -585,6 +592,12 @@ class SamplingParams(
|
||||
f"min_tokens must be less than or equal to "
|
||||
f"max_tokens={self.max_tokens}, got {self.min_tokens}."
|
||||
)
|
||||
if self.stream_interval is not None and self.stream_interval < 1:
|
||||
raise VLLMValidationError(
|
||||
f"stream_interval must be at least 1, got {self.stream_interval}.",
|
||||
parameter="stream_interval",
|
||||
value=self.stream_interval,
|
||||
)
|
||||
if self.logprobs is not None and self.logprobs != -1 and self.logprobs < 0:
|
||||
raise VLLMValidationError(
|
||||
f"logprobs must be non-negative or -1, got {self.logprobs}.",
|
||||
|
||||
@@ -224,6 +224,9 @@ class RequestState:
|
||||
if not sampling_params.detokenize:
|
||||
tokenizer = None
|
||||
output_kind = sampling_params.output_kind
|
||||
if sampling_params.stream_interval is not None:
|
||||
# clamp to the engine-level stream interval.
|
||||
stream_interval = max(sampling_params.stream_interval, stream_interval)
|
||||
logprobs_processor = LogprobsProcessor.from_new_request(
|
||||
tokenizer=tokenizer,
|
||||
request=request,
|
||||
|
||||
Reference in New Issue
Block a user