[Frontend] expose stream_interval as req sampling param (#49754)

Signed-off-by: walterbm <walter.beller.morales@gmail.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
Walter Beller-Morales
2026-07-27 11:30:37 +08:00
committed by GitHub
co-authored by mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent bf4f633b4c
commit 8040ef2426
5 changed files with 114 additions and 0 deletions
+76
View File
@@ -141,6 +141,82 @@ def test_incremental_detokenization(
assert not output_processor.has_unfinished_requests()
def test_request_stream_interval_raises_but_not_below_engine_default(
dummy_test_vectors,
):
"""A per-request stream_interval can raise the interval above the engine
default but not below it (values under the default clamp up), without
altering the generated text."""
engine_stream_interval = 5
# Request 0 (below the default) clamps up to 5; request 1 raises it to 10.
request_stream_intervals = [1, 10]
output_processor = OutputProcessor(
dummy_test_vectors.tokenizer,
log_stats=False,
stream_interval=engine_stream_interval,
)
requests = [
EngineCoreRequest(
request_id=f"request-{idx}-int",
external_req_id=f"request-{idx}",
prompt_token_ids=prompt_tokens,
mm_features=None,
arrival_time=0,
lora_request=None,
cache_salt=None,
data_parallel_rank=None,
sampling_params=SamplingParams(
skip_special_tokens=False,
spaces_between_special_tokens=False,
output_kind=RequestOutputKind.DELTA,
stop=[],
include_stop_str_in_output=False,
stream_interval=request_stream_intervals[idx],
),
pooling_params=None,
)
for idx, prompt_tokens in enumerate(
dummy_test_vectors.prompt_tokens[: len(request_stream_intervals)]
)
]
num_requests = len(requests)
engine_core = MockEngineCore(
tokens_list=dummy_test_vectors.generation_tokens[:num_requests],
prompts_list=dummy_test_vectors.prompt_tokens[:num_requests],
request_ids=[req.request_id for req in requests],
)
for request, prompt in zip(requests, dummy_test_vectors.prompt_strings):
output_processor.add_request(request, prompt)
gen_strings: dict[str, str] = {}
gen_tokens: dict[str, list[int]] = {}
while outputs := engine_core.get_outputs():
for request_output in output_processor.process_outputs(outputs).request_outputs:
request_id = request_output.request_id
new_tokens = request_output.outputs[0].token_ids
if request_id not in gen_strings:
gen_strings[request_id] = request_output.outputs[0].text
gen_tokens[request_id] = list(new_tokens)
assert len(new_tokens) == 1, f"{len(new_tokens)=}"
continue
gen_strings[request_id] += request_output.outputs[0].text
gen_tokens[request_id].extend(new_tokens)
if not request_output.finished:
requested = request_stream_intervals[int(request_id.split("-")[1])]
interval = max(requested, engine_stream_interval)
assert len(new_tokens) == interval, f"{len(new_tokens)=}, {interval=}"
for idx in range(num_requests):
request_id = f"request-{idx}"
assert gen_strings[request_id] == dummy_test_vectors.generation_strings[idx]
assert gen_tokens[request_id] == dummy_test_vectors.generation_tokens[idx]
assert not output_processor.has_unfinished_requests()
def _validate_logprobs(
gen_tokens: dict[str, list[int]],
gen_logprobs: dict[str, SampleLogprobs | None],