[Bugfix][Frontend] Preserve streaming chunk boundaries in RequestOutputCollector (#58941)

Signed-off-by: David Cheung <d7cheung@gmail.com>
Co-authored-by: Benedikt Falk <bfalk@nvidia.com>
Co-authored-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
David Cheung
2026-10-15 16:38:28 +00:00
committed by GitHub
co-authored by Benedikt Falk Nick Hill
parent a9e90b9d1d
commit 957d512e96
3 changed files with 103 additions and 2 deletions
+48
View File
@@ -42,6 +42,7 @@ from vllm.v1.engine.output_processor import (
OutputProcessor,
RequestOutputCollector,
RequestState,
StreamingInputOutputCollector,
)
from vllm.v1.engine.parallel_sampling import ParentRequest
from vllm.v1.metrics.loggers import PrometheusStatLogger
@@ -1642,6 +1643,53 @@ async def test_cumulative_output_collector_n():
assert third[0].text == "c"
def _make_boundary_output(
token_id: int, finish_reason: str | None, finished: bool = False
) -> RequestOutput:
return RequestOutput(
request_id="my-request-id",
prompt=None,
prompt_token_ids=[1, 2, 3],
prompt_logprobs=None,
outputs=[
CompletionOutput(
index=0,
text="a",
token_ids=[token_id],
cumulative_logprob=None,
logprobs=None,
finish_reason=finish_reason,
)
],
finished=finished,
)
@pytest.mark.asyncio
async def test_streaming_input_collector_preserves_chunk_boundary():
"""A pending chunk-boundary marker (finish_reason on a non-finished
output) must not be erased by merging the next chunk's output into it."""
collector = StreamingInputOutputCollector(
RequestOutputKind.DELTA, request_id="my-request-id-int"
)
collector.put(_make_boundary_output(0, "stop"))
collector.put(_make_boundary_output(1, None))
collector.put(_make_boundary_output(2, None))
boundary = await collector.get()
assert boundary.outputs[0].finish_reason == "stop"
assert boundary.outputs[0].token_ids == [0]
following = await collector.get()
assert following.outputs[0].finish_reason is None
assert following.outputs[0].token_ids == [1, 2]
assert collector.output is None
assert not collector._backlog
assert not collector.ready.is_set()
def _make_streaming_input_request(
request_id: str, prompt_token_ids: list[int]
) -> EngineCoreRequest: