[Bugfix][Frontend] Preserve streaming chunk boundaries in RequestOutputCollector (#58941)
Signed-off-by: David Cheung <d7cheung@gmail.com> Co-authored-by: Benedikt Falk <bfalk@nvidia.com> Co-authored-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
co-authored by
Benedikt Falk
Nick Hill
parent
a9e90b9d1d
commit
957d512e96
@@ -42,6 +42,7 @@ from vllm.v1.engine.output_processor import (
|
||||
OutputProcessor,
|
||||
RequestOutputCollector,
|
||||
RequestState,
|
||||
StreamingInputOutputCollector,
|
||||
)
|
||||
from vllm.v1.engine.parallel_sampling import ParentRequest
|
||||
from vllm.v1.metrics.loggers import PrometheusStatLogger
|
||||
@@ -1642,6 +1643,53 @@ async def test_cumulative_output_collector_n():
|
||||
assert third[0].text == "c"
|
||||
|
||||
|
||||
def _make_boundary_output(
|
||||
token_id: int, finish_reason: str | None, finished: bool = False
|
||||
) -> RequestOutput:
|
||||
return RequestOutput(
|
||||
request_id="my-request-id",
|
||||
prompt=None,
|
||||
prompt_token_ids=[1, 2, 3],
|
||||
prompt_logprobs=None,
|
||||
outputs=[
|
||||
CompletionOutput(
|
||||
index=0,
|
||||
text="a",
|
||||
token_ids=[token_id],
|
||||
cumulative_logprob=None,
|
||||
logprobs=None,
|
||||
finish_reason=finish_reason,
|
||||
)
|
||||
],
|
||||
finished=finished,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_streaming_input_collector_preserves_chunk_boundary():
|
||||
"""A pending chunk-boundary marker (finish_reason on a non-finished
|
||||
output) must not be erased by merging the next chunk's output into it."""
|
||||
collector = StreamingInputOutputCollector(
|
||||
RequestOutputKind.DELTA, request_id="my-request-id-int"
|
||||
)
|
||||
|
||||
collector.put(_make_boundary_output(0, "stop"))
|
||||
collector.put(_make_boundary_output(1, None))
|
||||
collector.put(_make_boundary_output(2, None))
|
||||
|
||||
boundary = await collector.get()
|
||||
assert boundary.outputs[0].finish_reason == "stop"
|
||||
assert boundary.outputs[0].token_ids == [0]
|
||||
|
||||
following = await collector.get()
|
||||
assert following.outputs[0].finish_reason is None
|
||||
assert following.outputs[0].token_ids == [1, 2]
|
||||
|
||||
assert collector.output is None
|
||||
assert not collector._backlog
|
||||
assert not collector.ready.is_set()
|
||||
|
||||
|
||||
def _make_streaming_input_request(
|
||||
request_id: str, prompt_token_ids: list[int]
|
||||
) -> EngineCoreRequest:
|
||||
|
||||
Reference in New Issue
Block a user