[Bugfix][Frontend] Preserve streaming chunk boundaries in RequestOutputCollector (#58941)

Signed-off-by: David Cheung <d7cheung@gmail.com>
Co-authored-by: Benedikt Falk <bfalk@nvidia.com>
Co-authored-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
David Cheung
2026-10-15 16:38:28 +00:00
committed by GitHub
co-authored by Benedikt Falk Nick Hill
parent a9e90b9d1d
commit 957d512e96
3 changed files with 103 additions and 2 deletions
+47
View File
@@ -113,6 +113,53 @@ class RequestOutputCollector:
self._input_stream_task = None
def _is_chunk_boundary(
output: RequestOutput | PoolingRequestOutput | Exception | None,
) -> bool:
"""Whether the output marks the end of a streaming-input chunk: a
finish_reason on a request that has not finished."""
return (
isinstance(output, RequestOutput)
and not output.finished
and any(completion.finished() for completion in output.outputs)
)
class StreamingInputOutputCollector(RequestOutputCollector):
"""Collector for streaming-input requests. Doesn't merge across input
chunk boundaries (a finish_reason on a non-finished output)."""
def __init__(self, output_kind: RequestOutputKind, request_id: str):
super().__init__(output_kind, request_id)
self._backlog: deque[RequestOutput] = deque()
def put(self, output: RequestOutput | PoolingRequestOutput | Exception) -> None:
if isinstance(output, Exception):
self._backlog.clear()
elif self._backlog or _is_chunk_boundary(self.output):
assert isinstance(output, RequestOutput)
if self._backlog and not _is_chunk_boundary(self._backlog[-1]):
self._backlog[-1].add(output, aggregate=self.aggregate)
else:
self._backlog.append(output)
return
super().put(output)
async def get(self) -> RequestOutput | PoolingRequestOutput:
output = await super().get()
if self._backlog:
self.output = self._backlog.popleft()
self.ready.set()
return output
def get_nowait(self) -> RequestOutput | PoolingRequestOutput | None:
output = super().get_nowait()
if self._backlog:
self.output = self._backlog.popleft()
self.ready.set()
return output
@dataclass
class OutputProcessorOutput:
request_outputs: list[RequestOutput | PoolingRequestOutput]