[Feature] add session based streaming input support to v1 (#28973)

Signed-off-by: Joshua Deng <joshuakdeng@gmail.com>
Signed-off-by: Patrick von Platen <patrick.v.platen@gmail.com>
Signed-off-by: Nick Hill <nickhill123@gmail.com>
Signed-off-by: Roger Wang <hey@rogerw.io>
Co-authored-by: Roger Wang <hey@rogerw.io>
Co-authored-by: Patrick von Platen <patrick.v.platen@gmail.com>
Co-authored-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
Joshua Deng
2026-01-24 12:06:28 -08:00
committed by GitHub
co-authored by Roger Wang Patrick von Platen Nick Hill
parent d4dbb7af63
commit 91601ff478
16 changed files with 2151 additions and 63 deletions
+130 -27
View File
@@ -2,7 +2,7 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import asyncio
from collections import defaultdict
from collections import defaultdict, deque
from collections.abc import Iterable
from dataclasses import dataclass
from typing import Any, cast
@@ -12,6 +12,7 @@ import torch
from vllm.lora.request import LoRARequest
from vllm.outputs import (
STREAM_FINISHED,
CompletionOutput,
PoolingOutput,
PoolingRequestOutput,
@@ -51,6 +52,8 @@ class RequestOutputCollector:
self.output: RequestOutput | PoolingRequestOutput | Exception | None = None
self.ready = asyncio.Event()
self._input_stream_task: asyncio.Task | None = None
def put(self, output: RequestOutput | PoolingRequestOutput | Exception) -> None:
"""Non-blocking put operation."""
if self.output is None or isinstance(output, Exception):
@@ -87,6 +90,16 @@ class RequestOutputCollector:
raise output
return output
def close(self):
if self._input_stream_task is not None:
self._input_stream_task.cancel()
self._input_stream_task = None
def __del__(self):
if (task := self._input_stream_task) is not None:
task.get_loop().call_soon_threadsafe(task.cancel)
self._input_stream_task = None
@dataclass
class OutputProcessorOutput:
@@ -94,6 +107,20 @@ class OutputProcessorOutput:
reqs_to_abort: list[str]
@dataclass
class StreamingUpdate:
"""Streaming input update data for output processor.
Contains the incremental prompt data to be applied to a request state
when the current sub-request completes.
"""
prompt: str | None
prompt_token_ids: list[int] | None
arrival_time: float
final: bool = False
class RequestState:
def __init__(
self,
@@ -116,6 +143,7 @@ class RequestState:
top_p: float | None = None,
n: int | None = None,
temperature: float | None = None,
stream_input: bool = False,
):
self.request_id = request_id
self.external_req_id = external_req_id
@@ -146,6 +174,31 @@ class RequestState:
self.stream_interval = stream_interval
self.sent_tokens_offset = 0 # Offset of sent tokens
# Streaming input queue
self.streaming_input = stream_input
self.input_chunk_queue: deque[StreamingUpdate] | None = (
deque() if stream_input else None
)
def apply_streaming_update(self, update: StreamingUpdate) -> None:
# Apply the update to the request state.
self.streaming_input = not update.final
# TODO also include relevant output tokens in new prompt here
# (match scheduler behavior).
if update.prompt:
self.prompt = (
(self.prompt + update.prompt) if self.prompt else update.prompt
)
if self.prompt_token_ids:
self.prompt_token_ids.extend(update.prompt_token_ids or ())
else:
self.prompt_token_ids = update.prompt_token_ids or []
assert self.prompt_token_ids is not None
self.prompt_len = len(self.prompt_token_ids)
if self.stats is not None:
self.stats.arrival_time = update.arrival_time
self.is_prefilling = True
@classmethod
def from_new_request(
cls,
@@ -205,6 +258,7 @@ class RequestState:
queue=queue,
log_stats=log_stats,
stream_interval=stream_interval,
stream_input=request.resumable,
)
def make_request_output(
@@ -405,7 +459,6 @@ class OutputProcessor:
a parent request, in which case the associated child requests are aborted
also.
"""
internal_req_ids = []
for request_id in request_ids:
if internal:
@@ -464,8 +517,10 @@ class OutputProcessor:
queue: RequestOutputCollector | None = None,
) -> None:
request_id = request.request_id
if request_id in self.request_states:
raise ValueError(f"Request id {request_id} already running.")
req_state = self.request_states.get(request_id)
if req_state is not None:
self._update_streaming_request_state(req_state, request, prompt)
return
req_state = RequestState.from_new_request(
tokenizer=self.tokenizer,
@@ -486,6 +541,39 @@ class OutputProcessor:
# Track the external_req_id -> [internal_req_id, ...] mapping
self.external_req_ids[req_state.external_req_id].append(request_id)
def _update_streaming_request_state(
self, req_state: RequestState, request: EngineCoreRequest, prompt: str | None
) -> None:
"""Queue a streaming update instead of immediately applying it."""
if not request.resumable:
# Final request - just mark completion, don't add its dummy tokens.
if req_state.input_chunk_queue is None:
# Engine already finished - emit final output and clean up.
self._finish_request(req_state)
if req_state.queue is not None:
# Emit a final output with finished=True
# to unblock the generate() loop.
req_state.queue.put(STREAM_FINISHED)
elif req_state.input_chunk_queue:
req_state.input_chunk_queue[-1].final = True
else:
req_state.streaming_input = False
return
update = StreamingUpdate(
prompt=prompt,
prompt_token_ids=request.prompt_token_ids,
arrival_time=request.arrival_time,
)
# Apply request updates now if the last input already completed.
if req_state.input_chunk_queue is None:
req_state.apply_streaming_update(update)
req_state.input_chunk_queue = deque()
else:
# Queue the streaming update otherwise.
req_state.input_chunk_queue.append(update)
def process_outputs(
self,
engine_core_outputs: list[EngineCoreOutput],
@@ -561,6 +649,9 @@ class OutputProcessor:
kv_transfer_params,
routed_experts,
):
if req_state.streaming_input:
request_output.finished = False
if req_state.queue is not None:
# AsyncLLM: put into queue for handling by generate().
req_state.queue.put(request_output)
@@ -570,36 +661,48 @@ class OutputProcessor:
# Free completed requests.
if finish_reason is not None:
self.request_states.pop(req_id)
if req_state.streaming_input:
if req_state.input_chunk_queue:
update = req_state.input_chunk_queue.popleft()
req_state.apply_streaming_update(update)
else:
req_state.input_chunk_queue = None
else:
self._finish_request(req_state)
if not engine_core_output.finished:
# If req not finished in EngineCore, but Detokenizer
# detected stop string, abort needed in EngineCore.
reqs_to_abort.append(req_id)
internal_ids = self.external_req_ids[req_state.external_req_id]
internal_ids.remove(req_id)
if not internal_ids:
del self.external_req_ids[req_state.external_req_id]
# Remove parent request if applicable.
parent_req = req_state.parent_req
if parent_req and not parent_req.child_requests:
self.parent_requests.pop(parent_req.request_id, None)
if not self.request_states:
self._requests_drained.set()
if not engine_core_output.finished:
# If req not finished in EngineCore, but Detokenizer
# detected stop string, abort needed in EngineCore.
reqs_to_abort.append(req_id)
# Track per-request stats
self._update_stats_from_finished(
req_state, finish_reason, iteration_stats
)
if self.tracer:
self.do_tracing(engine_core_output, req_state, iteration_stats)
# Track per-request stats
self._update_stats_from_finished(
req_state, finish_reason, iteration_stats
)
if self.tracer:
self.do_tracing(engine_core_output, req_state, iteration_stats)
return OutputProcessorOutput(
request_outputs=request_outputs,
reqs_to_abort=reqs_to_abort,
)
def _finish_request(self, req_state: RequestState) -> None:
req_id = req_state.request_id
self.request_states.pop(req_id)
internal_ids = self.external_req_ids[req_state.external_req_id]
internal_ids.remove(req_id)
if not internal_ids:
del self.external_req_ids[req_state.external_req_id]
# Remove parent request if applicable.
parent_req = req_state.parent_req
if parent_req and not parent_req.child_requests:
self.parent_requests.pop(parent_req.request_id, None)
if not self.request_states:
self._requests_drained.set()
def update_scheduler_stats(self, scheduler_stats: SchedulerStats | None):
self.lora_states.update_scheduler_stats(scheduler_stats)