[Frontend][last/N] Move all non-OpenAI content out of the OpenAI folder. (#56369)
Signed-off-by: wang.yuqi <yuqi.wang@daocloud.io>
This commit is contained in:
@@ -0,0 +1,65 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""SSE keep-alive comments for idle streaming responses.
|
||||
|
||||
Reverse proxies and tunnels (Cloudflare Tunnel, NGINX ``proxy_read_timeout``,
|
||||
AWS ALB, ...) can close a streaming connection when no bytes are sent for a
|
||||
while. That happens while a request is queued, during prefill, or in gaps
|
||||
between tokens. ``with_sse_keep_alive`` wraps the final SSE generator and emits
|
||||
a comment line (``": keep-alive\\n\\n"``) whenever it has been idle for
|
||||
``interval`` seconds. Comments are ignored by SSE-compliant clients but still
|
||||
count as bytes for proxy read-timeout logic.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import contextlib
|
||||
from collections.abc import AsyncGenerator
|
||||
|
||||
SSE_KEEP_ALIVE_COMMENT = ": keep-alive\n\n"
|
||||
|
||||
|
||||
def with_sse_keep_alive(
|
||||
generator: AsyncGenerator[str, None],
|
||||
interval: float,
|
||||
) -> AsyncGenerator[str, None]:
|
||||
"""Emit an SSE keep-alive comment when ``generator`` is idle.
|
||||
|
||||
A non-positive or non-finite ``interval`` returns ``generator`` unchanged,
|
||||
so the default path has no overhead. Otherwise a keep-alive comment is
|
||||
yielded whenever no chunk arrives within ``interval`` seconds.
|
||||
"""
|
||||
if not 0 < interval < float("inf"):
|
||||
return generator
|
||||
return _keep_alive_stream(generator, interval)
|
||||
|
||||
|
||||
async def _keep_alive_stream(
|
||||
generator: AsyncGenerator[str, None],
|
||||
interval: float,
|
||||
) -> AsyncGenerator[str, None]:
|
||||
# A single retained ``anext()`` task advances the generator; asyncio forbids
|
||||
# concurrent ``__anext__`` calls, and ``wait_for`` would cancel the task
|
||||
# mid-advance on timeout. ``asyncio.wait`` observes both outcomes without
|
||||
# cancelling the pending advance.
|
||||
next_task: asyncio.Task[str] = asyncio.create_task(anext(generator))
|
||||
try:
|
||||
while True:
|
||||
done, _ = await asyncio.wait({next_task}, timeout=interval)
|
||||
if not done:
|
||||
yield SSE_KEEP_ALIVE_COMMENT
|
||||
continue
|
||||
try:
|
||||
chunk = next_task.result()
|
||||
except StopAsyncIteration:
|
||||
return
|
||||
yield chunk
|
||||
next_task = asyncio.create_task(anext(generator))
|
||||
finally:
|
||||
next_task.cancel()
|
||||
# The task outcome was already handled in the loop; suppress so cleanup
|
||||
# never corrupts it (matching ``merge_async_iterators``).
|
||||
with contextlib.suppress(BaseException):
|
||||
await next_task
|
||||
# Close the upstream generator so its cleanup runs deterministically.
|
||||
with contextlib.suppress(BaseException):
|
||||
await generator.aclose()
|
||||
Reference in New Issue
Block a user