feat: add SSE keep-alive comments for idle streaming responses (#51034)

Signed-off-by: fuzzifikation <40043768+fuzzifikation@users.noreply.github.com>
This commit is contained in:
Dieter Schwarzmann
2026-08-23 18:13:02 -07:00
committed by GitHub
parent a7195188a4
commit 585bb07c70
6 changed files with 381 additions and 2 deletions
@@ -15,6 +15,7 @@ from vllm.entrypoints.openai.chat_completion.protocol import (
)
from vllm.entrypoints.openai.chat_completion.serving import OpenAIServingChat
from vllm.entrypoints.openai.engine.protocol import ErrorResponse
from vllm.entrypoints.openai.sse_keep_alive import with_sse_keep_alive
from vllm.entrypoints.serve.utils.api_utils import (
load_aware_call,
validate_json_request,
@@ -71,7 +72,12 @@ async def create_chat_completion(request: ChatCompletionRequest, raw_request: Re
headers=metrics_header(metrics_header_format),
)
return StreamingResponse(content=generator, media_type="text/event-stream")
args = getattr(raw_request.app.state, "args", None)
keep_alive_interval = getattr(args, "sse_keep_alive_interval", 0)
return StreamingResponse(
content=with_sse_keep_alive(generator, float(keep_alive_interval)),
media_type="text/event-stream",
)
@router.post(
+14
View File
@@ -137,6 +137,13 @@ class BaseFrontendArgs:
"""If set to True, enable tracking server_load_metrics in the app state."""
enable_force_include_usage: bool = False
"""If set to True, including usage on every request."""
sse_keep_alive_interval: int = 0
"""Send an SSE keep-alive comment line every this many seconds when a
`/v1/chat/completions` or `/v1/completions` streaming response is idle
(queued, prefill, or between tokens), to prevent reverse proxies/tunnels
with read timeouts from closing the connection. Defaults to 0, which
disables keep-alive comments entirely.
"""
enable_tokenizer_info_endpoint: bool = False
"""Enable the `/tokenizer_info` endpoint. May expose chat
templates and other tokenizer configuration."""
@@ -426,6 +433,13 @@ def validate_parsed_serve_args(args: argparse.Namespace):
if args.enable_log_outputs and not args.enable_log_requests:
raise TypeError("Error: --enable-log-outputs requires --enable-log-requests")
# SSE keep-alive interval must be zero (disabled) or a positive integer.
if getattr(args, "sse_keep_alive_interval", 0) < 0:
raise ValueError(
"Error: --sse-keep-alive-interval must be a non-negative integer "
"(0 disables keep-alive comments)."
)
if getattr(args, "enable_per_request_metrics", False) and getattr(
args, "disable_log_stats", False
):
@@ -13,6 +13,7 @@ from vllm.entrypoints.openai.completion.protocol import (
)
from vllm.entrypoints.openai.completion.serving import OpenAIServingCompletion
from vllm.entrypoints.openai.engine.protocol import ErrorResponse
from vllm.entrypoints.openai.sse_keep_alive import with_sse_keep_alive
from vllm.entrypoints.serve.utils.api_utils import (
load_aware_call,
validate_json_request,
@@ -63,7 +64,12 @@ async def create_completion(request: CompletionRequest, raw_request: Request):
headers=metrics_header(metrics_header_format),
)
return StreamingResponse(content=generator, media_type="text/event-stream")
args = getattr(raw_request.app.state, "args", None)
keep_alive_interval = getattr(args, "sse_keep_alive_interval", 0)
return StreamingResponse(
content=with_sse_keep_alive(generator, float(keep_alive_interval)),
media_type="text/event-stream",
)
def attach_router(app: FastAPI):
+65
View File
@@ -0,0 +1,65 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""SSE keep-alive comments for idle streaming responses.
Reverse proxies and tunnels (Cloudflare Tunnel, NGINX ``proxy_read_timeout``,
AWS ALB, ...) can close a streaming connection when no bytes are sent for a
while. That happens while a request is queued, during prefill, or in gaps
between tokens. ``with_sse_keep_alive`` wraps the final SSE generator and emits
a comment line (``": keep-alive\\n\\n"``) whenever it has been idle for
``interval`` seconds. Comments are ignored by SSE-compliant clients but still
count as bytes for proxy read-timeout logic.
"""
import asyncio
import contextlib
from collections.abc import AsyncGenerator
SSE_KEEP_ALIVE_COMMENT = ": keep-alive\n\n"
def with_sse_keep_alive(
generator: AsyncGenerator[str, None],
interval: float,
) -> AsyncGenerator[str, None]:
"""Emit an SSE keep-alive comment when ``generator`` is idle.
A non-positive or non-finite ``interval`` returns ``generator`` unchanged,
so the default path has no overhead. Otherwise a keep-alive comment is
yielded whenever no chunk arrives within ``interval`` seconds.
"""
if not 0 < interval < float("inf"):
return generator
return _keep_alive_stream(generator, interval)
async def _keep_alive_stream(
generator: AsyncGenerator[str, None],
interval: float,
) -> AsyncGenerator[str, None]:
# A single retained ``anext()`` task advances the generator; asyncio forbids
# concurrent ``__anext__`` calls, and ``wait_for`` would cancel the task
# mid-advance on timeout. ``asyncio.wait`` observes both outcomes without
# cancelling the pending advance.
next_task: asyncio.Task[str] = asyncio.create_task(anext(generator))
try:
while True:
done, _ = await asyncio.wait({next_task}, timeout=interval)
if not done:
yield SSE_KEEP_ALIVE_COMMENT
continue
try:
chunk = next_task.result()
except StopAsyncIteration:
return
yield chunk
next_task = asyncio.create_task(anext(generator))
finally:
next_task.cancel()
# The task outcome was already handled in the loop; suppress so cleanup
# never corrupts it (matching ``merge_async_iterators``).
with contextlib.suppress(BaseException):
await next_task
# Close the upstream generator so its cleanup runs deterministically.
with contextlib.suppress(BaseException):
await generator.aclose()