feat: add SSE keep-alive comments for idle streaming responses (#51034)

Signed-off-by: fuzzifikation <40043768+fuzzifikation@users.noreply.github.com>
This commit is contained in:
Dieter Schwarzmann
2026-08-23 18:13:02 -07:00
committed by GitHub
parent a7195188a4
commit 585bb07c70
6 changed files with 381 additions and 2 deletions
+30
View File
@@ -258,6 +258,36 @@ def test_subcommand_without_serve_args_skips_validation():
validate_parsed_serve_args(args)
@pytest.mark.parametrize(
"cli_args, expected",
[
([], 0),
(["--sse-keep-alive-interval", "0"], 0),
(["--sse-keep-alive-interval", "15"], 15),
],
)
def test_sse_keep_alive_interval_valid(serve_parser, cli_args, expected):
"""The default is 0 and zero or positive integers parse exactly."""
args = serve_parser.parse_args(args=cli_args)
assert args.sse_keep_alive_interval == expected
validate_parsed_serve_args(args)
@pytest.mark.parametrize("value", ["-1", "-5"])
def test_sse_keep_alive_interval_negative(serve_parser, value):
"""Negative intervals are rejected by validation."""
args = serve_parser.parse_args(args=["--sse-keep-alive-interval", value])
with pytest.raises(ValueError):
validate_parsed_serve_args(args)
@pytest.mark.parametrize("value", ["nan", "inf", "-inf", "0.5", "abc"])
def test_sse_keep_alive_interval_non_integer(serve_parser, value):
"""Non-integer values fail argparse's int parsing with SystemExit."""
with pytest.raises(SystemExit):
serve_parser.parse_args(args=["--sse-keep-alive-interval", value])
@pytest.mark.parametrize(
"cli_args, expected_middleware",
[
@@ -0,0 +1,258 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import asyncio
import pytest
from starlette.responses import StreamingResponse
from vllm.entrypoints.openai.sse_keep_alive import (
SSE_KEEP_ALIVE_COMMENT,
with_sse_keep_alive,
)
# --- Disabled path ----------------------------------------------------------
def test_disabled_returns_same_generator():
"""For 0, negative, and NaN intervals the exact same generator object must
be returned (identity), proving no wrapper or per-chunk overhead when
disabled."""
async def gen():
yield "x"
for disabled in (0.0, -5.0, float("nan"), float("inf")):
g = gen()
assert with_sse_keep_alive(g, disabled) is g
# --- Keep-alive emission ----------------------------------------------------
@pytest.mark.asyncio
async def test_keep_alive_during_silent_start():
"""A silent start (queue wait / prefill) emits keep-alive comments before
the first real chunk."""
release = asyncio.Event()
async def gen():
await release.wait() # simulates queue wait / prefill
yield "data: first\n\n"
out: list[str] = []
async for item in with_sse_keep_alive(gen(), 0.01):
out.append(item)
if item == SSE_KEEP_ALIVE_COMMENT:
release.set()
if item == "data: first\n\n":
break
assert SSE_KEEP_ALIVE_COMMENT in out
assert out.index(SSE_KEEP_ALIVE_COMMENT) < out.index("data: first\n\n")
@pytest.mark.asyncio
async def test_no_keep_alive_when_streaming_is_fast():
"""If chunks arrive faster than the interval, no comment is emitted and all
data is forwarded."""
async def gen():
for chunk in ["a", "b", "c"]:
yield chunk
out = [c async for c in with_sse_keep_alive(gen(), 0.5)]
assert out == ["a", "b", "c"]
assert SSE_KEEP_ALIVE_COMMENT not in out
@pytest.mark.asyncio
async def test_keep_alive_between_chunks_does_not_drop_data():
"""Keep-alives interleaved with real chunks must never drop or reorder
data."""
release_b = asyncio.Event()
release_c = asyncio.Event()
async def gen():
yield "data: a\n\n"
await release_b.wait()
yield "data: b\n\n"
await release_c.wait()
yield "data: c\n\n"
out: list[str] = []
async for item in with_sse_keep_alive(gen(), 0.01):
out.append(item)
if item == SSE_KEEP_ALIVE_COMMENT and not release_b.is_set():
release_b.set()
elif item == "data: b\n\n" and not release_c.is_set():
release_c.set()
elif item == "data: c\n\n":
break
assert SSE_KEEP_ALIVE_COMMENT in out
data_chunks = [c for c in out if c != SSE_KEEP_ALIVE_COMMENT]
assert data_chunks == ["data: a\n\n", "data: b\n\n", "data: c\n\n"]
@pytest.mark.asyncio
async def test_empty_generator_returns_immediately():
async def gen():
if False:
yield "never"
out = [c async for c in with_sse_keep_alive(gen(), 0.01)]
assert out == []
# --- Errors and cancellation ------------------------------------------------
@pytest.mark.asyncio
async def test_exception_propagates():
async def gen():
yield "data: before\n\n"
raise RuntimeError("boom")
wrapped = with_sse_keep_alive(gen(), 0.01)
assert await anext(wrapped) == "data: before\n\n"
with pytest.raises(RuntimeError, match="boom"):
await anext(wrapped)
@pytest.mark.asyncio
async def test_upstream_cancelled_error_propagates():
async def gen():
yield "data: before\n\n"
raise asyncio.CancelledError("upstream cancelled")
wrapped = with_sse_keep_alive(gen(), 0.01)
assert await anext(wrapped) == "data: before\n\n"
with pytest.raises(asyncio.CancelledError):
await anext(wrapped)
@pytest.mark.asyncio
async def test_aclose_closes_upstream():
"""Closing a wrapper that is mid-stream must complete promptly and close
the upstream generator (regression for the earlier leaked-task hang)."""
finalized = asyncio.Event()
async def gen():
try:
yield "chunk-0"
await asyncio.Event().wait() # block forever
finally:
finalized.set()
wrapped = with_sse_keep_alive(gen(), 10.0)
assert await anext(wrapped) == "chunk-0"
await asyncio.wait_for(wrapped.aclose(), timeout=2.0)
assert finalized.is_set()
@pytest.mark.asyncio
async def test_client_disconnect_cancels_upstream():
"""Cancelling the consumer mid-stream must cancel the pending anext and
close the upstream generator so its cleanup runs."""
finalized = asyncio.Event()
async def gen():
try:
for i in range(1000):
yield f"chunk-{i}"
await asyncio.Event().wait()
finally:
finalized.set()
wrapped = with_sse_keep_alive(gen(), 10.0)
assert await anext(wrapped) == "chunk-0"
task = asyncio.current_task()
assert task is not None
task.cancel()
with pytest.raises(asyncio.CancelledError):
await anext(wrapped)
assert finalized.is_set()
# --- Response-level integration (the shipped StreamingResponse path) --------
@pytest.mark.asyncio
async def test_streaming_response_emits_comment():
"""A keep-alive comment must reach the client through the real
StreamingResponse used by the routers."""
release = asyncio.Event()
async def gen():
await release.wait()
yield "data: hi\n\n"
response = StreamingResponse(
content=with_sse_keep_alive(gen(), 0.01),
media_type="text/event-stream",
)
body: list[bytes] = []
started = asyncio.Event()
async def send(message):
if message["type"] == "http.response.start":
started.set()
elif message["type"] == "http.response.body":
body.append(message["body"])
if SSE_KEEP_ALIVE_COMMENT.encode() in message["body"]:
release.set()
async def receive():
await asyncio.Event().wait()
await response(
{"type": "http", "asgi": {"spec_version": "2.3"}},
receive,
send,
)
data = b"".join(body).decode()
assert SSE_KEEP_ALIVE_COMMENT in data
assert "data: hi" in data
@pytest.mark.asyncio
async def test_streaming_response_disconnect_closes_upstream():
"""On the ASGI 2.3 disconnect path used by the pinned Uvicorn, Starlette
cancels streaming and the wrapper's finally must close the upstream
generator."""
finalized = asyncio.Event()
async def gen():
try:
await asyncio.Event().wait()
yield "never"
finally:
finalized.set()
response = StreamingResponse(
content=with_sse_keep_alive(gen(), 10.0),
media_type="text/event-stream",
)
started = asyncio.Event()
async def receive():
await started.wait()
return {"type": "http.disconnect"}
async def send(message):
if message["type"] == "http.response.start":
started.set()
await response(
{"type": "http", "asgi": {"spec_version": "2.3"}},
receive,
send,
)
assert finalized.is_set()
pending = [
t
for t in asyncio.all_tasks()
if t is not asyncio.current_task() and not t.done()
]
assert pending == []
@@ -15,6 +15,7 @@ from vllm.entrypoints.openai.chat_completion.protocol import (
)
from vllm.entrypoints.openai.chat_completion.serving import OpenAIServingChat
from vllm.entrypoints.openai.engine.protocol import ErrorResponse
from vllm.entrypoints.openai.sse_keep_alive import with_sse_keep_alive
from vllm.entrypoints.serve.utils.api_utils import (
load_aware_call,
validate_json_request,
@@ -71,7 +72,12 @@ async def create_chat_completion(request: ChatCompletionRequest, raw_request: Re
headers=metrics_header(metrics_header_format),
)
return StreamingResponse(content=generator, media_type="text/event-stream")
args = getattr(raw_request.app.state, "args", None)
keep_alive_interval = getattr(args, "sse_keep_alive_interval", 0)
return StreamingResponse(
content=with_sse_keep_alive(generator, float(keep_alive_interval)),
media_type="text/event-stream",
)
@router.post(
+14
View File
@@ -137,6 +137,13 @@ class BaseFrontendArgs:
"""If set to True, enable tracking server_load_metrics in the app state."""
enable_force_include_usage: bool = False
"""If set to True, including usage on every request."""
sse_keep_alive_interval: int = 0
"""Send an SSE keep-alive comment line every this many seconds when a
`/v1/chat/completions` or `/v1/completions` streaming response is idle
(queued, prefill, or between tokens), to prevent reverse proxies/tunnels
with read timeouts from closing the connection. Defaults to 0, which
disables keep-alive comments entirely.
"""
enable_tokenizer_info_endpoint: bool = False
"""Enable the `/tokenizer_info` endpoint. May expose chat
templates and other tokenizer configuration."""
@@ -426,6 +433,13 @@ def validate_parsed_serve_args(args: argparse.Namespace):
if args.enable_log_outputs and not args.enable_log_requests:
raise TypeError("Error: --enable-log-outputs requires --enable-log-requests")
# SSE keep-alive interval must be zero (disabled) or a positive integer.
if getattr(args, "sse_keep_alive_interval", 0) < 0:
raise ValueError(
"Error: --sse-keep-alive-interval must be a non-negative integer "
"(0 disables keep-alive comments)."
)
if getattr(args, "enable_per_request_metrics", False) and getattr(
args, "disable_log_stats", False
):
@@ -13,6 +13,7 @@ from vllm.entrypoints.openai.completion.protocol import (
)
from vllm.entrypoints.openai.completion.serving import OpenAIServingCompletion
from vllm.entrypoints.openai.engine.protocol import ErrorResponse
from vllm.entrypoints.openai.sse_keep_alive import with_sse_keep_alive
from vllm.entrypoints.serve.utils.api_utils import (
load_aware_call,
validate_json_request,
@@ -63,7 +64,12 @@ async def create_completion(request: CompletionRequest, raw_request: Request):
headers=metrics_header(metrics_header_format),
)
return StreamingResponse(content=generator, media_type="text/event-stream")
args = getattr(raw_request.app.state, "args", None)
keep_alive_interval = getattr(args, "sse_keep_alive_interval", 0)
return StreamingResponse(
content=with_sse_keep_alive(generator, float(keep_alive_interval)),
media_type="text/event-stream",
)
def attach_router(app: FastAPI):
+65
View File
@@ -0,0 +1,65 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""SSE keep-alive comments for idle streaming responses.
Reverse proxies and tunnels (Cloudflare Tunnel, NGINX ``proxy_read_timeout``,
AWS ALB, ...) can close a streaming connection when no bytes are sent for a
while. That happens while a request is queued, during prefill, or in gaps
between tokens. ``with_sse_keep_alive`` wraps the final SSE generator and emits
a comment line (``": keep-alive\\n\\n"``) whenever it has been idle for
``interval`` seconds. Comments are ignored by SSE-compliant clients but still
count as bytes for proxy read-timeout logic.
"""
import asyncio
import contextlib
from collections.abc import AsyncGenerator
SSE_KEEP_ALIVE_COMMENT = ": keep-alive\n\n"
def with_sse_keep_alive(
generator: AsyncGenerator[str, None],
interval: float,
) -> AsyncGenerator[str, None]:
"""Emit an SSE keep-alive comment when ``generator`` is idle.
A non-positive or non-finite ``interval`` returns ``generator`` unchanged,
so the default path has no overhead. Otherwise a keep-alive comment is
yielded whenever no chunk arrives within ``interval`` seconds.
"""
if not 0 < interval < float("inf"):
return generator
return _keep_alive_stream(generator, interval)
async def _keep_alive_stream(
generator: AsyncGenerator[str, None],
interval: float,
) -> AsyncGenerator[str, None]:
# A single retained ``anext()`` task advances the generator; asyncio forbids
# concurrent ``__anext__`` calls, and ``wait_for`` would cancel the task
# mid-advance on timeout. ``asyncio.wait`` observes both outcomes without
# cancelling the pending advance.
next_task: asyncio.Task[str] = asyncio.create_task(anext(generator))
try:
while True:
done, _ = await asyncio.wait({next_task}, timeout=interval)
if not done:
yield SSE_KEEP_ALIVE_COMMENT
continue
try:
chunk = next_task.result()
except StopAsyncIteration:
return
yield chunk
next_task = asyncio.create_task(anext(generator))
finally:
next_task.cancel()
# The task outcome was already handled in the loop; suppress so cleanup
# never corrupts it (matching ``merge_async_iterators``).
with contextlib.suppress(BaseException):
await next_task
# Close the upstream generator so its cleanup runs deterministically.
with contextlib.suppress(BaseException):
await generator.aclose()