From 585bb07c7001ce52737ff0d18183d7a4b663cce4 Mon Sep 17 00:00:00 2001 From: Dieter Schwarzmann <40043768+fuzzifikation@users.noreply.github.com> Date: Mon, 24 Aug 2026 03:13:02 +0200 Subject: [PATCH] feat: add SSE keep-alive comments for idle streaming responses (#51034) Signed-off-by: fuzzifikation <40043768+fuzzifikation@users.noreply.github.com> --- tests/entrypoints/openai/test_cli_args.py | 30 ++ .../entrypoints/openai/test_sse_keep_alive.py | 258 ++++++++++++++++++ .../openai/chat_completion/api_router.py | 8 +- vllm/entrypoints/openai/cli_args.py | 14 + .../openai/completion/api_router.py | 8 +- vllm/entrypoints/openai/sse_keep_alive.py | 65 +++++ 6 files changed, 381 insertions(+), 2 deletions(-) create mode 100644 tests/entrypoints/openai/test_sse_keep_alive.py create mode 100644 vllm/entrypoints/openai/sse_keep_alive.py diff --git a/tests/entrypoints/openai/test_cli_args.py b/tests/entrypoints/openai/test_cli_args.py index c8c5fd0f55..9297d355ca 100644 --- a/tests/entrypoints/openai/test_cli_args.py +++ b/tests/entrypoints/openai/test_cli_args.py @@ -258,6 +258,36 @@ def test_subcommand_without_serve_args_skips_validation(): validate_parsed_serve_args(args) +@pytest.mark.parametrize( + "cli_args, expected", + [ + ([], 0), + (["--sse-keep-alive-interval", "0"], 0), + (["--sse-keep-alive-interval", "15"], 15), + ], +) +def test_sse_keep_alive_interval_valid(serve_parser, cli_args, expected): + """The default is 0 and zero or positive integers parse exactly.""" + args = serve_parser.parse_args(args=cli_args) + assert args.sse_keep_alive_interval == expected + validate_parsed_serve_args(args) + + +@pytest.mark.parametrize("value", ["-1", "-5"]) +def test_sse_keep_alive_interval_negative(serve_parser, value): + """Negative intervals are rejected by validation.""" + args = serve_parser.parse_args(args=["--sse-keep-alive-interval", value]) + with pytest.raises(ValueError): + validate_parsed_serve_args(args) + + +@pytest.mark.parametrize("value", ["nan", "inf", "-inf", "0.5", "abc"]) +def test_sse_keep_alive_interval_non_integer(serve_parser, value): + """Non-integer values fail argparse's int parsing with SystemExit.""" + with pytest.raises(SystemExit): + serve_parser.parse_args(args=["--sse-keep-alive-interval", value]) + + @pytest.mark.parametrize( "cli_args, expected_middleware", [ diff --git a/tests/entrypoints/openai/test_sse_keep_alive.py b/tests/entrypoints/openai/test_sse_keep_alive.py new file mode 100644 index 0000000000..74f5ee3ba2 --- /dev/null +++ b/tests/entrypoints/openai/test_sse_keep_alive.py @@ -0,0 +1,258 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +import asyncio + +import pytest +from starlette.responses import StreamingResponse + +from vllm.entrypoints.openai.sse_keep_alive import ( + SSE_KEEP_ALIVE_COMMENT, + with_sse_keep_alive, +) + +# --- Disabled path ---------------------------------------------------------- + + +def test_disabled_returns_same_generator(): + """For 0, negative, and NaN intervals the exact same generator object must + be returned (identity), proving no wrapper or per-chunk overhead when + disabled.""" + + async def gen(): + yield "x" + + for disabled in (0.0, -5.0, float("nan"), float("inf")): + g = gen() + assert with_sse_keep_alive(g, disabled) is g + + +# --- Keep-alive emission ---------------------------------------------------- + + +@pytest.mark.asyncio +async def test_keep_alive_during_silent_start(): + """A silent start (queue wait / prefill) emits keep-alive comments before + the first real chunk.""" + release = asyncio.Event() + + async def gen(): + await release.wait() # simulates queue wait / prefill + yield "data: first\n\n" + + out: list[str] = [] + async for item in with_sse_keep_alive(gen(), 0.01): + out.append(item) + if item == SSE_KEEP_ALIVE_COMMENT: + release.set() + if item == "data: first\n\n": + break + + assert SSE_KEEP_ALIVE_COMMENT in out + assert out.index(SSE_KEEP_ALIVE_COMMENT) < out.index("data: first\n\n") + + +@pytest.mark.asyncio +async def test_no_keep_alive_when_streaming_is_fast(): + """If chunks arrive faster than the interval, no comment is emitted and all + data is forwarded.""" + + async def gen(): + for chunk in ["a", "b", "c"]: + yield chunk + + out = [c async for c in with_sse_keep_alive(gen(), 0.5)] + assert out == ["a", "b", "c"] + assert SSE_KEEP_ALIVE_COMMENT not in out + + +@pytest.mark.asyncio +async def test_keep_alive_between_chunks_does_not_drop_data(): + """Keep-alives interleaved with real chunks must never drop or reorder + data.""" + release_b = asyncio.Event() + release_c = asyncio.Event() + + async def gen(): + yield "data: a\n\n" + await release_b.wait() + yield "data: b\n\n" + await release_c.wait() + yield "data: c\n\n" + + out: list[str] = [] + async for item in with_sse_keep_alive(gen(), 0.01): + out.append(item) + if item == SSE_KEEP_ALIVE_COMMENT and not release_b.is_set(): + release_b.set() + elif item == "data: b\n\n" and not release_c.is_set(): + release_c.set() + elif item == "data: c\n\n": + break + + assert SSE_KEEP_ALIVE_COMMENT in out + data_chunks = [c for c in out if c != SSE_KEEP_ALIVE_COMMENT] + assert data_chunks == ["data: a\n\n", "data: b\n\n", "data: c\n\n"] + + +@pytest.mark.asyncio +async def test_empty_generator_returns_immediately(): + async def gen(): + if False: + yield "never" + + out = [c async for c in with_sse_keep_alive(gen(), 0.01)] + assert out == [] + + +# --- Errors and cancellation ------------------------------------------------ + + +@pytest.mark.asyncio +async def test_exception_propagates(): + async def gen(): + yield "data: before\n\n" + raise RuntimeError("boom") + + wrapped = with_sse_keep_alive(gen(), 0.01) + assert await anext(wrapped) == "data: before\n\n" + with pytest.raises(RuntimeError, match="boom"): + await anext(wrapped) + + +@pytest.mark.asyncio +async def test_upstream_cancelled_error_propagates(): + async def gen(): + yield "data: before\n\n" + raise asyncio.CancelledError("upstream cancelled") + + wrapped = with_sse_keep_alive(gen(), 0.01) + assert await anext(wrapped) == "data: before\n\n" + with pytest.raises(asyncio.CancelledError): + await anext(wrapped) + + +@pytest.mark.asyncio +async def test_aclose_closes_upstream(): + """Closing a wrapper that is mid-stream must complete promptly and close + the upstream generator (regression for the earlier leaked-task hang).""" + finalized = asyncio.Event() + + async def gen(): + try: + yield "chunk-0" + await asyncio.Event().wait() # block forever + finally: + finalized.set() + + wrapped = with_sse_keep_alive(gen(), 10.0) + assert await anext(wrapped) == "chunk-0" + await asyncio.wait_for(wrapped.aclose(), timeout=2.0) + assert finalized.is_set() + + +@pytest.mark.asyncio +async def test_client_disconnect_cancels_upstream(): + """Cancelling the consumer mid-stream must cancel the pending anext and + close the upstream generator so its cleanup runs.""" + finalized = asyncio.Event() + + async def gen(): + try: + for i in range(1000): + yield f"chunk-{i}" + await asyncio.Event().wait() + finally: + finalized.set() + + wrapped = with_sse_keep_alive(gen(), 10.0) + assert await anext(wrapped) == "chunk-0" + task = asyncio.current_task() + assert task is not None + task.cancel() + with pytest.raises(asyncio.CancelledError): + await anext(wrapped) + assert finalized.is_set() + + +# --- Response-level integration (the shipped StreamingResponse path) -------- + + +@pytest.mark.asyncio +async def test_streaming_response_emits_comment(): + """A keep-alive comment must reach the client through the real + StreamingResponse used by the routers.""" + release = asyncio.Event() + + async def gen(): + await release.wait() + yield "data: hi\n\n" + + response = StreamingResponse( + content=with_sse_keep_alive(gen(), 0.01), + media_type="text/event-stream", + ) + body: list[bytes] = [] + started = asyncio.Event() + + async def send(message): + if message["type"] == "http.response.start": + started.set() + elif message["type"] == "http.response.body": + body.append(message["body"]) + if SSE_KEEP_ALIVE_COMMENT.encode() in message["body"]: + release.set() + + async def receive(): + await asyncio.Event().wait() + + await response( + {"type": "http", "asgi": {"spec_version": "2.3"}}, + receive, + send, + ) + data = b"".join(body).decode() + assert SSE_KEEP_ALIVE_COMMENT in data + assert "data: hi" in data + + +@pytest.mark.asyncio +async def test_streaming_response_disconnect_closes_upstream(): + """On the ASGI 2.3 disconnect path used by the pinned Uvicorn, Starlette + cancels streaming and the wrapper's finally must close the upstream + generator.""" + finalized = asyncio.Event() + + async def gen(): + try: + await asyncio.Event().wait() + yield "never" + finally: + finalized.set() + + response = StreamingResponse( + content=with_sse_keep_alive(gen(), 10.0), + media_type="text/event-stream", + ) + started = asyncio.Event() + + async def receive(): + await started.wait() + return {"type": "http.disconnect"} + + async def send(message): + if message["type"] == "http.response.start": + started.set() + + await response( + {"type": "http", "asgi": {"spec_version": "2.3"}}, + receive, + send, + ) + assert finalized.is_set() + pending = [ + t + for t in asyncio.all_tasks() + if t is not asyncio.current_task() and not t.done() + ] + assert pending == [] diff --git a/vllm/entrypoints/openai/chat_completion/api_router.py b/vllm/entrypoints/openai/chat_completion/api_router.py index 6f3289ede4..30b53678bb 100644 --- a/vllm/entrypoints/openai/chat_completion/api_router.py +++ b/vllm/entrypoints/openai/chat_completion/api_router.py @@ -15,6 +15,7 @@ from vllm.entrypoints.openai.chat_completion.protocol import ( ) from vllm.entrypoints.openai.chat_completion.serving import OpenAIServingChat from vllm.entrypoints.openai.engine.protocol import ErrorResponse +from vllm.entrypoints.openai.sse_keep_alive import with_sse_keep_alive from vllm.entrypoints.serve.utils.api_utils import ( load_aware_call, validate_json_request, @@ -71,7 +72,12 @@ async def create_chat_completion(request: ChatCompletionRequest, raw_request: Re headers=metrics_header(metrics_header_format), ) - return StreamingResponse(content=generator, media_type="text/event-stream") + args = getattr(raw_request.app.state, "args", None) + keep_alive_interval = getattr(args, "sse_keep_alive_interval", 0) + return StreamingResponse( + content=with_sse_keep_alive(generator, float(keep_alive_interval)), + media_type="text/event-stream", + ) @router.post( diff --git a/vllm/entrypoints/openai/cli_args.py b/vllm/entrypoints/openai/cli_args.py index 49730d34b8..2ffd21b8c5 100644 --- a/vllm/entrypoints/openai/cli_args.py +++ b/vllm/entrypoints/openai/cli_args.py @@ -137,6 +137,13 @@ class BaseFrontendArgs: """If set to True, enable tracking server_load_metrics in the app state.""" enable_force_include_usage: bool = False """If set to True, including usage on every request.""" + sse_keep_alive_interval: int = 0 + """Send an SSE keep-alive comment line every this many seconds when a + `/v1/chat/completions` or `/v1/completions` streaming response is idle + (queued, prefill, or between tokens), to prevent reverse proxies/tunnels + with read timeouts from closing the connection. Defaults to 0, which + disables keep-alive comments entirely. + """ enable_tokenizer_info_endpoint: bool = False """Enable the `/tokenizer_info` endpoint. May expose chat templates and other tokenizer configuration.""" @@ -426,6 +433,13 @@ def validate_parsed_serve_args(args: argparse.Namespace): if args.enable_log_outputs and not args.enable_log_requests: raise TypeError("Error: --enable-log-outputs requires --enable-log-requests") + # SSE keep-alive interval must be zero (disabled) or a positive integer. + if getattr(args, "sse_keep_alive_interval", 0) < 0: + raise ValueError( + "Error: --sse-keep-alive-interval must be a non-negative integer " + "(0 disables keep-alive comments)." + ) + if getattr(args, "enable_per_request_metrics", False) and getattr( args, "disable_log_stats", False ): diff --git a/vllm/entrypoints/openai/completion/api_router.py b/vllm/entrypoints/openai/completion/api_router.py index 441aef165c..d5bef1ebf4 100644 --- a/vllm/entrypoints/openai/completion/api_router.py +++ b/vllm/entrypoints/openai/completion/api_router.py @@ -13,6 +13,7 @@ from vllm.entrypoints.openai.completion.protocol import ( ) from vllm.entrypoints.openai.completion.serving import OpenAIServingCompletion from vllm.entrypoints.openai.engine.protocol import ErrorResponse +from vllm.entrypoints.openai.sse_keep_alive import with_sse_keep_alive from vllm.entrypoints.serve.utils.api_utils import ( load_aware_call, validate_json_request, @@ -63,7 +64,12 @@ async def create_completion(request: CompletionRequest, raw_request: Request): headers=metrics_header(metrics_header_format), ) - return StreamingResponse(content=generator, media_type="text/event-stream") + args = getattr(raw_request.app.state, "args", None) + keep_alive_interval = getattr(args, "sse_keep_alive_interval", 0) + return StreamingResponse( + content=with_sse_keep_alive(generator, float(keep_alive_interval)), + media_type="text/event-stream", + ) def attach_router(app: FastAPI): diff --git a/vllm/entrypoints/openai/sse_keep_alive.py b/vllm/entrypoints/openai/sse_keep_alive.py new file mode 100644 index 0000000000..ec9ef177b0 --- /dev/null +++ b/vllm/entrypoints/openai/sse_keep_alive.py @@ -0,0 +1,65 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""SSE keep-alive comments for idle streaming responses. + +Reverse proxies and tunnels (Cloudflare Tunnel, NGINX ``proxy_read_timeout``, +AWS ALB, ...) can close a streaming connection when no bytes are sent for a +while. That happens while a request is queued, during prefill, or in gaps +between tokens. ``with_sse_keep_alive`` wraps the final SSE generator and emits +a comment line (``": keep-alive\\n\\n"``) whenever it has been idle for +``interval`` seconds. Comments are ignored by SSE-compliant clients but still +count as bytes for proxy read-timeout logic. +""" + +import asyncio +import contextlib +from collections.abc import AsyncGenerator + +SSE_KEEP_ALIVE_COMMENT = ": keep-alive\n\n" + + +def with_sse_keep_alive( + generator: AsyncGenerator[str, None], + interval: float, +) -> AsyncGenerator[str, None]: + """Emit an SSE keep-alive comment when ``generator`` is idle. + + A non-positive or non-finite ``interval`` returns ``generator`` unchanged, + so the default path has no overhead. Otherwise a keep-alive comment is + yielded whenever no chunk arrives within ``interval`` seconds. + """ + if not 0 < interval < float("inf"): + return generator + return _keep_alive_stream(generator, interval) + + +async def _keep_alive_stream( + generator: AsyncGenerator[str, None], + interval: float, +) -> AsyncGenerator[str, None]: + # A single retained ``anext()`` task advances the generator; asyncio forbids + # concurrent ``__anext__`` calls, and ``wait_for`` would cancel the task + # mid-advance on timeout. ``asyncio.wait`` observes both outcomes without + # cancelling the pending advance. + next_task: asyncio.Task[str] = asyncio.create_task(anext(generator)) + try: + while True: + done, _ = await asyncio.wait({next_task}, timeout=interval) + if not done: + yield SSE_KEEP_ALIVE_COMMENT + continue + try: + chunk = next_task.result() + except StopAsyncIteration: + return + yield chunk + next_task = asyncio.create_task(anext(generator)) + finally: + next_task.cancel() + # The task outcome was already handled in the loop; suppress so cleanup + # never corrupts it (matching ``merge_async_iterators``). + with contextlib.suppress(BaseException): + await next_task + # Close the upstream generator so its cleanup runs deterministically. + with contextlib.suppress(BaseException): + await generator.aclose()