feat: add SSE keep-alive comments for idle streaming responses (#51034)
Signed-off-by: fuzzifikation <40043768+fuzzifikation@users.noreply.github.com>
This commit is contained in:
@@ -258,6 +258,36 @@ def test_subcommand_without_serve_args_skips_validation():
|
||||
validate_parsed_serve_args(args)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"cli_args, expected",
|
||||
[
|
||||
([], 0),
|
||||
(["--sse-keep-alive-interval", "0"], 0),
|
||||
(["--sse-keep-alive-interval", "15"], 15),
|
||||
],
|
||||
)
|
||||
def test_sse_keep_alive_interval_valid(serve_parser, cli_args, expected):
|
||||
"""The default is 0 and zero or positive integers parse exactly."""
|
||||
args = serve_parser.parse_args(args=cli_args)
|
||||
assert args.sse_keep_alive_interval == expected
|
||||
validate_parsed_serve_args(args)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("value", ["-1", "-5"])
|
||||
def test_sse_keep_alive_interval_negative(serve_parser, value):
|
||||
"""Negative intervals are rejected by validation."""
|
||||
args = serve_parser.parse_args(args=["--sse-keep-alive-interval", value])
|
||||
with pytest.raises(ValueError):
|
||||
validate_parsed_serve_args(args)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("value", ["nan", "inf", "-inf", "0.5", "abc"])
|
||||
def test_sse_keep_alive_interval_non_integer(serve_parser, value):
|
||||
"""Non-integer values fail argparse's int parsing with SystemExit."""
|
||||
with pytest.raises(SystemExit):
|
||||
serve_parser.parse_args(args=["--sse-keep-alive-interval", value])
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"cli_args, expected_middleware",
|
||||
[
|
||||
|
||||
@@ -0,0 +1,258 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import asyncio
|
||||
|
||||
import pytest
|
||||
from starlette.responses import StreamingResponse
|
||||
|
||||
from vllm.entrypoints.openai.sse_keep_alive import (
|
||||
SSE_KEEP_ALIVE_COMMENT,
|
||||
with_sse_keep_alive,
|
||||
)
|
||||
|
||||
# --- Disabled path ----------------------------------------------------------
|
||||
|
||||
|
||||
def test_disabled_returns_same_generator():
|
||||
"""For 0, negative, and NaN intervals the exact same generator object must
|
||||
be returned (identity), proving no wrapper or per-chunk overhead when
|
||||
disabled."""
|
||||
|
||||
async def gen():
|
||||
yield "x"
|
||||
|
||||
for disabled in (0.0, -5.0, float("nan"), float("inf")):
|
||||
g = gen()
|
||||
assert with_sse_keep_alive(g, disabled) is g
|
||||
|
||||
|
||||
# --- Keep-alive emission ----------------------------------------------------
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_keep_alive_during_silent_start():
|
||||
"""A silent start (queue wait / prefill) emits keep-alive comments before
|
||||
the first real chunk."""
|
||||
release = asyncio.Event()
|
||||
|
||||
async def gen():
|
||||
await release.wait() # simulates queue wait / prefill
|
||||
yield "data: first\n\n"
|
||||
|
||||
out: list[str] = []
|
||||
async for item in with_sse_keep_alive(gen(), 0.01):
|
||||
out.append(item)
|
||||
if item == SSE_KEEP_ALIVE_COMMENT:
|
||||
release.set()
|
||||
if item == "data: first\n\n":
|
||||
break
|
||||
|
||||
assert SSE_KEEP_ALIVE_COMMENT in out
|
||||
assert out.index(SSE_KEEP_ALIVE_COMMENT) < out.index("data: first\n\n")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_no_keep_alive_when_streaming_is_fast():
|
||||
"""If chunks arrive faster than the interval, no comment is emitted and all
|
||||
data is forwarded."""
|
||||
|
||||
async def gen():
|
||||
for chunk in ["a", "b", "c"]:
|
||||
yield chunk
|
||||
|
||||
out = [c async for c in with_sse_keep_alive(gen(), 0.5)]
|
||||
assert out == ["a", "b", "c"]
|
||||
assert SSE_KEEP_ALIVE_COMMENT not in out
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_keep_alive_between_chunks_does_not_drop_data():
|
||||
"""Keep-alives interleaved with real chunks must never drop or reorder
|
||||
data."""
|
||||
release_b = asyncio.Event()
|
||||
release_c = asyncio.Event()
|
||||
|
||||
async def gen():
|
||||
yield "data: a\n\n"
|
||||
await release_b.wait()
|
||||
yield "data: b\n\n"
|
||||
await release_c.wait()
|
||||
yield "data: c\n\n"
|
||||
|
||||
out: list[str] = []
|
||||
async for item in with_sse_keep_alive(gen(), 0.01):
|
||||
out.append(item)
|
||||
if item == SSE_KEEP_ALIVE_COMMENT and not release_b.is_set():
|
||||
release_b.set()
|
||||
elif item == "data: b\n\n" and not release_c.is_set():
|
||||
release_c.set()
|
||||
elif item == "data: c\n\n":
|
||||
break
|
||||
|
||||
assert SSE_KEEP_ALIVE_COMMENT in out
|
||||
data_chunks = [c for c in out if c != SSE_KEEP_ALIVE_COMMENT]
|
||||
assert data_chunks == ["data: a\n\n", "data: b\n\n", "data: c\n\n"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_empty_generator_returns_immediately():
|
||||
async def gen():
|
||||
if False:
|
||||
yield "never"
|
||||
|
||||
out = [c async for c in with_sse_keep_alive(gen(), 0.01)]
|
||||
assert out == []
|
||||
|
||||
|
||||
# --- Errors and cancellation ------------------------------------------------
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_exception_propagates():
|
||||
async def gen():
|
||||
yield "data: before\n\n"
|
||||
raise RuntimeError("boom")
|
||||
|
||||
wrapped = with_sse_keep_alive(gen(), 0.01)
|
||||
assert await anext(wrapped) == "data: before\n\n"
|
||||
with pytest.raises(RuntimeError, match="boom"):
|
||||
await anext(wrapped)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_upstream_cancelled_error_propagates():
|
||||
async def gen():
|
||||
yield "data: before\n\n"
|
||||
raise asyncio.CancelledError("upstream cancelled")
|
||||
|
||||
wrapped = with_sse_keep_alive(gen(), 0.01)
|
||||
assert await anext(wrapped) == "data: before\n\n"
|
||||
with pytest.raises(asyncio.CancelledError):
|
||||
await anext(wrapped)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_aclose_closes_upstream():
|
||||
"""Closing a wrapper that is mid-stream must complete promptly and close
|
||||
the upstream generator (regression for the earlier leaked-task hang)."""
|
||||
finalized = asyncio.Event()
|
||||
|
||||
async def gen():
|
||||
try:
|
||||
yield "chunk-0"
|
||||
await asyncio.Event().wait() # block forever
|
||||
finally:
|
||||
finalized.set()
|
||||
|
||||
wrapped = with_sse_keep_alive(gen(), 10.0)
|
||||
assert await anext(wrapped) == "chunk-0"
|
||||
await asyncio.wait_for(wrapped.aclose(), timeout=2.0)
|
||||
assert finalized.is_set()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_client_disconnect_cancels_upstream():
|
||||
"""Cancelling the consumer mid-stream must cancel the pending anext and
|
||||
close the upstream generator so its cleanup runs."""
|
||||
finalized = asyncio.Event()
|
||||
|
||||
async def gen():
|
||||
try:
|
||||
for i in range(1000):
|
||||
yield f"chunk-{i}"
|
||||
await asyncio.Event().wait()
|
||||
finally:
|
||||
finalized.set()
|
||||
|
||||
wrapped = with_sse_keep_alive(gen(), 10.0)
|
||||
assert await anext(wrapped) == "chunk-0"
|
||||
task = asyncio.current_task()
|
||||
assert task is not None
|
||||
task.cancel()
|
||||
with pytest.raises(asyncio.CancelledError):
|
||||
await anext(wrapped)
|
||||
assert finalized.is_set()
|
||||
|
||||
|
||||
# --- Response-level integration (the shipped StreamingResponse path) --------
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_streaming_response_emits_comment():
|
||||
"""A keep-alive comment must reach the client through the real
|
||||
StreamingResponse used by the routers."""
|
||||
release = asyncio.Event()
|
||||
|
||||
async def gen():
|
||||
await release.wait()
|
||||
yield "data: hi\n\n"
|
||||
|
||||
response = StreamingResponse(
|
||||
content=with_sse_keep_alive(gen(), 0.01),
|
||||
media_type="text/event-stream",
|
||||
)
|
||||
body: list[bytes] = []
|
||||
started = asyncio.Event()
|
||||
|
||||
async def send(message):
|
||||
if message["type"] == "http.response.start":
|
||||
started.set()
|
||||
elif message["type"] == "http.response.body":
|
||||
body.append(message["body"])
|
||||
if SSE_KEEP_ALIVE_COMMENT.encode() in message["body"]:
|
||||
release.set()
|
||||
|
||||
async def receive():
|
||||
await asyncio.Event().wait()
|
||||
|
||||
await response(
|
||||
{"type": "http", "asgi": {"spec_version": "2.3"}},
|
||||
receive,
|
||||
send,
|
||||
)
|
||||
data = b"".join(body).decode()
|
||||
assert SSE_KEEP_ALIVE_COMMENT in data
|
||||
assert "data: hi" in data
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_streaming_response_disconnect_closes_upstream():
|
||||
"""On the ASGI 2.3 disconnect path used by the pinned Uvicorn, Starlette
|
||||
cancels streaming and the wrapper's finally must close the upstream
|
||||
generator."""
|
||||
finalized = asyncio.Event()
|
||||
|
||||
async def gen():
|
||||
try:
|
||||
await asyncio.Event().wait()
|
||||
yield "never"
|
||||
finally:
|
||||
finalized.set()
|
||||
|
||||
response = StreamingResponse(
|
||||
content=with_sse_keep_alive(gen(), 10.0),
|
||||
media_type="text/event-stream",
|
||||
)
|
||||
started = asyncio.Event()
|
||||
|
||||
async def receive():
|
||||
await started.wait()
|
||||
return {"type": "http.disconnect"}
|
||||
|
||||
async def send(message):
|
||||
if message["type"] == "http.response.start":
|
||||
started.set()
|
||||
|
||||
await response(
|
||||
{"type": "http", "asgi": {"spec_version": "2.3"}},
|
||||
receive,
|
||||
send,
|
||||
)
|
||||
assert finalized.is_set()
|
||||
pending = [
|
||||
t
|
||||
for t in asyncio.all_tasks()
|
||||
if t is not asyncio.current_task() and not t.done()
|
||||
]
|
||||
assert pending == []
|
||||
@@ -15,6 +15,7 @@ from vllm.entrypoints.openai.chat_completion.protocol import (
|
||||
)
|
||||
from vllm.entrypoints.openai.chat_completion.serving import OpenAIServingChat
|
||||
from vllm.entrypoints.openai.engine.protocol import ErrorResponse
|
||||
from vllm.entrypoints.openai.sse_keep_alive import with_sse_keep_alive
|
||||
from vllm.entrypoints.serve.utils.api_utils import (
|
||||
load_aware_call,
|
||||
validate_json_request,
|
||||
@@ -71,7 +72,12 @@ async def create_chat_completion(request: ChatCompletionRequest, raw_request: Re
|
||||
headers=metrics_header(metrics_header_format),
|
||||
)
|
||||
|
||||
return StreamingResponse(content=generator, media_type="text/event-stream")
|
||||
args = getattr(raw_request.app.state, "args", None)
|
||||
keep_alive_interval = getattr(args, "sse_keep_alive_interval", 0)
|
||||
return StreamingResponse(
|
||||
content=with_sse_keep_alive(generator, float(keep_alive_interval)),
|
||||
media_type="text/event-stream",
|
||||
)
|
||||
|
||||
|
||||
@router.post(
|
||||
|
||||
@@ -137,6 +137,13 @@ class BaseFrontendArgs:
|
||||
"""If set to True, enable tracking server_load_metrics in the app state."""
|
||||
enable_force_include_usage: bool = False
|
||||
"""If set to True, including usage on every request."""
|
||||
sse_keep_alive_interval: int = 0
|
||||
"""Send an SSE keep-alive comment line every this many seconds when a
|
||||
`/v1/chat/completions` or `/v1/completions` streaming response is idle
|
||||
(queued, prefill, or between tokens), to prevent reverse proxies/tunnels
|
||||
with read timeouts from closing the connection. Defaults to 0, which
|
||||
disables keep-alive comments entirely.
|
||||
"""
|
||||
enable_tokenizer_info_endpoint: bool = False
|
||||
"""Enable the `/tokenizer_info` endpoint. May expose chat
|
||||
templates and other tokenizer configuration."""
|
||||
@@ -426,6 +433,13 @@ def validate_parsed_serve_args(args: argparse.Namespace):
|
||||
if args.enable_log_outputs and not args.enable_log_requests:
|
||||
raise TypeError("Error: --enable-log-outputs requires --enable-log-requests")
|
||||
|
||||
# SSE keep-alive interval must be zero (disabled) or a positive integer.
|
||||
if getattr(args, "sse_keep_alive_interval", 0) < 0:
|
||||
raise ValueError(
|
||||
"Error: --sse-keep-alive-interval must be a non-negative integer "
|
||||
"(0 disables keep-alive comments)."
|
||||
)
|
||||
|
||||
if getattr(args, "enable_per_request_metrics", False) and getattr(
|
||||
args, "disable_log_stats", False
|
||||
):
|
||||
|
||||
@@ -13,6 +13,7 @@ from vllm.entrypoints.openai.completion.protocol import (
|
||||
)
|
||||
from vllm.entrypoints.openai.completion.serving import OpenAIServingCompletion
|
||||
from vllm.entrypoints.openai.engine.protocol import ErrorResponse
|
||||
from vllm.entrypoints.openai.sse_keep_alive import with_sse_keep_alive
|
||||
from vllm.entrypoints.serve.utils.api_utils import (
|
||||
load_aware_call,
|
||||
validate_json_request,
|
||||
@@ -63,7 +64,12 @@ async def create_completion(request: CompletionRequest, raw_request: Request):
|
||||
headers=metrics_header(metrics_header_format),
|
||||
)
|
||||
|
||||
return StreamingResponse(content=generator, media_type="text/event-stream")
|
||||
args = getattr(raw_request.app.state, "args", None)
|
||||
keep_alive_interval = getattr(args, "sse_keep_alive_interval", 0)
|
||||
return StreamingResponse(
|
||||
content=with_sse_keep_alive(generator, float(keep_alive_interval)),
|
||||
media_type="text/event-stream",
|
||||
)
|
||||
|
||||
|
||||
def attach_router(app: FastAPI):
|
||||
|
||||
@@ -0,0 +1,65 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""SSE keep-alive comments for idle streaming responses.
|
||||
|
||||
Reverse proxies and tunnels (Cloudflare Tunnel, NGINX ``proxy_read_timeout``,
|
||||
AWS ALB, ...) can close a streaming connection when no bytes are sent for a
|
||||
while. That happens while a request is queued, during prefill, or in gaps
|
||||
between tokens. ``with_sse_keep_alive`` wraps the final SSE generator and emits
|
||||
a comment line (``": keep-alive\\n\\n"``) whenever it has been idle for
|
||||
``interval`` seconds. Comments are ignored by SSE-compliant clients but still
|
||||
count as bytes for proxy read-timeout logic.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import contextlib
|
||||
from collections.abc import AsyncGenerator
|
||||
|
||||
SSE_KEEP_ALIVE_COMMENT = ": keep-alive\n\n"
|
||||
|
||||
|
||||
def with_sse_keep_alive(
|
||||
generator: AsyncGenerator[str, None],
|
||||
interval: float,
|
||||
) -> AsyncGenerator[str, None]:
|
||||
"""Emit an SSE keep-alive comment when ``generator`` is idle.
|
||||
|
||||
A non-positive or non-finite ``interval`` returns ``generator`` unchanged,
|
||||
so the default path has no overhead. Otherwise a keep-alive comment is
|
||||
yielded whenever no chunk arrives within ``interval`` seconds.
|
||||
"""
|
||||
if not 0 < interval < float("inf"):
|
||||
return generator
|
||||
return _keep_alive_stream(generator, interval)
|
||||
|
||||
|
||||
async def _keep_alive_stream(
|
||||
generator: AsyncGenerator[str, None],
|
||||
interval: float,
|
||||
) -> AsyncGenerator[str, None]:
|
||||
# A single retained ``anext()`` task advances the generator; asyncio forbids
|
||||
# concurrent ``__anext__`` calls, and ``wait_for`` would cancel the task
|
||||
# mid-advance on timeout. ``asyncio.wait`` observes both outcomes without
|
||||
# cancelling the pending advance.
|
||||
next_task: asyncio.Task[str] = asyncio.create_task(anext(generator))
|
||||
try:
|
||||
while True:
|
||||
done, _ = await asyncio.wait({next_task}, timeout=interval)
|
||||
if not done:
|
||||
yield SSE_KEEP_ALIVE_COMMENT
|
||||
continue
|
||||
try:
|
||||
chunk = next_task.result()
|
||||
except StopAsyncIteration:
|
||||
return
|
||||
yield chunk
|
||||
next_task = asyncio.create_task(anext(generator))
|
||||
finally:
|
||||
next_task.cancel()
|
||||
# The task outcome was already handled in the loop; suppress so cleanup
|
||||
# never corrupts it (matching ``merge_async_iterators``).
|
||||
with contextlib.suppress(BaseException):
|
||||
await next_task
|
||||
# Close the upstream generator so its cleanup runs deterministically.
|
||||
with contextlib.suppress(BaseException):
|
||||
await generator.aclose()
|
||||
Reference in New Issue
Block a user