From fa312090cb1d972aa96f6b0ea174b7a32b4b0548 Mon Sep 17 00:00:00 2001 From: Aaron Hao Date: Thu, 8 Oct 2026 21:40:13 -0700 Subject: [PATCH] [Frontend] Accept each connection on the least loaded API worker (#60723) Signed-off-by: hao-aaron Co-authored-by: aoshen02 Co-authored-by: Claude Opus 5.5 --- .../launchers/test_balanced_accepts.py | 218 ++++++++++++++++++ .../entrypoints/launchers/api_server/entry.py | 11 +- vllm/entrypoints/launchers/launcher.py | 130 ++++++++++- vllm/v1/utils.py | 10 + 4 files changed, 365 insertions(+), 4 deletions(-) create mode 100644 tests/entrypoints/launchers/test_balanced_accepts.py diff --git a/tests/entrypoints/launchers/test_balanced_accepts.py b/tests/entrypoints/launchers/test_balanced_accepts.py new file mode 100644 index 0000000000..3ac1025ca8 --- /dev/null +++ b/tests/entrypoints/launchers/test_balanced_accepts.py @@ -0,0 +1,218 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""NoSignalServer(peer_loads=...): API workers sharing one listening socket +accept each connection on the worker with the fewest open connections.""" + +import asyncio +import datetime +import multiprocessing +import os +import socket +import ssl +from collections import Counter + +import pytest +import uvicorn +import uvloop + +from vllm.entrypoints.launchers.launcher import NoSignalServer + +TIMEOUT_S = 30 + + +def _bound_socket() -> socket.socket: + # Like setup_server: bound, not yet listening. + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + sock.bind(("127.0.0.1", 0)) + return sock + + +def _app(hold_s: float = 0.0): + pid = str(os.getpid()).encode() + + async def app(scope, receive, send): + if scope["type"] == "lifespan": + while (await receive())["type"] != "lifespan.shutdown": + await send({"type": "lifespan.startup.complete"}) + await send({"type": "lifespan.shutdown.complete"}) + return + await asyncio.sleep(hold_s) + headers = [(b"content-length", str(len(pid)).encode())] + await send({"type": "http.response.start", "status": 200, "headers": headers}) + await send({"type": "http.response.body", "body": pid}) + + return app + + +async def _get(port: int, requests: int = 1, ssl_ctx=None) -> list[bytes]: + """``requests`` requests on one keep-alive connection; returns the bodies.""" + reader, writer = await asyncio.open_connection("127.0.0.1", port, ssl=ssl_ctx) + bodies = [] + for _ in range(requests): + writer.write(b"GET / HTTP/1.1\r\nHost: t\r\n\r\n") + head = await asyncio.wait_for(reader.readuntil(b"\r\n\r\n"), TIMEOUT_S) + assert head.startswith(b"HTTP/1.1 200") + length = int(head.split(b"content-length: ")[1].split(b"\r\n")[0]) + bodies.append(await asyncio.wait_for(reader.readexactly(length), TIMEOUT_S)) + writer.close() + return bodies + + +async def _serve(sock, requests, loads=None, **config_kwargs): + config = uvicorn.Config(_app(), lifespan="on", log_level="warning", **config_kwargs) + loads = loads if loads is not None else [0] + server = NoSignalServer(config, peer_loads=(loads, 0)) + task = asyncio.create_task(server.serve(sockets=[sock])) + while not server.started: + assert not task.done(), task.result() + await asyncio.sleep(0.01) + try: + return await requests(sock.getsockname()[1], server) + finally: + server.should_exit = True + await asyncio.wait_for(task, TIMEOUT_S) + # The listener is closed, nothing is left accepting, and peers no + # longer see this worker as a target. + assert sock.fileno() == -1 + assert not server._accept_tasks + assert loads[0] == 2**31 - 1 + + +def _run(coro, loop_impl): + return uvloop.run(coro) if loop_impl == "uvloop" else asyncio.run(coro) + + +@pytest.mark.parametrize("loop_impl", ["asyncio", "uvloop"]) +def test_requests_and_keep_alive_served(loop_impl): + async def requests(port, server): + return await _get(port), await _get(port, requests=3) + + one, three = _run(_serve(_bound_socket(), requests), loop_impl) + assert one == [str(os.getpid()).encode()] + assert len(three) == 3 + + +def _self_signed(tmp_path): + x509 = pytest.importorskip("cryptography.x509") + from cryptography.hazmat.primitives import hashes, serialization + from cryptography.hazmat.primitives.asymmetric import ec + from cryptography.x509.oid import NameOID + + key = ec.generate_private_key(ec.SECP256R1()) + name = x509.Name([x509.NameAttribute(NameOID.COMMON_NAME, "127.0.0.1")]) + now = datetime.datetime.now(datetime.UTC) + cert = ( + x509.CertificateBuilder() + .subject_name(name) + .issuer_name(name) + .public_key(key.public_key()) + .serial_number(x509.random_serial_number()) + .not_valid_before(now - datetime.timedelta(minutes=1)) + .not_valid_after(now + datetime.timedelta(hours=1)) + .sign(key, hashes.SHA256()) + ) + keyfile, certfile = tmp_path / "key.pem", tmp_path / "cert.pem" + keyfile.write_bytes( + key.private_bytes( + serialization.Encoding.PEM, + serialization.PrivateFormat.PKCS8, + serialization.NoEncryption(), + ) + ) + certfile.write_bytes(cert.public_bytes(serialization.Encoding.PEM)) + return str(keyfile), str(certfile) + + +def test_tls_handshake_failure_keeps_serving(tmp_path): + keyfile, certfile = _self_signed(tmp_path) + client_ctx = ssl.create_default_context() + client_ctx.check_hostname = False + client_ctx.verify_mode = ssl.CERT_NONE + + async def requests(port, server): + # A plain-text client fails the handshake; the next TLS client is served. + reader, writer = await asyncio.open_connection("127.0.0.1", port) + writer.write(b"GET / HTTP/1.1\r\nHost: t\r\n\r\n") + await asyncio.wait_for(reader.read(), TIMEOUT_S) + writer.close() + return await _get(port, ssl_ctx=client_ctx) + + bodies = uvloop.run( + _serve(_bound_socket(), requests, ssl_keyfile=keyfile, ssl_certfile=certfile) + ) + assert bodies == [str(os.getpid()).encode()] + + +def test_busier_worker_still_serves(): + # This worker is index 0; peer 1 claims no connections but never accepts. + loads = [0, 0] + + async def requests(port, server): + reader, writer = await asyncio.open_connection("127.0.0.1", port) + bodies = await _get(port) + assert loads[0] >= 1 # the held connection was published + writer.close() + return bodies + + assert uvloop.run(_serve(_bound_socket(), requests, loads)) == [ + str(os.getpid()).encode() + ] + + +def _worker(sock, loads, index, ready, stop): + async def main(): + config = uvicorn.Config(_app(hold_s=1), lifespan="on", log_level="warning") + server = NoSignalServer(config, peer_loads=(loads, index)) + task = asyncio.create_task(server.serve(sockets=[sock])) + while not server.started: + await asyncio.sleep(0.01) + ready.release() + await asyncio.get_running_loop().run_in_executor(None, stop.wait) + server.should_exit = True + await task + + # API workers run on uvloop, whose own server accepts a whole burst at once. + uvloop.run(main()) + + +@pytest.mark.skipif((os.cpu_count() or 1) < 8, reason="needs idle cores per worker") +@pytest.mark.parametrize("arrival_s", [0, 1]) +def test_connections_balanced_over_workers(arrival_s): + """Both a burst and a steady stream of connections end up balanced; with + the event loop's own server one worker takes most of either.""" + workers, connections = 4, 400 + sock = _bound_socket() + ctx = multiprocessing.get_context("spawn") + ready, stop = ctx.Semaphore(0), ctx.Event() + loads = ctx.RawArray("i", workers) + procs = [ + ctx.Process(target=_worker, args=(sock, loads, i, ready, stop)) + for i in range(workers) + ] + for p in procs: + p.start() + try: + for _ in range(workers): + assert ready.acquire(timeout=120) + + async def get_after(delay): + await asyncio.sleep(delay) + return await _get(sock.getsockname()[1]) + + async def arrive(): + delays = [arrival_s * i / connections for i in range(connections)] + results = await asyncio.gather(*(get_after(d) for d in delays)) + return Counter(body for (body,) in results) + + counts = uvloop.run(arrive()) + assert sum(counts.values()) == connections + # Fair share is 100. + assert max(counts.values()) <= 5 * connections // (4 * workers), counts + finally: + stop.set() + for p in procs: + p.join(30) + if p.is_alive(): + p.kill() + sock.close() diff --git a/vllm/entrypoints/launchers/api_server/entry.py b/vllm/entrypoints/launchers/api_server/entry.py index 28bb696f2c..e5ba536e11 100644 --- a/vllm/entrypoints/launchers/api_server/entry.py +++ b/vllm/entrypoints/launchers/api_server/entry.py @@ -8,7 +8,7 @@ import signal import socket import tempfile from argparse import Namespace -from collections.abc import AsyncIterator +from collections.abc import AsyncIterator, MutableSequence from contextlib import asynccontextmanager from typing import Any @@ -118,6 +118,7 @@ async def build_and_serve( listen_address: str, sock: socket.socket, args: Namespace, + peer_loads: tuple[MutableSequence[int], int] | None = None, **uvicorn_kwargs, ) -> asyncio.Task: """Build FastAPI app, initialize state, and start serving. @@ -142,6 +143,7 @@ async def build_and_serve( app, sock=sock, enable_ssl_refresh=args.enable_ssl_refresh, + peer_loads=peer_loads, host=args.host, port=args.port, log_level=args.uvicorn_log_level, @@ -185,12 +187,17 @@ async def run_server_worker( if args.reasoning_parser_plugin and len(args.reasoning_parser_plugin) > 3: ReasoningParserManager.import_reasoning_parser(args.reasoning_parser_plugin) + peer_loads = None + if client_config and "api_server_loads" in client_config: + loads = client_config.pop("api_server_loads") + peer_loads = (loads, client_config["client_index"]) + async with build_async_engine_client( args, client_config=client_config, ) as engine_client: shutdown_task = await build_and_serve( - engine_client, listen_address, sock, args, **uvicorn_kwargs + engine_client, listen_address, sock, args, peer_loads, **uvicorn_kwargs ) # NB: Await server shutdown only after the backend context is exited try: diff --git a/vllm/entrypoints/launchers/launcher.py b/vllm/entrypoints/launchers/launcher.py index 9fabfa3277..44161439b6 100644 --- a/vllm/entrypoints/launchers/launcher.py +++ b/vllm/entrypoints/launchers/launcher.py @@ -5,7 +5,7 @@ import asyncio import contextlib import signal import socket -from collections.abc import Generator +from collections.abc import Generator, MutableSequence from functools import partial from typing import Any @@ -34,23 +34,149 @@ from .utils.constants import ( logger = init_logger(__name__) +# A worker with more connections than another leaves a pending connection to +# the less loaded one, rechecking every 1 ms, for up to 5 ms (it may be busy). +_ACCEPT_DEFER_CHECKS = 5 +# The load of a worker that is not accepting (also the initial value of the +# shared array), so it is never the least loaded. +_NOT_ACCEPTING = 2**31 - 1 + class NoSignalServer(uvicorn.Server): """Uvicorn server that never installs its own SIGINT/SIGTERM handlers. Callers register their own handlers on the event loop for graceful shutdown; uvicorn's would race with and override them (see #49668). + + With ``peer_loads`` (several API workers on one shared socket: a shared + array with one slot per worker, and this worker's index), each worker + publishes its number of open connections, and a pending connection is + accepted by the least loaded worker. The event loop's own server would + accept every queued connection in whichever worker wakes first. """ + def __init__( + self, + config: uvicorn.Config, + peer_loads: tuple[MutableSequence[int], int] | None = None, + ): + super().__init__(config) + self.peer_loads = peer_loads + self._accept_tasks: set[asyncio.Task] = set() + self._connecting = 0 + @contextlib.contextmanager def capture_signals(self) -> Generator[None, None, None]: yield + def _publish_load(self) -> None: + assert self.peer_loads is not None + loads, index = self.peer_loads + loads[index] = len(self.server_state.connections) + self._connecting + + def _least_loaded(self) -> bool: + assert self.peer_loads is not None + loads, index = self.peer_loads + return loads[index] <= min(loads) + + async def startup(self, sockets: list[socket.socket] | None = None) -> None: + if not (self.peer_loads and sockets): + return await super().startup(sockets) + # With no sockets uvicorn starts no asyncio server; _accept serves + # them with uvicorn's protocol instead. + await super().startup(sockets=[]) + if not self.started: # lifespan startup failed (uvicorn < 0.50) + return + self._publish_load() + for sock in sockets: + sock.listen(self.config.backlog) + sock.setblocking(False) + self._track(self._accept(sock)) + + def _track(self, coro) -> None: + task = asyncio.create_task(coro) + self._accept_tasks.add(task) + task.add_done_callback(self._accept_tasks.discard) + + async def _accept(self, sock: socket.socket) -> None: + loop = asyncio.get_running_loop() + while True: + await _readable(loop, sock) + # Connections closed since the last accept; peers refresh likewise. + self._publish_load() + for _ in range(_ACCEPT_DEFER_CHECKS): + if self._least_loaded(): + break + await asyncio.sleep(0.001) + try: + conn, _ = sock.accept() + except (BlockingIOError, InterruptedError, ConnectionAbortedError): + continue # another worker took it + except OSError: + # e.g. EMFILE; asyncio's own server also retries after 1 s. + logger.exception("Error accepting a connection") + await asyncio.sleep(1) + continue + self._connecting += 1 + self._publish_load() + # Its own task, so a TLS handshake does not hold up the next accept. + self._track(self._connect(loop, conn)) + + async def _connect(self, loop: asyncio.AbstractEventLoop, conn: socket.socket): + config = self.config + try: + await loop.connect_accepted_socket( + lambda: config.http_protocol_class( # type: ignore[call-arg] + config=config, + server_state=self.server_state, + app_state=self.lifespan.state, + _loop=loop, + ), + conn, + ssl=config.ssl, + ) + except asyncio.CancelledError: + conn.close() + raise + except Exception: + # e.g. a failed TLS handshake + logger.debug("Accepted connection failed to start", exc_info=True) + conn.close() + finally: + self._connecting -= 1 + self._publish_load() + + async def shutdown(self, sockets: list[socket.socket] | None = None) -> None: + # Before super() closes the listening sockets. + tasks = list(self._accept_tasks) + for task in tasks: + task.cancel() + await asyncio.gather(*tasks, return_exceptions=True) + if self.peer_loads is not None: + loads, index = self.peer_loads + loads[index] = _NOT_ACCEPTING + await super().shutdown(sockets) + + +async def _readable(loop: asyncio.AbstractEventLoop, sock: socket.socket) -> None: + ready: asyncio.Future[None] = loop.create_future() + + def wake() -> None: + if not ready.done(): + ready.set_result(None) + + loop.add_reader(sock, wake) + try: + await ready + finally: + loop.remove_reader(sock) + async def serve_http( app: FastAPI, sock: socket.socket | None, enable_ssl_refresh: bool = False, + peer_loads: tuple[MutableSequence[int], int] | None = None, **uvicorn_kwargs: Any, ): """Start a FastAPI app using Uvicorn, with support for custom Uvicorn config @@ -96,7 +222,7 @@ async def serve_http( config.h11_max_incomplete_event_size = h11_max_incomplete_event_size config.h11_max_header_count = h11_max_header_count config.load() - server = NoSignalServer(config) + server = NoSignalServer(config, peer_loads=peer_loads) app.state.server = server loop = asyncio.get_running_loop() diff --git a/vllm/v1/utils.py b/vllm/v1/utils.py index 868864f0ca..b8d98ff1cf 100644 --- a/vllm/v1/utils.py +++ b/vllm/v1/utils.py @@ -213,6 +213,14 @@ class APIServerProcessManager: SharedAdmissionStats.num_counters(num_servers), ) + # Open connections per API server, to accept on the least loaded one; + # a server that has not started yet is never the least loaded. + api_server_loads = ( + spawn_context.RawArray("i", [2**31 - 1] * num_servers) + if num_servers > 1 + else None + ) + listeners = [*input_listeners, *output_listeners] self._listeners = listeners try: @@ -228,6 +236,8 @@ class APIServerProcessManager: "client_count": num_servers, "client_index": i, } + if api_server_loads is not None: + client_config["api_server_loads"] = api_server_loads if admission_counters is not None: client_config["mp_admission_counters"] = admission_counters if stats_update_address is not None: