[Frontend] Accept each connection on the least loaded API worker (#60723)

Signed-off-by: hao-aaron <aaronhao@inferact.ai>
Co-authored-by: aoshen02 <aoshen02@users.noreply.github.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Aaron Hao
2026-10-09 04:40:13 +00:00
committed by GitHub
co-authored by aoshen02 Claude Opus 5.5
parent 81d9c45e4b
commit fa312090cb
4 changed files with 365 additions and 4 deletions
@@ -0,0 +1,218 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""NoSignalServer(peer_loads=...): API workers sharing one listening socket
accept each connection on the worker with the fewest open connections."""
import asyncio
import datetime
import multiprocessing
import os
import socket
import ssl
from collections import Counter
import pytest
import uvicorn
import uvloop
from vllm.entrypoints.launchers.launcher import NoSignalServer
TIMEOUT_S = 30
def _bound_socket() -> socket.socket:
# Like setup_server: bound, not yet listening.
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
sock.bind(("127.0.0.1", 0))
return sock
def _app(hold_s: float = 0.0):
pid = str(os.getpid()).encode()
async def app(scope, receive, send):
if scope["type"] == "lifespan":
while (await receive())["type"] != "lifespan.shutdown":
await send({"type": "lifespan.startup.complete"})
await send({"type": "lifespan.shutdown.complete"})
return
await asyncio.sleep(hold_s)
headers = [(b"content-length", str(len(pid)).encode())]
await send({"type": "http.response.start", "status": 200, "headers": headers})
await send({"type": "http.response.body", "body": pid})
return app
async def _get(port: int, requests: int = 1, ssl_ctx=None) -> list[bytes]:
"""``requests`` requests on one keep-alive connection; returns the bodies."""
reader, writer = await asyncio.open_connection("127.0.0.1", port, ssl=ssl_ctx)
bodies = []
for _ in range(requests):
writer.write(b"GET / HTTP/1.1\r\nHost: t\r\n\r\n")
head = await asyncio.wait_for(reader.readuntil(b"\r\n\r\n"), TIMEOUT_S)
assert head.startswith(b"HTTP/1.1 200")
length = int(head.split(b"content-length: ")[1].split(b"\r\n")[0])
bodies.append(await asyncio.wait_for(reader.readexactly(length), TIMEOUT_S))
writer.close()
return bodies
async def _serve(sock, requests, loads=None, **config_kwargs):
config = uvicorn.Config(_app(), lifespan="on", log_level="warning", **config_kwargs)
loads = loads if loads is not None else [0]
server = NoSignalServer(config, peer_loads=(loads, 0))
task = asyncio.create_task(server.serve(sockets=[sock]))
while not server.started:
assert not task.done(), task.result()
await asyncio.sleep(0.01)
try:
return await requests(sock.getsockname()[1], server)
finally:
server.should_exit = True
await asyncio.wait_for(task, TIMEOUT_S)
# The listener is closed, nothing is left accepting, and peers no
# longer see this worker as a target.
assert sock.fileno() == -1
assert not server._accept_tasks
assert loads[0] == 2**31 - 1
def _run(coro, loop_impl):
return uvloop.run(coro) if loop_impl == "uvloop" else asyncio.run(coro)
@pytest.mark.parametrize("loop_impl", ["asyncio", "uvloop"])
def test_requests_and_keep_alive_served(loop_impl):
async def requests(port, server):
return await _get(port), await _get(port, requests=3)
one, three = _run(_serve(_bound_socket(), requests), loop_impl)
assert one == [str(os.getpid()).encode()]
assert len(three) == 3
def _self_signed(tmp_path):
x509 = pytest.importorskip("cryptography.x509")
from cryptography.hazmat.primitives import hashes, serialization
from cryptography.hazmat.primitives.asymmetric import ec
from cryptography.x509.oid import NameOID
key = ec.generate_private_key(ec.SECP256R1())
name = x509.Name([x509.NameAttribute(NameOID.COMMON_NAME, "127.0.0.1")])
now = datetime.datetime.now(datetime.UTC)
cert = (
x509.CertificateBuilder()
.subject_name(name)
.issuer_name(name)
.public_key(key.public_key())
.serial_number(x509.random_serial_number())
.not_valid_before(now - datetime.timedelta(minutes=1))
.not_valid_after(now + datetime.timedelta(hours=1))
.sign(key, hashes.SHA256())
)
keyfile, certfile = tmp_path / "key.pem", tmp_path / "cert.pem"
keyfile.write_bytes(
key.private_bytes(
serialization.Encoding.PEM,
serialization.PrivateFormat.PKCS8,
serialization.NoEncryption(),
)
)
certfile.write_bytes(cert.public_bytes(serialization.Encoding.PEM))
return str(keyfile), str(certfile)
def test_tls_handshake_failure_keeps_serving(tmp_path):
keyfile, certfile = _self_signed(tmp_path)
client_ctx = ssl.create_default_context()
client_ctx.check_hostname = False
client_ctx.verify_mode = ssl.CERT_NONE
async def requests(port, server):
# A plain-text client fails the handshake; the next TLS client is served.
reader, writer = await asyncio.open_connection("127.0.0.1", port)
writer.write(b"GET / HTTP/1.1\r\nHost: t\r\n\r\n")
await asyncio.wait_for(reader.read(), TIMEOUT_S)
writer.close()
return await _get(port, ssl_ctx=client_ctx)
bodies = uvloop.run(
_serve(_bound_socket(), requests, ssl_keyfile=keyfile, ssl_certfile=certfile)
)
assert bodies == [str(os.getpid()).encode()]
def test_busier_worker_still_serves():
# This worker is index 0; peer 1 claims no connections but never accepts.
loads = [0, 0]
async def requests(port, server):
reader, writer = await asyncio.open_connection("127.0.0.1", port)
bodies = await _get(port)
assert loads[0] >= 1 # the held connection was published
writer.close()
return bodies
assert uvloop.run(_serve(_bound_socket(), requests, loads)) == [
str(os.getpid()).encode()
]
def _worker(sock, loads, index, ready, stop):
async def main():
config = uvicorn.Config(_app(hold_s=1), lifespan="on", log_level="warning")
server = NoSignalServer(config, peer_loads=(loads, index))
task = asyncio.create_task(server.serve(sockets=[sock]))
while not server.started:
await asyncio.sleep(0.01)
ready.release()
await asyncio.get_running_loop().run_in_executor(None, stop.wait)
server.should_exit = True
await task
# API workers run on uvloop, whose own server accepts a whole burst at once.
uvloop.run(main())
@pytest.mark.skipif((os.cpu_count() or 1) < 8, reason="needs idle cores per worker")
@pytest.mark.parametrize("arrival_s", [0, 1])
def test_connections_balanced_over_workers(arrival_s):
"""Both a burst and a steady stream of connections end up balanced; with
the event loop's own server one worker takes most of either."""
workers, connections = 4, 400
sock = _bound_socket()
ctx = multiprocessing.get_context("spawn")
ready, stop = ctx.Semaphore(0), ctx.Event()
loads = ctx.RawArray("i", workers)
procs = [
ctx.Process(target=_worker, args=(sock, loads, i, ready, stop))
for i in range(workers)
]
for p in procs:
p.start()
try:
for _ in range(workers):
assert ready.acquire(timeout=120)
async def get_after(delay):
await asyncio.sleep(delay)
return await _get(sock.getsockname()[1])
async def arrive():
delays = [arrival_s * i / connections for i in range(connections)]
results = await asyncio.gather(*(get_after(d) for d in delays))
return Counter(body for (body,) in results)
counts = uvloop.run(arrive())
assert sum(counts.values()) == connections
# Fair share is 100.
assert max(counts.values()) <= 5 * connections // (4 * workers), counts
finally:
stop.set()
for p in procs:
p.join(30)
if p.is_alive():
p.kill()
sock.close()
@@ -8,7 +8,7 @@ import signal
import socket
import tempfile
from argparse import Namespace
from collections.abc import AsyncIterator
from collections.abc import AsyncIterator, MutableSequence
from contextlib import asynccontextmanager
from typing import Any
@@ -118,6 +118,7 @@ async def build_and_serve(
listen_address: str,
sock: socket.socket,
args: Namespace,
peer_loads: tuple[MutableSequence[int], int] | None = None,
**uvicorn_kwargs,
) -> asyncio.Task:
"""Build FastAPI app, initialize state, and start serving.
@@ -142,6 +143,7 @@ async def build_and_serve(
app,
sock=sock,
enable_ssl_refresh=args.enable_ssl_refresh,
peer_loads=peer_loads,
host=args.host,
port=args.port,
log_level=args.uvicorn_log_level,
@@ -185,12 +187,17 @@ async def run_server_worker(
if args.reasoning_parser_plugin and len(args.reasoning_parser_plugin) > 3:
ReasoningParserManager.import_reasoning_parser(args.reasoning_parser_plugin)
peer_loads = None
if client_config and "api_server_loads" in client_config:
loads = client_config.pop("api_server_loads")
peer_loads = (loads, client_config["client_index"])
async with build_async_engine_client(
args,
client_config=client_config,
) as engine_client:
shutdown_task = await build_and_serve(
engine_client, listen_address, sock, args, **uvicorn_kwargs
engine_client, listen_address, sock, args, peer_loads, **uvicorn_kwargs
)
# NB: Await server shutdown only after the backend context is exited
try:
+128 -2
View File
@@ -5,7 +5,7 @@ import asyncio
import contextlib
import signal
import socket
from collections.abc import Generator
from collections.abc import Generator, MutableSequence
from functools import partial
from typing import Any
@@ -34,23 +34,149 @@ from .utils.constants import (
logger = init_logger(__name__)
# A worker with more connections than another leaves a pending connection to
# the less loaded one, rechecking every 1 ms, for up to 5 ms (it may be busy).
_ACCEPT_DEFER_CHECKS = 5
# The load of a worker that is not accepting (also the initial value of the
# shared array), so it is never the least loaded.
_NOT_ACCEPTING = 2**31 - 1
class NoSignalServer(uvicorn.Server):
"""Uvicorn server that never installs its own SIGINT/SIGTERM handlers.
Callers register their own handlers on the event loop for graceful
shutdown; uvicorn's would race with and override them (see #49668).
With ``peer_loads`` (several API workers on one shared socket: a shared
array with one slot per worker, and this worker's index), each worker
publishes its number of open connections, and a pending connection is
accepted by the least loaded worker. The event loop's own server would
accept every queued connection in whichever worker wakes first.
"""
def __init__(
self,
config: uvicorn.Config,
peer_loads: tuple[MutableSequence[int], int] | None = None,
):
super().__init__(config)
self.peer_loads = peer_loads
self._accept_tasks: set[asyncio.Task] = set()
self._connecting = 0
@contextlib.contextmanager
def capture_signals(self) -> Generator[None, None, None]:
yield
def _publish_load(self) -> None:
assert self.peer_loads is not None
loads, index = self.peer_loads
loads[index] = len(self.server_state.connections) + self._connecting
def _least_loaded(self) -> bool:
assert self.peer_loads is not None
loads, index = self.peer_loads
return loads[index] <= min(loads)
async def startup(self, sockets: list[socket.socket] | None = None) -> None:
if not (self.peer_loads and sockets):
return await super().startup(sockets)
# With no sockets uvicorn starts no asyncio server; _accept serves
# them with uvicorn's protocol instead.
await super().startup(sockets=[])
if not self.started: # lifespan startup failed (uvicorn < 0.50)
return
self._publish_load()
for sock in sockets:
sock.listen(self.config.backlog)
sock.setblocking(False)
self._track(self._accept(sock))
def _track(self, coro) -> None:
task = asyncio.create_task(coro)
self._accept_tasks.add(task)
task.add_done_callback(self._accept_tasks.discard)
async def _accept(self, sock: socket.socket) -> None:
loop = asyncio.get_running_loop()
while True:
await _readable(loop, sock)
# Connections closed since the last accept; peers refresh likewise.
self._publish_load()
for _ in range(_ACCEPT_DEFER_CHECKS):
if self._least_loaded():
break
await asyncio.sleep(0.001)
try:
conn, _ = sock.accept()
except (BlockingIOError, InterruptedError, ConnectionAbortedError):
continue # another worker took it
except OSError:
# e.g. EMFILE; asyncio's own server also retries after 1 s.
logger.exception("Error accepting a connection")
await asyncio.sleep(1)
continue
self._connecting += 1
self._publish_load()
# Its own task, so a TLS handshake does not hold up the next accept.
self._track(self._connect(loop, conn))
async def _connect(self, loop: asyncio.AbstractEventLoop, conn: socket.socket):
config = self.config
try:
await loop.connect_accepted_socket(
lambda: config.http_protocol_class( # type: ignore[call-arg]
config=config,
server_state=self.server_state,
app_state=self.lifespan.state,
_loop=loop,
),
conn,
ssl=config.ssl,
)
except asyncio.CancelledError:
conn.close()
raise
except Exception:
# e.g. a failed TLS handshake
logger.debug("Accepted connection failed to start", exc_info=True)
conn.close()
finally:
self._connecting -= 1
self._publish_load()
async def shutdown(self, sockets: list[socket.socket] | None = None) -> None:
# Before super() closes the listening sockets.
tasks = list(self._accept_tasks)
for task in tasks:
task.cancel()
await asyncio.gather(*tasks, return_exceptions=True)
if self.peer_loads is not None:
loads, index = self.peer_loads
loads[index] = _NOT_ACCEPTING
await super().shutdown(sockets)
async def _readable(loop: asyncio.AbstractEventLoop, sock: socket.socket) -> None:
ready: asyncio.Future[None] = loop.create_future()
def wake() -> None:
if not ready.done():
ready.set_result(None)
loop.add_reader(sock, wake)
try:
await ready
finally:
loop.remove_reader(sock)
async def serve_http(
app: FastAPI,
sock: socket.socket | None,
enable_ssl_refresh: bool = False,
peer_loads: tuple[MutableSequence[int], int] | None = None,
**uvicorn_kwargs: Any,
):
"""Start a FastAPI app using Uvicorn, with support for custom Uvicorn config
@@ -96,7 +222,7 @@ async def serve_http(
config.h11_max_incomplete_event_size = h11_max_incomplete_event_size
config.h11_max_header_count = h11_max_header_count
config.load()
server = NoSignalServer(config)
server = NoSignalServer(config, peer_loads=peer_loads)
app.state.server = server
loop = asyncio.get_running_loop()
+10
View File
@@ -213,6 +213,14 @@ class APIServerProcessManager:
SharedAdmissionStats.num_counters(num_servers),
)
# Open connections per API server, to accept on the least loaded one;
# a server that has not started yet is never the least loaded.
api_server_loads = (
spawn_context.RawArray("i", [2**31 - 1] * num_servers)
if num_servers > 1
else None
)
listeners = [*input_listeners, *output_listeners]
self._listeners = listeners
try:
@@ -228,6 +236,8 @@ class APIServerProcessManager:
"client_count": num_servers,
"client_index": i,
}
if api_server_loads is not None:
client_config["api_server_loads"] = api_server_loads
if admission_counters is not None:
client_config["mp_admission_counters"] = admission_counters
if stats_update_address is not None: