[Frontend] Accept each connection on the least loaded API worker (#60723)

Signed-off-by: hao-aaron <aaronhao@inferact.ai>
Co-authored-by: aoshen02 <aoshen02@users.noreply.github.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Aaron Hao
2026-10-09 04:40:13 +00:00
committed by GitHub
co-authored by aoshen02 Claude Opus 5.5
parent 81d9c45e4b
commit fa312090cb
4 changed files with 365 additions and 4 deletions
@@ -8,7 +8,7 @@ import signal
import socket
import tempfile
from argparse import Namespace
from collections.abc import AsyncIterator
from collections.abc import AsyncIterator, MutableSequence
from contextlib import asynccontextmanager
from typing import Any
@@ -118,6 +118,7 @@ async def build_and_serve(
listen_address: str,
sock: socket.socket,
args: Namespace,
peer_loads: tuple[MutableSequence[int], int] | None = None,
**uvicorn_kwargs,
) -> asyncio.Task:
"""Build FastAPI app, initialize state, and start serving.
@@ -142,6 +143,7 @@ async def build_and_serve(
app,
sock=sock,
enable_ssl_refresh=args.enable_ssl_refresh,
peer_loads=peer_loads,
host=args.host,
port=args.port,
log_level=args.uvicorn_log_level,
@@ -185,12 +187,17 @@ async def run_server_worker(
if args.reasoning_parser_plugin and len(args.reasoning_parser_plugin) > 3:
ReasoningParserManager.import_reasoning_parser(args.reasoning_parser_plugin)
peer_loads = None
if client_config and "api_server_loads" in client_config:
loads = client_config.pop("api_server_loads")
peer_loads = (loads, client_config["client_index"])
async with build_async_engine_client(
args,
client_config=client_config,
) as engine_client:
shutdown_task = await build_and_serve(
engine_client, listen_address, sock, args, **uvicorn_kwargs
engine_client, listen_address, sock, args, peer_loads, **uvicorn_kwargs
)
# NB: Await server shutdown only after the backend context is exited
try: