[Frontend] Accept each connection on the least loaded API worker (#60723)
Signed-off-by: hao-aaron <aaronhao@inferact.ai> Co-authored-by: aoshen02 <aoshen02@users.noreply.github.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
aoshen02
Claude Opus 5.5
parent
81d9c45e4b
commit
fa312090cb
@@ -8,7 +8,7 @@ import signal
|
||||
import socket
|
||||
import tempfile
|
||||
from argparse import Namespace
|
||||
from collections.abc import AsyncIterator
|
||||
from collections.abc import AsyncIterator, MutableSequence
|
||||
from contextlib import asynccontextmanager
|
||||
from typing import Any
|
||||
|
||||
@@ -118,6 +118,7 @@ async def build_and_serve(
|
||||
listen_address: str,
|
||||
sock: socket.socket,
|
||||
args: Namespace,
|
||||
peer_loads: tuple[MutableSequence[int], int] | None = None,
|
||||
**uvicorn_kwargs,
|
||||
) -> asyncio.Task:
|
||||
"""Build FastAPI app, initialize state, and start serving.
|
||||
@@ -142,6 +143,7 @@ async def build_and_serve(
|
||||
app,
|
||||
sock=sock,
|
||||
enable_ssl_refresh=args.enable_ssl_refresh,
|
||||
peer_loads=peer_loads,
|
||||
host=args.host,
|
||||
port=args.port,
|
||||
log_level=args.uvicorn_log_level,
|
||||
@@ -185,12 +187,17 @@ async def run_server_worker(
|
||||
if args.reasoning_parser_plugin and len(args.reasoning_parser_plugin) > 3:
|
||||
ReasoningParserManager.import_reasoning_parser(args.reasoning_parser_plugin)
|
||||
|
||||
peer_loads = None
|
||||
if client_config and "api_server_loads" in client_config:
|
||||
loads = client_config.pop("api_server_loads")
|
||||
peer_loads = (loads, client_config["client_index"])
|
||||
|
||||
async with build_async_engine_client(
|
||||
args,
|
||||
client_config=client_config,
|
||||
) as engine_client:
|
||||
shutdown_task = await build_and_serve(
|
||||
engine_client, listen_address, sock, args, **uvicorn_kwargs
|
||||
engine_client, listen_address, sock, args, peer_loads, **uvicorn_kwargs
|
||||
)
|
||||
# NB: Await server shutdown only after the backend context is exited
|
||||
try:
|
||||
|
||||
Reference in New Issue
Block a user