[Frontend] Accept each connection on the least loaded API worker (#60723)

Signed-off-by: hao-aaron <aaronhao@inferact.ai>
Co-authored-by: aoshen02 <aoshen02@users.noreply.github.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Aaron Hao
2026-10-09 04:40:13 +00:00
committed by GitHub
co-authored by aoshen02 Claude Opus 5.5
parent 81d9c45e4b
commit fa312090cb
4 changed files with 365 additions and 4 deletions
+10
View File
@@ -213,6 +213,14 @@ class APIServerProcessManager:
SharedAdmissionStats.num_counters(num_servers),
)
# Open connections per API server, to accept on the least loaded one;
# a server that has not started yet is never the least loaded.
api_server_loads = (
spawn_context.RawArray("i", [2**31 - 1] * num_servers)
if num_servers > 1
else None
)
listeners = [*input_listeners, *output_listeners]
self._listeners = listeners
try:
@@ -228,6 +236,8 @@ class APIServerProcessManager:
"client_count": num_servers,
"client_index": i,
}
if api_server_loads is not None:
client_config["api_server_loads"] = api_server_loads
if admission_counters is not None:
client_config["mp_admission_counters"] = admission_counters
if stats_update_address is not None: