[Frontend] Accept each connection on the least loaded API worker (#60723)
Signed-off-by: hao-aaron <aaronhao@inferact.ai> Co-authored-by: aoshen02 <aoshen02@users.noreply.github.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
aoshen02
Claude Opus 5.5
parent
81d9c45e4b
commit
fa312090cb
@@ -213,6 +213,14 @@ class APIServerProcessManager:
|
||||
SharedAdmissionStats.num_counters(num_servers),
|
||||
)
|
||||
|
||||
# Open connections per API server, to accept on the least loaded one;
|
||||
# a server that has not started yet is never the least loaded.
|
||||
api_server_loads = (
|
||||
spawn_context.RawArray("i", [2**31 - 1] * num_servers)
|
||||
if num_servers > 1
|
||||
else None
|
||||
)
|
||||
|
||||
listeners = [*input_listeners, *output_listeners]
|
||||
self._listeners = listeners
|
||||
try:
|
||||
@@ -228,6 +236,8 @@ class APIServerProcessManager:
|
||||
"client_count": num_servers,
|
||||
"client_index": i,
|
||||
}
|
||||
if api_server_loads is not None:
|
||||
client_config["api_server_loads"] = api_server_loads
|
||||
if admission_counters is not None:
|
||||
client_config["mp_admission_counters"] = admission_counters
|
||||
if stats_update_address is not None:
|
||||
|
||||
Reference in New Issue
Block a user