[Bugfix] MoRIIO toy P/D proxy: fix DP-rank index aliasing + harden for high-concurrency bursts (#46115)

Signed-off-by: Edwin Lim <edwin.lim@mangoboost.io>
Co-authored-by: Claude <noreply@anthropic.com>
Co-authored-by: QinPR <1905873179@qq.com>
Co-authored-by: Peiran Qin <66068739+QinPR@users.noreply.github.com>
This commit is contained in:
limeward
2026-07-17 12:35:04 -07:00
committed by GitHub
co-authored by Claude QinPR Peiran Qin
parent 5784507da4
commit c4cd2bd544
2 changed files with 268 additions and 13 deletions
@@ -210,8 +210,31 @@ async def stream_decode_response(session, response, request_id):
await session.close()
def example_round_robin_dp_loader(request_number, dp_size):
return request_nums % dp_size
def flat_interleaved_dp_route(request_number, instances):
"""Flat round-robin over the full (instance, dp_rank) slot space.
ONE counter over (n_instances * dp_size) slots, so instance-selection and
DP-rank-selection are derived from the SAME index and can never alias. The
previous scheme computed instance = req % n and rank = req % dp from the
same counter with n | dp, which locked each instance to a stride-n subset
of its ranks (e.g. 2 prefill instances -> 4 of 8 ranks each -> half the
GPUs never receive a request, so the deployment falsely appears not to
scale).
Interleaved order — inst0_r0, inst1_r0, inst0_r1, inst1_r1, ... — so
consecutive requests alternate instances AND every rank gets walked.
Assumes homogeneous dp_size across a role's instances (true for the
DP<->DP and DP<->TP deployments this proxy targets). Returns
(instance_index, dp_rank); dp_rank is None when dp_size == 1 (e.g. a TP
decode), which avoids forwarding an out-of-range data-parallel rank.
"""
n = len(instances)
dp = instances[0]["dp_size"]
slot = (request_number - 1) % (n * dp)
inst_idx = slot % n
dp_rank = (slot // n) if dp > 1 else None
return inst_idx, dp_rank
@app.route("/health", methods=["GET"])
@@ -252,18 +275,21 @@ async def handle_request(api: str, request: Request):
503,
)
)
pid = request_nums % len(prefill_instances)
did = request_nums % len(decode_instances)
# Flat interleaved round-robin (see flat_interleaved_dp_route): ONE
# counter over the full (instance, dp_rank) slot space per role, so
# instance-selection and DP-rank-selection derive from the same index
# and can never alias. The old scheme keyed both on request_nums with
# n_instances | dp_size, stranding half the ranks (e.g. in 2P_DP8EP).
pid, selected_prefill_dp_rank = flat_interleaved_dp_route(
request_nums, prefill_instances
)
# Decode instance selection uses the same interleaved walk; in READ
# mode the decode reads KV from selected_prefill_dp_rank, so the
# decode's own dp_rank is not forwarded here.
did, _ = flat_interleaved_dp_route(request_nums, decode_instances)
prefill_instance_endpoint = prefill_instances[pid]
decode_instance_endpoint = decode_instances[did]
selected_prefill_dp_rank = None
if prefill_instance_endpoint["dp_size"] > 1:
selected_prefill_dp_rank = example_round_robin_dp_loader(
request_nums // len(prefill_instance_endpoint),
prefill_instance_endpoint["dp_size"],
)
# Embed both zmq_addresses in the request_id so the connector can parse
# the peer's host/ports from it, similar to P2P-NCCL
uid = str(uuid.uuid4()).replace("-", "")
@@ -427,9 +453,33 @@ if __name__ == "__main__":
args = parser.parse_args()
t = start_service_discovery("0.0.0.0", 36367)
app.debug = True
# High-concurrency hardening. Quart's app.run() uses a shallow listen
# backlog (100) and, with app.debug=True, adds per-request overhead that
# starves the single accept loop. Under a burst of ~512 simultaneous client
# connections the backlog overflows and the kernel RSTs the excess, so
# clients see "ClientOSError: [Errno 104] Connection reset by peer" before
# any response (~16% request loss at c=512). Serve via hypercorn with debug
# OFF and a deep backlog so the burst QUEUES (higher TTFT) instead of being
# reset -> 100% request success.
app.debug = False
app.config["BODY_TIMEOUT"] = 360000
app.config["RESPONSE_TIMEOUT"] = 360000
app.run(host="0.0.0.0", port=args.port)
import asyncio
import os
from hypercorn.asyncio import serve as _hypercorn_serve
from hypercorn.config import Config as _HypercornConfig
_hcfg = _HypercornConfig()
_hcfg.bind = [f"0.0.0.0:{args.port}"]
# Deep listen backlog so a wide connection burst queues, not RSTs. NOTE:
# effective backlog is capped by the host's net.core.somaxconn (proxy runs
# --network host); kernel 6.x defaults to 4096. Override via
# PROXY_LISTEN_BACKLOG.
_hcfg.backlog = int(os.environ.get("PROXY_LISTEN_BACKLOG", "4096"))
# Long-lived SSE streams (8k1k decode ~5 min): never reap on keepalive.
_hcfg.keep_alive_timeout = 360000.0
asyncio.run(_hypercorn_serve(app, _hcfg))
t.join()