[Bugfix] MoRIIO toy P/D proxy: fix DP-rank index aliasing + harden for high-concurrency bursts (#46115)

Signed-off-by: Edwin Lim <edwin.lim@mangoboost.io>
Co-authored-by: Claude <noreply@anthropic.com>
Co-authored-by: QinPR <1905873179@qq.com>
Co-authored-by: Peiran Qin <66068739+QinPR@users.noreply.github.com>
This commit is contained in:
limeward
2026-07-17 12:35:04 -07:00
committed by GitHub
co-authored by Claude QinPR Peiran Qin
parent 5784507da4
commit c4cd2bd544
2 changed files with 268 additions and 13 deletions
@@ -210,8 +210,31 @@ async def stream_decode_response(session, response, request_id):
await session.close()
def example_round_robin_dp_loader(request_number, dp_size):
return request_nums % dp_size
def flat_interleaved_dp_route(request_number, instances):
"""Flat round-robin over the full (instance, dp_rank) slot space.
ONE counter over (n_instances * dp_size) slots, so instance-selection and
DP-rank-selection are derived from the SAME index and can never alias. The
previous scheme computed instance = req % n and rank = req % dp from the
same counter with n | dp, which locked each instance to a stride-n subset
of its ranks (e.g. 2 prefill instances -> 4 of 8 ranks each -> half the
GPUs never receive a request, so the deployment falsely appears not to
scale).
Interleaved order — inst0_r0, inst1_r0, inst0_r1, inst1_r1, ... — so
consecutive requests alternate instances AND every rank gets walked.
Assumes homogeneous dp_size across a role's instances (true for the
DP<->DP and DP<->TP deployments this proxy targets). Returns
(instance_index, dp_rank); dp_rank is None when dp_size == 1 (e.g. a TP
decode), which avoids forwarding an out-of-range data-parallel rank.
"""
n = len(instances)
dp = instances[0]["dp_size"]
slot = (request_number - 1) % (n * dp)
inst_idx = slot % n
dp_rank = (slot // n) if dp > 1 else None
return inst_idx, dp_rank
@app.route("/health", methods=["GET"])
@@ -252,18 +275,21 @@ async def handle_request(api: str, request: Request):
503,
)
)
pid = request_nums % len(prefill_instances)
did = request_nums % len(decode_instances)
# Flat interleaved round-robin (see flat_interleaved_dp_route): ONE
# counter over the full (instance, dp_rank) slot space per role, so
# instance-selection and DP-rank-selection derive from the same index
# and can never alias. The old scheme keyed both on request_nums with
# n_instances | dp_size, stranding half the ranks (e.g. in 2P_DP8EP).
pid, selected_prefill_dp_rank = flat_interleaved_dp_route(
request_nums, prefill_instances
)
# Decode instance selection uses the same interleaved walk; in READ
# mode the decode reads KV from selected_prefill_dp_rank, so the
# decode's own dp_rank is not forwarded here.
did, _ = flat_interleaved_dp_route(request_nums, decode_instances)
prefill_instance_endpoint = prefill_instances[pid]
decode_instance_endpoint = decode_instances[did]
selected_prefill_dp_rank = None
if prefill_instance_endpoint["dp_size"] > 1:
selected_prefill_dp_rank = example_round_robin_dp_loader(
request_nums // len(prefill_instance_endpoint),
prefill_instance_endpoint["dp_size"],
)
# Embed both zmq_addresses in the request_id so the connector can parse
# the peer's host/ports from it, similar to P2P-NCCL
uid = str(uuid.uuid4()).replace("-", "")
@@ -427,9 +453,33 @@ if __name__ == "__main__":
args = parser.parse_args()
t = start_service_discovery("0.0.0.0", 36367)
app.debug = True
# High-concurrency hardening. Quart's app.run() uses a shallow listen
# backlog (100) and, with app.debug=True, adds per-request overhead that
# starves the single accept loop. Under a burst of ~512 simultaneous client
# connections the backlog overflows and the kernel RSTs the excess, so
# clients see "ClientOSError: [Errno 104] Connection reset by peer" before
# any response (~16% request loss at c=512). Serve via hypercorn with debug
# OFF and a deep backlog so the burst QUEUES (higher TTFT) instead of being
# reset -> 100% request success.
app.debug = False
app.config["BODY_TIMEOUT"] = 360000
app.config["RESPONSE_TIMEOUT"] = 360000
app.run(host="0.0.0.0", port=args.port)
import asyncio
import os
from hypercorn.asyncio import serve as _hypercorn_serve
from hypercorn.config import Config as _HypercornConfig
_hcfg = _HypercornConfig()
_hcfg.bind = [f"0.0.0.0:{args.port}"]
# Deep listen backlog so a wide connection burst queues, not RSTs. NOTE:
# effective backlog is capped by the host's net.core.somaxconn (proxy runs
# --network host); kernel 6.x defaults to 4096. Override via
# PROXY_LISTEN_BACKLOG.
_hcfg.backlog = int(os.environ.get("PROXY_LISTEN_BACKLOG", "4096"))
# Long-lived SSE streams (8k1k decode ~5 min): never reap on keepalive.
_hcfg.keep_alive_timeout = 360000.0
asyncio.run(_hypercorn_serve(app, _hcfg))
t.join()
@@ -0,0 +1,205 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Hardware-fair request routing in the MoRIIO toy P/D proxy.
Exercises the REAL ``flat_interleaved_dp_route`` from the toy proxy (loaded from
its ``examples/`` path with heavy deps stubbed) — no routing logic is copied
here. This is the request-distribution half of end-to-end fairness: the proxy
must hand every prefill/decode (instance, dp_rank) slot an equal share so no GPU
is starved. The connector-side read routing is covered in
``test_moriio_routing_fairness.py``.
Regression target: the previous scheme derived ``instance = req % n_instances``
and ``dp_rank = req % dp_size`` from the same counter, so when
``n_instances | dp_size`` each instance was locked to a stride-``n`` subset of
its ranks — e.g. 2 prefill instances x DP8 stranded 4 of every node's 8 GPUs.
``flat_interleaved_dp_route`` walks ONE counter over the full
``(instance, dp_rank)`` slot space, so the two selections can never alias.
Role shapes below are exactly those in the RFC (#46107) deployments:
(1, 1) 1P/1D TP8 (2, 1) 2P/2D TP8 (4, 1) 4D TP8
(1, 8) 1D DP8EP (2, 8) 2P DP8EP (3, 8) 3D DP8EP
"""
import contextlib
import importlib.util
import sys
import types
from collections import Counter
from pathlib import Path
from typing import cast
import pytest
_MISSING = object()
PROXY_REL = "examples/disaggregated/disaggregated_serving/moriio_toy_proxy_server.py"
def _module(name, **attrs):
module = types.ModuleType(name)
for attr, value in attrs.items():
setattr(module, attr, value)
return module
def _package(name):
module = _module(name)
module.__path__ = []
return module
class _QuartStub:
def __init__(self, *a, **k):
pass
def route(self, *a, **k):
return lambda fn: fn
def post(self, *a, **k):
return self.route()
async def _make_response_stub(value):
return value
@contextlib.contextmanager
def _proxy_import_stubs():
"""Stub the proxy's external deps so the module imports for a pure unit test.
Only third-party/vllm imports are stubbed; the routing function under test
is executed as-is from the real module.
"""
common = "vllm.distributed.kv_transfer.kv_connector.v1.moriio.moriio_common"
class _MoRIIOConstants:
TRANSFER_PREFIX = "moriio-transfer"
stubs = {
"aiohttp": _module("aiohttp"),
"msgpack": _module("msgpack"),
"zmq": _module("zmq"),
"quart": _module(
"quart",
Quart=_QuartStub,
Request=object,
make_response=_make_response_stub,
request=object(),
),
"vllm": _package("vllm"),
"vllm.distributed": _package("vllm.distributed"),
"vllm.distributed.kv_transfer": _package("vllm.distributed.kv_transfer"),
"vllm.distributed.kv_transfer.kv_connector": _package(
"vllm.distributed.kv_transfer.kv_connector"
),
"vllm.distributed.kv_transfer.kv_connector.v1": _package(
"vllm.distributed.kv_transfer.kv_connector.v1"
),
"vllm.distributed.kv_transfer.kv_connector.v1.moriio": _package(
"vllm.distributed.kv_transfer.kv_connector.v1.moriio"
),
common: _module(common, MoRIIOConstants=_MoRIIOConstants),
}
saved = {}
for name, module in stubs.items():
if name not in sys.modules:
saved[name] = _MISSING
sys.modules[name] = module
try:
yield
finally:
for name, previous in saved.items():
if previous is _MISSING:
sys.modules.pop(name, None)
else:
sys.modules[name] = cast(types.ModuleType, previous)
def _load_proxy_module():
path = Path(__file__).parents[4] / PROXY_REL
spec = importlib.util.spec_from_file_location("moriio_proxy_under_test", path)
assert spec is not None and spec.loader is not None
module = importlib.util.module_from_spec(spec)
with _proxy_import_stubs():
spec.loader.exec_module(module)
return module
@pytest.fixture(scope="module")
def route():
return _load_proxy_module().flat_interleaved_dp_route
def _instances(n: int, dp_size: int):
# Only dp_size is read by the router; tp_size carried for realism.
return [{"dp_size": dp_size, "tp_size": 8 // dp_size} for _ in range(n)]
# (n_instances, dp_size) for every distinct P/D role shape in the RFC configs.
ROLE_SHAPES = [(1, 1), (2, 1), (4, 1), (1, 8), (2, 8), (3, 8)]
SHAPE_IDS = [f"n{n}_dp{dp}" for n, dp in ROLE_SHAPES]
def _route_n(route, instances, count):
# Proxy uses 1-indexed request numbers (slot = (request_number - 1) % ...).
return [route(rn, instances) for rn in range(1, count + 1)]
@pytest.mark.parametrize(("n", "dp"), ROLE_SHAPES, ids=SHAPE_IDS)
def test_full_slot_space_is_covered_uniformly(route, n, dp):
instances = _instances(n, dp)
period = n * dp
# Three full cycles -> every (instance, dp_rank) slot must be hit the same
# number of times (exactly uniform, no starved slot, no aliasing).
hits = Counter(_route_n(route, instances, period * 3))
expected: set[tuple[int, int | None]]
if dp == 1:
expected = {(inst, None) for inst in range(n)}
else:
expected = {(inst, r) for inst in range(n) for r in range(dp)}
assert set(hits) == expected, f"missing slots: {expected - set(hits)}"
assert max(hits.values()) == min(hits.values())
@pytest.mark.parametrize(("n", "dp"), ROLE_SHAPES, ids=SHAPE_IDS)
def test_instance_and_dp_rank_marginals_are_balanced(route, n, dp):
instances = _instances(n, dp)
routed = _route_n(route, instances, n * dp * 5)
inst_counts = Counter(inst for inst, _ in routed)
assert set(inst_counts) == set(range(n))
assert max(inst_counts.values()) == min(inst_counts.values())
dp_counts = Counter(r for _, r in routed)
if dp == 1:
assert set(dp_counts) == {None}
else:
assert set(dp_counts) == set(range(dp))
assert max(dp_counts.values()) == min(dp_counts.values())
def test_tp_instance_forwards_no_dp_rank(route):
# dp_size == 1 (a TP instance) must yield dp_rank None so the proxy never
# forwards an out-of-range data-parallel rank.
assert all(r is None for _, r in _route_n(route, _instances(2, 1), 8))
def test_two_instance_dp8_gives_every_node_all_ranks(route):
# Direct regression for the stranded-GPU bug: with 2 prefill instances x DP8
# each instance must receive ALL 8 dp ranks (16 distinct slots), not a
# stride-2 subset of 4.
routed = _route_n(route, _instances(2, 8), 2 * 8)
per_instance: dict[int, set] = {0: set(), 1: set()}
for inst, dp_rank in routed:
per_instance[inst].add(dp_rank)
assert per_instance[0] == set(range(8))
assert per_instance[1] == set(range(8))
def test_consecutive_requests_alternate_instances(route):
# Interleaved order spreads consecutive requests across instances rather
# than filling one instance's ranks before moving on.
insts = [inst for inst, _ in _route_n(route, _instances(3, 8), 3)]
assert insts == [0, 1, 2]