From c4cd2bd5440bca29039e5b6f33b695a95721bd3e Mon Sep 17 00:00:00 2001 From: limeward <32970461+edwinlim0919@users.noreply.github.com> Date: Fri, 17 Jul 2026 12:35:04 -0700 Subject: [PATCH] [Bugfix] MoRIIO toy P/D proxy: fix DP-rank index aliasing + harden for high-concurrency bursts (#46115) Signed-off-by: Edwin Lim Co-authored-by: Claude Co-authored-by: QinPR <1905873179@qq.com> Co-authored-by: Peiran Qin <66068739+QinPR@users.noreply.github.com> --- .../moriio_toy_proxy_server.py | 76 +++++-- .../unit/test_moriio_proxy_routing.py | 205 ++++++++++++++++++ 2 files changed, 268 insertions(+), 13 deletions(-) create mode 100644 tests/v1/kv_connector/unit/test_moriio_proxy_routing.py diff --git a/examples/disaggregated/disaggregated_serving/moriio_toy_proxy_server.py b/examples/disaggregated/disaggregated_serving/moriio_toy_proxy_server.py index 6f7d210ff3..d2d5be7607 100644 --- a/examples/disaggregated/disaggregated_serving/moriio_toy_proxy_server.py +++ b/examples/disaggregated/disaggregated_serving/moriio_toy_proxy_server.py @@ -210,8 +210,31 @@ async def stream_decode_response(session, response, request_id): await session.close() -def example_round_robin_dp_loader(request_number, dp_size): - return request_nums % dp_size +def flat_interleaved_dp_route(request_number, instances): + """Flat round-robin over the full (instance, dp_rank) slot space. + + ONE counter over (n_instances * dp_size) slots, so instance-selection and + DP-rank-selection are derived from the SAME index and can never alias. The + previous scheme computed instance = req % n and rank = req % dp from the + same counter with n | dp, which locked each instance to a stride-n subset + of its ranks (e.g. 2 prefill instances -> 4 of 8 ranks each -> half the + GPUs never receive a request, so the deployment falsely appears not to + scale). + + Interleaved order — inst0_r0, inst1_r0, inst0_r1, inst1_r1, ... — so + consecutive requests alternate instances AND every rank gets walked. + + Assumes homogeneous dp_size across a role's instances (true for the + DP<->DP and DP<->TP deployments this proxy targets). Returns + (instance_index, dp_rank); dp_rank is None when dp_size == 1 (e.g. a TP + decode), which avoids forwarding an out-of-range data-parallel rank. + """ + n = len(instances) + dp = instances[0]["dp_size"] + slot = (request_number - 1) % (n * dp) + inst_idx = slot % n + dp_rank = (slot // n) if dp > 1 else None + return inst_idx, dp_rank @app.route("/health", methods=["GET"]) @@ -252,18 +275,21 @@ async def handle_request(api: str, request: Request): 503, ) ) - pid = request_nums % len(prefill_instances) - did = request_nums % len(decode_instances) + # Flat interleaved round-robin (see flat_interleaved_dp_route): ONE + # counter over the full (instance, dp_rank) slot space per role, so + # instance-selection and DP-rank-selection derive from the same index + # and can never alias. The old scheme keyed both on request_nums with + # n_instances | dp_size, stranding half the ranks (e.g. in 2P_DP8EP). + pid, selected_prefill_dp_rank = flat_interleaved_dp_route( + request_nums, prefill_instances + ) + # Decode instance selection uses the same interleaved walk; in READ + # mode the decode reads KV from selected_prefill_dp_rank, so the + # decode's own dp_rank is not forwarded here. + did, _ = flat_interleaved_dp_route(request_nums, decode_instances) prefill_instance_endpoint = prefill_instances[pid] decode_instance_endpoint = decode_instances[did] - selected_prefill_dp_rank = None - if prefill_instance_endpoint["dp_size"] > 1: - selected_prefill_dp_rank = example_round_robin_dp_loader( - request_nums // len(prefill_instance_endpoint), - prefill_instance_endpoint["dp_size"], - ) - # Embed both zmq_addresses in the request_id so the connector can parse # the peer's host/ports from it, similar to P2P-NCCL uid = str(uuid.uuid4()).replace("-", "") @@ -427,9 +453,33 @@ if __name__ == "__main__": args = parser.parse_args() t = start_service_discovery("0.0.0.0", 36367) - app.debug = True + # High-concurrency hardening. Quart's app.run() uses a shallow listen + # backlog (100) and, with app.debug=True, adds per-request overhead that + # starves the single accept loop. Under a burst of ~512 simultaneous client + # connections the backlog overflows and the kernel RSTs the excess, so + # clients see "ClientOSError: [Errno 104] Connection reset by peer" before + # any response (~16% request loss at c=512). Serve via hypercorn with debug + # OFF and a deep backlog so the burst QUEUES (higher TTFT) instead of being + # reset -> 100% request success. + app.debug = False app.config["BODY_TIMEOUT"] = 360000 app.config["RESPONSE_TIMEOUT"] = 360000 - app.run(host="0.0.0.0", port=args.port) + import asyncio + import os + + from hypercorn.asyncio import serve as _hypercorn_serve + from hypercorn.config import Config as _HypercornConfig + + _hcfg = _HypercornConfig() + _hcfg.bind = [f"0.0.0.0:{args.port}"] + # Deep listen backlog so a wide connection burst queues, not RSTs. NOTE: + # effective backlog is capped by the host's net.core.somaxconn (proxy runs + # --network host); kernel 6.x defaults to 4096. Override via + # PROXY_LISTEN_BACKLOG. + _hcfg.backlog = int(os.environ.get("PROXY_LISTEN_BACKLOG", "4096")) + # Long-lived SSE streams (8k1k decode ~5 min): never reap on keepalive. + _hcfg.keep_alive_timeout = 360000.0 + + asyncio.run(_hypercorn_serve(app, _hcfg)) t.join() diff --git a/tests/v1/kv_connector/unit/test_moriio_proxy_routing.py b/tests/v1/kv_connector/unit/test_moriio_proxy_routing.py new file mode 100644 index 0000000000..71ee0ef5d7 --- /dev/null +++ b/tests/v1/kv_connector/unit/test_moriio_proxy_routing.py @@ -0,0 +1,205 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Hardware-fair request routing in the MoRIIO toy P/D proxy. + +Exercises the REAL ``flat_interleaved_dp_route`` from the toy proxy (loaded from +its ``examples/`` path with heavy deps stubbed) — no routing logic is copied +here. This is the request-distribution half of end-to-end fairness: the proxy +must hand every prefill/decode (instance, dp_rank) slot an equal share so no GPU +is starved. The connector-side read routing is covered in +``test_moriio_routing_fairness.py``. + +Regression target: the previous scheme derived ``instance = req % n_instances`` +and ``dp_rank = req % dp_size`` from the same counter, so when +``n_instances | dp_size`` each instance was locked to a stride-``n`` subset of +its ranks — e.g. 2 prefill instances x DP8 stranded 4 of every node's 8 GPUs. +``flat_interleaved_dp_route`` walks ONE counter over the full +``(instance, dp_rank)`` slot space, so the two selections can never alias. + +Role shapes below are exactly those in the RFC (#46107) deployments: + (1, 1) 1P/1D TP8 (2, 1) 2P/2D TP8 (4, 1) 4D TP8 + (1, 8) 1D DP8EP (2, 8) 2P DP8EP (3, 8) 3D DP8EP +""" + +import contextlib +import importlib.util +import sys +import types +from collections import Counter +from pathlib import Path +from typing import cast + +import pytest + +_MISSING = object() + +PROXY_REL = "examples/disaggregated/disaggregated_serving/moriio_toy_proxy_server.py" + + +def _module(name, **attrs): + module = types.ModuleType(name) + for attr, value in attrs.items(): + setattr(module, attr, value) + return module + + +def _package(name): + module = _module(name) + module.__path__ = [] + return module + + +class _QuartStub: + def __init__(self, *a, **k): + pass + + def route(self, *a, **k): + return lambda fn: fn + + def post(self, *a, **k): + return self.route() + + +async def _make_response_stub(value): + return value + + +@contextlib.contextmanager +def _proxy_import_stubs(): + """Stub the proxy's external deps so the module imports for a pure unit test. + + Only third-party/vllm imports are stubbed; the routing function under test + is executed as-is from the real module. + """ + common = "vllm.distributed.kv_transfer.kv_connector.v1.moriio.moriio_common" + + class _MoRIIOConstants: + TRANSFER_PREFIX = "moriio-transfer" + + stubs = { + "aiohttp": _module("aiohttp"), + "msgpack": _module("msgpack"), + "zmq": _module("zmq"), + "quart": _module( + "quart", + Quart=_QuartStub, + Request=object, + make_response=_make_response_stub, + request=object(), + ), + "vllm": _package("vllm"), + "vllm.distributed": _package("vllm.distributed"), + "vllm.distributed.kv_transfer": _package("vllm.distributed.kv_transfer"), + "vllm.distributed.kv_transfer.kv_connector": _package( + "vllm.distributed.kv_transfer.kv_connector" + ), + "vllm.distributed.kv_transfer.kv_connector.v1": _package( + "vllm.distributed.kv_transfer.kv_connector.v1" + ), + "vllm.distributed.kv_transfer.kv_connector.v1.moriio": _package( + "vllm.distributed.kv_transfer.kv_connector.v1.moriio" + ), + common: _module(common, MoRIIOConstants=_MoRIIOConstants), + } + saved = {} + for name, module in stubs.items(): + if name not in sys.modules: + saved[name] = _MISSING + sys.modules[name] = module + try: + yield + finally: + for name, previous in saved.items(): + if previous is _MISSING: + sys.modules.pop(name, None) + else: + sys.modules[name] = cast(types.ModuleType, previous) + + +def _load_proxy_module(): + path = Path(__file__).parents[4] / PROXY_REL + spec = importlib.util.spec_from_file_location("moriio_proxy_under_test", path) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + with _proxy_import_stubs(): + spec.loader.exec_module(module) + return module + + +@pytest.fixture(scope="module") +def route(): + return _load_proxy_module().flat_interleaved_dp_route + + +def _instances(n: int, dp_size: int): + # Only dp_size is read by the router; tp_size carried for realism. + return [{"dp_size": dp_size, "tp_size": 8 // dp_size} for _ in range(n)] + + +# (n_instances, dp_size) for every distinct P/D role shape in the RFC configs. +ROLE_SHAPES = [(1, 1), (2, 1), (4, 1), (1, 8), (2, 8), (3, 8)] +SHAPE_IDS = [f"n{n}_dp{dp}" for n, dp in ROLE_SHAPES] + + +def _route_n(route, instances, count): + # Proxy uses 1-indexed request numbers (slot = (request_number - 1) % ...). + return [route(rn, instances) for rn in range(1, count + 1)] + + +@pytest.mark.parametrize(("n", "dp"), ROLE_SHAPES, ids=SHAPE_IDS) +def test_full_slot_space_is_covered_uniformly(route, n, dp): + instances = _instances(n, dp) + period = n * dp + # Three full cycles -> every (instance, dp_rank) slot must be hit the same + # number of times (exactly uniform, no starved slot, no aliasing). + hits = Counter(_route_n(route, instances, period * 3)) + + expected: set[tuple[int, int | None]] + if dp == 1: + expected = {(inst, None) for inst in range(n)} + else: + expected = {(inst, r) for inst in range(n) for r in range(dp)} + assert set(hits) == expected, f"missing slots: {expected - set(hits)}" + assert max(hits.values()) == min(hits.values()) + + +@pytest.mark.parametrize(("n", "dp"), ROLE_SHAPES, ids=SHAPE_IDS) +def test_instance_and_dp_rank_marginals_are_balanced(route, n, dp): + instances = _instances(n, dp) + routed = _route_n(route, instances, n * dp * 5) + + inst_counts = Counter(inst for inst, _ in routed) + assert set(inst_counts) == set(range(n)) + assert max(inst_counts.values()) == min(inst_counts.values()) + + dp_counts = Counter(r for _, r in routed) + if dp == 1: + assert set(dp_counts) == {None} + else: + assert set(dp_counts) == set(range(dp)) + assert max(dp_counts.values()) == min(dp_counts.values()) + + +def test_tp_instance_forwards_no_dp_rank(route): + # dp_size == 1 (a TP instance) must yield dp_rank None so the proxy never + # forwards an out-of-range data-parallel rank. + assert all(r is None for _, r in _route_n(route, _instances(2, 1), 8)) + + +def test_two_instance_dp8_gives_every_node_all_ranks(route): + # Direct regression for the stranded-GPU bug: with 2 prefill instances x DP8 + # each instance must receive ALL 8 dp ranks (16 distinct slots), not a + # stride-2 subset of 4. + routed = _route_n(route, _instances(2, 8), 2 * 8) + per_instance: dict[int, set] = {0: set(), 1: set()} + for inst, dp_rank in routed: + per_instance[inst].add(dp_rank) + assert per_instance[0] == set(range(8)) + assert per_instance[1] == set(range(8)) + + +def test_consecutive_requests_alternate_instances(route): + # Interleaved order spreads consecutive requests across instances rather + # than filling one instance's ranks before moving on. + insts = [inst for inst, _ in _route_n(route, _instances(3, 8), 3)] + assert insts == [0, 1, 2]