[Bugfix] MoRIIO toy P/D proxy: fix DP-rank index aliasing + harden for high-concurrency bursts (#46115)
Signed-off-by: Edwin Lim <edwin.lim@mangoboost.io> Co-authored-by: Claude <noreply@anthropic.com> Co-authored-by: QinPR <1905873179@qq.com> Co-authored-by: Peiran Qin <66068739+QinPR@users.noreply.github.com>
This commit is contained in:
co-authored by
Claude
QinPR
Peiran Qin
parent
5784507da4
commit
c4cd2bd544
@@ -210,8 +210,31 @@ async def stream_decode_response(session, response, request_id):
|
||||
await session.close()
|
||||
|
||||
|
||||
def example_round_robin_dp_loader(request_number, dp_size):
|
||||
return request_nums % dp_size
|
||||
def flat_interleaved_dp_route(request_number, instances):
|
||||
"""Flat round-robin over the full (instance, dp_rank) slot space.
|
||||
|
||||
ONE counter over (n_instances * dp_size) slots, so instance-selection and
|
||||
DP-rank-selection are derived from the SAME index and can never alias. The
|
||||
previous scheme computed instance = req % n and rank = req % dp from the
|
||||
same counter with n | dp, which locked each instance to a stride-n subset
|
||||
of its ranks (e.g. 2 prefill instances -> 4 of 8 ranks each -> half the
|
||||
GPUs never receive a request, so the deployment falsely appears not to
|
||||
scale).
|
||||
|
||||
Interleaved order — inst0_r0, inst1_r0, inst0_r1, inst1_r1, ... — so
|
||||
consecutive requests alternate instances AND every rank gets walked.
|
||||
|
||||
Assumes homogeneous dp_size across a role's instances (true for the
|
||||
DP<->DP and DP<->TP deployments this proxy targets). Returns
|
||||
(instance_index, dp_rank); dp_rank is None when dp_size == 1 (e.g. a TP
|
||||
decode), which avoids forwarding an out-of-range data-parallel rank.
|
||||
"""
|
||||
n = len(instances)
|
||||
dp = instances[0]["dp_size"]
|
||||
slot = (request_number - 1) % (n * dp)
|
||||
inst_idx = slot % n
|
||||
dp_rank = (slot // n) if dp > 1 else None
|
||||
return inst_idx, dp_rank
|
||||
|
||||
|
||||
@app.route("/health", methods=["GET"])
|
||||
@@ -252,18 +275,21 @@ async def handle_request(api: str, request: Request):
|
||||
503,
|
||||
)
|
||||
)
|
||||
pid = request_nums % len(prefill_instances)
|
||||
did = request_nums % len(decode_instances)
|
||||
# Flat interleaved round-robin (see flat_interleaved_dp_route): ONE
|
||||
# counter over the full (instance, dp_rank) slot space per role, so
|
||||
# instance-selection and DP-rank-selection derive from the same index
|
||||
# and can never alias. The old scheme keyed both on request_nums with
|
||||
# n_instances | dp_size, stranding half the ranks (e.g. in 2P_DP8EP).
|
||||
pid, selected_prefill_dp_rank = flat_interleaved_dp_route(
|
||||
request_nums, prefill_instances
|
||||
)
|
||||
# Decode instance selection uses the same interleaved walk; in READ
|
||||
# mode the decode reads KV from selected_prefill_dp_rank, so the
|
||||
# decode's own dp_rank is not forwarded here.
|
||||
did, _ = flat_interleaved_dp_route(request_nums, decode_instances)
|
||||
prefill_instance_endpoint = prefill_instances[pid]
|
||||
decode_instance_endpoint = decode_instances[did]
|
||||
|
||||
selected_prefill_dp_rank = None
|
||||
if prefill_instance_endpoint["dp_size"] > 1:
|
||||
selected_prefill_dp_rank = example_round_robin_dp_loader(
|
||||
request_nums // len(prefill_instance_endpoint),
|
||||
prefill_instance_endpoint["dp_size"],
|
||||
)
|
||||
|
||||
# Embed both zmq_addresses in the request_id so the connector can parse
|
||||
# the peer's host/ports from it, similar to P2P-NCCL
|
||||
uid = str(uuid.uuid4()).replace("-", "")
|
||||
@@ -427,9 +453,33 @@ if __name__ == "__main__":
|
||||
args = parser.parse_args()
|
||||
|
||||
t = start_service_discovery("0.0.0.0", 36367)
|
||||
app.debug = True
|
||||
# High-concurrency hardening. Quart's app.run() uses a shallow listen
|
||||
# backlog (100) and, with app.debug=True, adds per-request overhead that
|
||||
# starves the single accept loop. Under a burst of ~512 simultaneous client
|
||||
# connections the backlog overflows and the kernel RSTs the excess, so
|
||||
# clients see "ClientOSError: [Errno 104] Connection reset by peer" before
|
||||
# any response (~16% request loss at c=512). Serve via hypercorn with debug
|
||||
# OFF and a deep backlog so the burst QUEUES (higher TTFT) instead of being
|
||||
# reset -> 100% request success.
|
||||
app.debug = False
|
||||
app.config["BODY_TIMEOUT"] = 360000
|
||||
app.config["RESPONSE_TIMEOUT"] = 360000
|
||||
|
||||
app.run(host="0.0.0.0", port=args.port)
|
||||
import asyncio
|
||||
import os
|
||||
|
||||
from hypercorn.asyncio import serve as _hypercorn_serve
|
||||
from hypercorn.config import Config as _HypercornConfig
|
||||
|
||||
_hcfg = _HypercornConfig()
|
||||
_hcfg.bind = [f"0.0.0.0:{args.port}"]
|
||||
# Deep listen backlog so a wide connection burst queues, not RSTs. NOTE:
|
||||
# effective backlog is capped by the host's net.core.somaxconn (proxy runs
|
||||
# --network host); kernel 6.x defaults to 4096. Override via
|
||||
# PROXY_LISTEN_BACKLOG.
|
||||
_hcfg.backlog = int(os.environ.get("PROXY_LISTEN_BACKLOG", "4096"))
|
||||
# Long-lived SSE streams (8k1k decode ~5 min): never reap on keepalive.
|
||||
_hcfg.keep_alive_timeout = 360000.0
|
||||
|
||||
asyncio.run(_hypercorn_serve(app, _hcfg))
|
||||
t.join()
|
||||
|
||||
@@ -0,0 +1,205 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Hardware-fair request routing in the MoRIIO toy P/D proxy.
|
||||
|
||||
Exercises the REAL ``flat_interleaved_dp_route`` from the toy proxy (loaded from
|
||||
its ``examples/`` path with heavy deps stubbed) — no routing logic is copied
|
||||
here. This is the request-distribution half of end-to-end fairness: the proxy
|
||||
must hand every prefill/decode (instance, dp_rank) slot an equal share so no GPU
|
||||
is starved. The connector-side read routing is covered in
|
||||
``test_moriio_routing_fairness.py``.
|
||||
|
||||
Regression target: the previous scheme derived ``instance = req % n_instances``
|
||||
and ``dp_rank = req % dp_size`` from the same counter, so when
|
||||
``n_instances | dp_size`` each instance was locked to a stride-``n`` subset of
|
||||
its ranks — e.g. 2 prefill instances x DP8 stranded 4 of every node's 8 GPUs.
|
||||
``flat_interleaved_dp_route`` walks ONE counter over the full
|
||||
``(instance, dp_rank)`` slot space, so the two selections can never alias.
|
||||
|
||||
Role shapes below are exactly those in the RFC (#46107) deployments:
|
||||
(1, 1) 1P/1D TP8 (2, 1) 2P/2D TP8 (4, 1) 4D TP8
|
||||
(1, 8) 1D DP8EP (2, 8) 2P DP8EP (3, 8) 3D DP8EP
|
||||
"""
|
||||
|
||||
import contextlib
|
||||
import importlib.util
|
||||
import sys
|
||||
import types
|
||||
from collections import Counter
|
||||
from pathlib import Path
|
||||
from typing import cast
|
||||
|
||||
import pytest
|
||||
|
||||
_MISSING = object()
|
||||
|
||||
PROXY_REL = "examples/disaggregated/disaggregated_serving/moriio_toy_proxy_server.py"
|
||||
|
||||
|
||||
def _module(name, **attrs):
|
||||
module = types.ModuleType(name)
|
||||
for attr, value in attrs.items():
|
||||
setattr(module, attr, value)
|
||||
return module
|
||||
|
||||
|
||||
def _package(name):
|
||||
module = _module(name)
|
||||
module.__path__ = []
|
||||
return module
|
||||
|
||||
|
||||
class _QuartStub:
|
||||
def __init__(self, *a, **k):
|
||||
pass
|
||||
|
||||
def route(self, *a, **k):
|
||||
return lambda fn: fn
|
||||
|
||||
def post(self, *a, **k):
|
||||
return self.route()
|
||||
|
||||
|
||||
async def _make_response_stub(value):
|
||||
return value
|
||||
|
||||
|
||||
@contextlib.contextmanager
|
||||
def _proxy_import_stubs():
|
||||
"""Stub the proxy's external deps so the module imports for a pure unit test.
|
||||
|
||||
Only third-party/vllm imports are stubbed; the routing function under test
|
||||
is executed as-is from the real module.
|
||||
"""
|
||||
common = "vllm.distributed.kv_transfer.kv_connector.v1.moriio.moriio_common"
|
||||
|
||||
class _MoRIIOConstants:
|
||||
TRANSFER_PREFIX = "moriio-transfer"
|
||||
|
||||
stubs = {
|
||||
"aiohttp": _module("aiohttp"),
|
||||
"msgpack": _module("msgpack"),
|
||||
"zmq": _module("zmq"),
|
||||
"quart": _module(
|
||||
"quart",
|
||||
Quart=_QuartStub,
|
||||
Request=object,
|
||||
make_response=_make_response_stub,
|
||||
request=object(),
|
||||
),
|
||||
"vllm": _package("vllm"),
|
||||
"vllm.distributed": _package("vllm.distributed"),
|
||||
"vllm.distributed.kv_transfer": _package("vllm.distributed.kv_transfer"),
|
||||
"vllm.distributed.kv_transfer.kv_connector": _package(
|
||||
"vllm.distributed.kv_transfer.kv_connector"
|
||||
),
|
||||
"vllm.distributed.kv_transfer.kv_connector.v1": _package(
|
||||
"vllm.distributed.kv_transfer.kv_connector.v1"
|
||||
),
|
||||
"vllm.distributed.kv_transfer.kv_connector.v1.moriio": _package(
|
||||
"vllm.distributed.kv_transfer.kv_connector.v1.moriio"
|
||||
),
|
||||
common: _module(common, MoRIIOConstants=_MoRIIOConstants),
|
||||
}
|
||||
saved = {}
|
||||
for name, module in stubs.items():
|
||||
if name not in sys.modules:
|
||||
saved[name] = _MISSING
|
||||
sys.modules[name] = module
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
for name, previous in saved.items():
|
||||
if previous is _MISSING:
|
||||
sys.modules.pop(name, None)
|
||||
else:
|
||||
sys.modules[name] = cast(types.ModuleType, previous)
|
||||
|
||||
|
||||
def _load_proxy_module():
|
||||
path = Path(__file__).parents[4] / PROXY_REL
|
||||
spec = importlib.util.spec_from_file_location("moriio_proxy_under_test", path)
|
||||
assert spec is not None and spec.loader is not None
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
with _proxy_import_stubs():
|
||||
spec.loader.exec_module(module)
|
||||
return module
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def route():
|
||||
return _load_proxy_module().flat_interleaved_dp_route
|
||||
|
||||
|
||||
def _instances(n: int, dp_size: int):
|
||||
# Only dp_size is read by the router; tp_size carried for realism.
|
||||
return [{"dp_size": dp_size, "tp_size": 8 // dp_size} for _ in range(n)]
|
||||
|
||||
|
||||
# (n_instances, dp_size) for every distinct P/D role shape in the RFC configs.
|
||||
ROLE_SHAPES = [(1, 1), (2, 1), (4, 1), (1, 8), (2, 8), (3, 8)]
|
||||
SHAPE_IDS = [f"n{n}_dp{dp}" for n, dp in ROLE_SHAPES]
|
||||
|
||||
|
||||
def _route_n(route, instances, count):
|
||||
# Proxy uses 1-indexed request numbers (slot = (request_number - 1) % ...).
|
||||
return [route(rn, instances) for rn in range(1, count + 1)]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("n", "dp"), ROLE_SHAPES, ids=SHAPE_IDS)
|
||||
def test_full_slot_space_is_covered_uniformly(route, n, dp):
|
||||
instances = _instances(n, dp)
|
||||
period = n * dp
|
||||
# Three full cycles -> every (instance, dp_rank) slot must be hit the same
|
||||
# number of times (exactly uniform, no starved slot, no aliasing).
|
||||
hits = Counter(_route_n(route, instances, period * 3))
|
||||
|
||||
expected: set[tuple[int, int | None]]
|
||||
if dp == 1:
|
||||
expected = {(inst, None) for inst in range(n)}
|
||||
else:
|
||||
expected = {(inst, r) for inst in range(n) for r in range(dp)}
|
||||
assert set(hits) == expected, f"missing slots: {expected - set(hits)}"
|
||||
assert max(hits.values()) == min(hits.values())
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("n", "dp"), ROLE_SHAPES, ids=SHAPE_IDS)
|
||||
def test_instance_and_dp_rank_marginals_are_balanced(route, n, dp):
|
||||
instances = _instances(n, dp)
|
||||
routed = _route_n(route, instances, n * dp * 5)
|
||||
|
||||
inst_counts = Counter(inst for inst, _ in routed)
|
||||
assert set(inst_counts) == set(range(n))
|
||||
assert max(inst_counts.values()) == min(inst_counts.values())
|
||||
|
||||
dp_counts = Counter(r for _, r in routed)
|
||||
if dp == 1:
|
||||
assert set(dp_counts) == {None}
|
||||
else:
|
||||
assert set(dp_counts) == set(range(dp))
|
||||
assert max(dp_counts.values()) == min(dp_counts.values())
|
||||
|
||||
|
||||
def test_tp_instance_forwards_no_dp_rank(route):
|
||||
# dp_size == 1 (a TP instance) must yield dp_rank None so the proxy never
|
||||
# forwards an out-of-range data-parallel rank.
|
||||
assert all(r is None for _, r in _route_n(route, _instances(2, 1), 8))
|
||||
|
||||
|
||||
def test_two_instance_dp8_gives_every_node_all_ranks(route):
|
||||
# Direct regression for the stranded-GPU bug: with 2 prefill instances x DP8
|
||||
# each instance must receive ALL 8 dp ranks (16 distinct slots), not a
|
||||
# stride-2 subset of 4.
|
||||
routed = _route_n(route, _instances(2, 8), 2 * 8)
|
||||
per_instance: dict[int, set] = {0: set(), 1: set()}
|
||||
for inst, dp_rank in routed:
|
||||
per_instance[inst].add(dp_rank)
|
||||
assert per_instance[0] == set(range(8))
|
||||
assert per_instance[1] == set(range(8))
|
||||
|
||||
|
||||
def test_consecutive_requests_alternate_instances(route):
|
||||
# Interleaved order spreads consecutive requests across instances rather
|
||||
# than filling one instance's ranks before moving on.
|
||||
insts = [inst for inst, _ in _route_n(route, _instances(3, 8), 3)]
|
||||
assert insts == [0, 1, 2]
|
||||
Reference in New Issue
Block a user