llmd+vllm+mori-ep(inter node wide-ep)+mori-io(write) for 2p2d with dp=ep=16 tp=1 (#45043)

Signed-off-by: Shiksha Patel <shikpate@amd.com>
Co-authored-by: Douglas Lehr <91553416+dllehr-amd@users.noreply.github.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
shikamd123
2026-08-04 00:13:15 +00:00
committed by GitHub
co-authored by Douglas Lehr mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent c810937573
commit f42761204f
8 changed files with 710 additions and 120 deletions
@@ -1,5 +1,20 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
#
# Minimal single-node reference proxy for MoRI-IO prefill/decode disaggregation.
# It demonstrates the DP-rank pinning contract the connector expects: pick one
# prefill DP rank per request (flat_interleaved_dp_route) and pin BOTH legs to it
# via the ``X-data-parallel-rank`` header plus ``kv_transfer_params``
# (``remote_dp_rank`` / ``remote_dp_size`` / ``remote_tp_size``).
#
# Scope: single-node only. It intentionally does NOT supply the cross-pod
# Wide-EP (2P2D) peer info -- ``remote_hosts`` / ``multi_pod_hosts``,
# ``data_parallel_size_local``, ``is_request_leader``. In real multi-pod
# deployments that peer info is produced by a production routing sidecar, e.g.:
# * llm-d-router (https://github.com/llm-d/llm-d-router)
# * vLLM's own vllm-router (examples in the vllm-project org)
# * or any NIXL-shaped router that sets the same kv_transfer_params contract.
# Use this file as a protocol reference, not a production router.
import argparse
import asyncio
import copy
@@ -181,7 +196,7 @@ async def send_request_to_prefill(
raise RuntimeError(error_message)
async def start_decode_request(endpoint, req_data, request_id):
async def start_decode_request(endpoint, req_data, request_id, selected_dp_rank=None):
session = aiohttp.ClientSession(
timeout=aiohttp.ClientTimeout(total=6 * 6000 * 6000)
)
@@ -189,6 +204,10 @@ async def start_decode_request(endpoint, req_data, request_id):
"Authorization": f"Bearer {os.environ.get('OPENAI_API_KEY')}",
"X-Request-Id": request_id,
}
# Pin the decode leg to the same DP rank as its prefill leg so both land on
# the same rank (vLLM reads this header directly; no engine-side routing).
if selected_dp_rank is not None:
headers["X-data-parallel-rank"] = str(selected_dp_rank)
response = await session.post(url=endpoint, json=req_data, headers=headers)
return session, response
@@ -361,7 +380,9 @@ async def handle_request(api: str, request: Request):
decode_request_url = decode_instance_endpoint["request_address"] + api
decode_request_task = asyncio.create_task(
start_decode_request(decode_request_url, req_data, request_id)
start_decode_request(
decode_request_url, req_data, request_id, selected_prefill_dp_rank
)
)
session, decode_response = await decode_request_task