[Fix][MoRI] Align MoRI-IO message format with P2pNcclConnector and vllm-router (#39565)
Signed-off-by: simondanielsson <simon.danielsson99@hotmail.com> Co-authored-by: Matvei Pashkovskii <mpashkov@amd.com>
This commit is contained in:
co-authored by
Matvei Pashkovskii
parent
b8401a9bf4
commit
ac58e2a170
@@ -10,7 +10,6 @@ import uuid
|
||||
|
||||
import aiohttp
|
||||
import msgpack
|
||||
import regex as re
|
||||
import zmq
|
||||
from quart import Quart, Request, make_response, request
|
||||
|
||||
@@ -25,32 +24,10 @@ decode_instances: list[dict] = []
|
||||
request_nums = 0
|
||||
app = Quart(__name__)
|
||||
|
||||
IP_PORT_PATTERN = re.compile(r"//(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}):(\d+)")
|
||||
|
||||
|
||||
TRANSFER_TYPE = None
|
||||
|
||||
|
||||
def _append_whole_dict_unique(target_list, data_dict):
|
||||
new_filtered = {k: v for k, v in data_dict.items() if k != "index"}
|
||||
for existed in target_list:
|
||||
existed_filtered = {k: v for k, v in existed.items() if k != "index"}
|
||||
if existed_filtered == new_filtered:
|
||||
return False
|
||||
print("!!APPEND!!", data_dict)
|
||||
target_list.append(data_dict)
|
||||
transfer_mode = data_dict.get("transfer_mode", "unknown")
|
||||
global TRANSFER_TYPE
|
||||
|
||||
if TRANSFER_TYPE is None:
|
||||
TRANSFER_TYPE = transfer_mode
|
||||
logger.info("SET TRANSFER TYPE TO %s", TRANSFER_TYPE)
|
||||
elif transfer_mode != TRANSFER_TYPE:
|
||||
raise ValueError(f"mismatched transfer mode {TRANSFER_TYPE} vs {transfer_mode}")
|
||||
|
||||
return True
|
||||
|
||||
|
||||
_list_lock = threading.RLock()
|
||||
|
||||
|
||||
@@ -68,23 +45,81 @@ def _listen_for_register(hostname, port):
|
||||
if router_socket in socks:
|
||||
remote_addr, msg = router_socket.recv_multipart()
|
||||
data = msgpack.loads(msg)
|
||||
if data["type"] == "HELLO":
|
||||
if data.get("type") == "HELLO":
|
||||
pass
|
||||
elif (
|
||||
data["type"] == "register"
|
||||
and data["role"] == "P"
|
||||
and data["request_address"] not in prefill_instances
|
||||
):
|
||||
with _list_lock:
|
||||
_append_whole_dict_unique(prefill_instances, data)
|
||||
elif data.get("type") in ("P", "D"):
|
||||
role = data["type"]
|
||||
required_keys = {
|
||||
"http_address",
|
||||
"zmq_address",
|
||||
"dp_size",
|
||||
"tp_size",
|
||||
"transfer_mode",
|
||||
}
|
||||
missing = required_keys - data.keys()
|
||||
if missing:
|
||||
logger.error(
|
||||
"Registration message missing required keys %s; skipping",
|
||||
missing,
|
||||
)
|
||||
continue
|
||||
# Derive request_address from http_address
|
||||
# api path suffix is appended at request time
|
||||
instance = {
|
||||
"role": role,
|
||||
"request_address": f"http://{data['http_address']}/v1",
|
||||
"http_address": data["http_address"],
|
||||
"zmq_address": data["zmq_address"],
|
||||
"dp_size": data["dp_size"],
|
||||
"tp_size": data["tp_size"],
|
||||
"transfer_mode": data["transfer_mode"],
|
||||
}
|
||||
# zmq_address format: "host:IP,handshake:PORT,notify:PORT"
|
||||
# Stored verbatim; embedded into the request_id by handle_request.
|
||||
|
||||
elif (
|
||||
data["type"] == "register"
|
||||
and data["role"] == "D"
|
||||
and data["request_address"] not in decode_instances
|
||||
):
|
||||
global TRANSFER_TYPE
|
||||
transfer_mode = instance["transfer_mode"]
|
||||
target_list = prefill_instances if role == "P" else decode_instances
|
||||
with _list_lock:
|
||||
_append_whole_dict_unique(decode_instances, data)
|
||||
if TRANSFER_TYPE is None:
|
||||
TRANSFER_TYPE = transfer_mode
|
||||
logger.info("SET TRANSFER TYPE TO %s", TRANSFER_TYPE)
|
||||
elif transfer_mode != TRANSFER_TYPE:
|
||||
logger.error(
|
||||
"Mismatched transfer mode: expected %s, got %s;"
|
||||
" skipping registration of %s",
|
||||
TRANSFER_TYPE,
|
||||
transfer_mode,
|
||||
data["http_address"],
|
||||
)
|
||||
continue
|
||||
existing_idx = next(
|
||||
(
|
||||
idx
|
||||
for idx, i in enumerate(target_list)
|
||||
if i.get("http_address") == data["http_address"]
|
||||
),
|
||||
None,
|
||||
)
|
||||
if existing_idx is not None:
|
||||
target_list[existing_idx] = instance
|
||||
logger.info(
|
||||
"Updated existing %s instance: %s",
|
||||
"Prefill" if role == "P" else "Decode",
|
||||
instance,
|
||||
)
|
||||
else:
|
||||
target_list.append(instance)
|
||||
logger.info(
|
||||
"Registered %s instance: %s",
|
||||
"Prefill" if role == "P" else "Decode",
|
||||
instance,
|
||||
)
|
||||
else:
|
||||
logger.warning(
|
||||
"Received message with unrecognized type %r; ignoring",
|
||||
data.get("type"),
|
||||
)
|
||||
|
||||
|
||||
def start_service_discovery(hostname, port):
|
||||
@@ -101,7 +136,7 @@ def start_service_discovery(hostname, port):
|
||||
|
||||
|
||||
async def send_request_to_prefill(
|
||||
endpoint, req_data, request_id, d_endpoint, dip, dport, selected_prefill_dp_rank
|
||||
endpoint, req_data, request_id, selected_prefill_dp_rank
|
||||
):
|
||||
req_data_copy = req_data
|
||||
|
||||
@@ -109,12 +144,8 @@ async def send_request_to_prefill(
|
||||
{
|
||||
"do_remote_decode": True,
|
||||
"do_remote_prefill": False,
|
||||
"remote_handshake_port": d_endpoint["handshake_port"],
|
||||
"remote_notify_port": d_endpoint["notify_port"],
|
||||
"remote_engine_id": None,
|
||||
"remote_block_ids": None,
|
||||
"remote_host": dip,
|
||||
"remote_port": dport,
|
||||
}
|
||||
)
|
||||
req_data_copy["stream"] = False
|
||||
@@ -197,14 +228,7 @@ async def handle_request(api: str, request: Request):
|
||||
global request_nums
|
||||
request_nums += 1
|
||||
|
||||
def extract_ip_port_fast(url):
|
||||
match = IP_PORT_PATTERN.search(url)
|
||||
if not match:
|
||||
raise ValueError(f"Invalid URL format: {url}")
|
||||
return match.groups()
|
||||
|
||||
req_data = await request.get_json()
|
||||
request_id = str(uuid.uuid4())
|
||||
|
||||
prefill_instance_endpoint = None
|
||||
decode_instance_endpoint = None
|
||||
@@ -230,7 +254,14 @@ async def handle_request(api: str, request: Request):
|
||||
prefill_instance_endpoint["dp_size"],
|
||||
)
|
||||
|
||||
dip, dport = extract_ip_port_fast(decode_instance_endpoint["request_address"])
|
||||
# Embed both zmq_addresses in the request_id so the connector can parse
|
||||
# the peer's host/ports from it, similar to P2P-NCCL
|
||||
uid = str(uuid.uuid4()).replace("-", "")
|
||||
request_id = (
|
||||
f"___prefill_addr_{prefill_instance_endpoint['zmq_address']}"
|
||||
f"___decode_addr_{decode_instance_endpoint['zmq_address']}"
|
||||
f"_{uid}"
|
||||
)
|
||||
|
||||
transfer_id = f"{MoRIIOConstants.TRANSFER_PREFIX}-{str(uuid.uuid4())}"
|
||||
|
||||
@@ -251,35 +282,30 @@ async def handle_request(api: str, request: Request):
|
||||
prefill_request_url,
|
||||
req_data_to_prefill,
|
||||
request_id,
|
||||
decode_instance_endpoint,
|
||||
dip,
|
||||
dport,
|
||||
selected_prefill_dp_rank,
|
||||
)
|
||||
)
|
||||
ip, port = extract_ip_port_fast(prefill_request_url)
|
||||
|
||||
req_data["max_tokens"] -= 1
|
||||
|
||||
req_data["kv_transfer_params"] = {
|
||||
"do_remote_decode": False,
|
||||
"do_remote_prefill": True,
|
||||
"remote_handshake_port": prefill_instance_endpoint["handshake_port"],
|
||||
"remote_notify_port": prefill_instance_endpoint["notify_port"],
|
||||
"remote_engine_id": None,
|
||||
"remote_block_ids": None,
|
||||
"remote_host": ip,
|
||||
"remote_port": port,
|
||||
"transfer_id": transfer_id,
|
||||
}
|
||||
if TRANSFER_TYPE == "READ":
|
||||
# In read mode, prefill and decode are executed serially.
|
||||
prefill_response = await send_prefill_task
|
||||
req_data["kv_transfer_params"]["remote_engine_id"] = prefill_response[
|
||||
"kv_transfer_params"
|
||||
]["remote_engine_id"]
|
||||
req_data["kv_transfer_params"]["remote_block_ids"] = prefill_response[
|
||||
"kv_transfer_params"
|
||||
]["remote_block_ids"]
|
||||
prefill_kv = prefill_response["kv_transfer_params"]
|
||||
req_data["kv_transfer_params"]["remote_engine_id"] = prefill_kv[
|
||||
"remote_engine_id"
|
||||
]
|
||||
req_data["kv_transfer_params"]["remote_block_ids"] = prefill_kv[
|
||||
"remote_block_ids"
|
||||
]
|
||||
req_data["kv_transfer_params"]["transfer_id"] = prefill_kv["transfer_id"]
|
||||
|
||||
req_data["kv_transfer_params"]["remote_dp_size"] = prefill_instance_endpoint[
|
||||
"dp_size"
|
||||
@@ -290,7 +316,6 @@ async def handle_request(api: str, request: Request):
|
||||
|
||||
if selected_prefill_dp_rank is not None:
|
||||
req_data["kv_transfer_params"]["remote_dp_rank"] = selected_prefill_dp_rank
|
||||
req_data["kv_transfer_params"]["transfer_id"] = transfer_id
|
||||
|
||||
decode_request_url = decode_instance_endpoint["request_address"] + api
|
||||
decode_request_task = asyncio.create_task(
|
||||
|
||||
Reference in New Issue
Block a user