[ROCm][P/D] Fix MoRIIO WRITE mode for mixed KV layouts (#46290)

Signed-off-by: Tan Pin Siang <tanpinsiang@gmail.com>
Co-authored-by: vllmellm <vllm.ellm@embeddedllm.com>
Co-authored-by: Hongxia Yang <hongxia.yang@amd.com>
Co-authored-by: Jun Kang Chow <junkangchow@gmail.com>
Co-authored-by: Chun Fang <chun.fang@amd.com>
Co-authored-by: TianDi101 <ditian12@amd.com>
Co-authored-by: functionstackx <47992694+functionstackx@users.noreply.github.com>
This commit is contained in:
Tan Pin Siang
2026-06-23 12:12:51 +08:00
committed by GitHub
co-authored by vllmellm Hongxia Yang Jun Kang Chow Chun Fang TianDi101 functionstackx
parent a8481be7a9
commit 7e47fb72b5
6 changed files with 1104 additions and 195 deletions
@@ -78,8 +78,17 @@ class RemoteAllocInfo:
block_ids: list[int]
writes_done: int = 0
writes_expected: int | None = None
decode_dp_rank: int = 0
transfer_offset: tuple[list[int], list[int], list[int]] | None = None
completion_request_id: str | None = None
completion_remote_notify_port: int | None = None
completion_remote_ip: str | None = None
completion_notified: bool = False
transfer_statuses: list[Any] = field(default_factory=list)
transfer_offsets: dict[
tuple[tuple[int, ...], tuple[int, ...], torch.dtype],
tuple[list[int], list[int], list[int]],
] = field(default_factory=dict)
class ROLE(Enum):
@@ -434,12 +443,21 @@ class MoRIIOConnectorMetadata(KVConnectorMetadata):
):
transfer_id = kv_transfer_params["transfer_id"]
# Parse host/ports from the request_id. The router embeds both zmq_addresses
# in the request_id
peer_zmq = get_peer_zmq_from_request_id(request_id, is_producer=write_mode)
remote_host, remote_handshake_port, remote_notify_port = (
parse_moriio_zmq_address(peer_zmq)
)
remote_host = kv_transfer_params.get("remote_host")
remote_handshake_port = kv_transfer_params.get("remote_handshake_port")
remote_notify_port = kv_transfer_params.get("remote_notify_port")
if (
remote_host is None
or remote_handshake_port is None
or remote_notify_port is None
):
# Parse host/ports from the request_id. The router embeds both
# zmq_addresses in PD request IDs, but WRITE decode requests may carry
# a plain request ID and get the remote address via kv_transfer_params.
peer_zmq = get_peer_zmq_from_request_id(request_id, is_producer=write_mode)
remote_host, remote_handshake_port, remote_notify_port = (
parse_moriio_zmq_address(peer_zmq)
)
_req = ReqMeta(
transfer_id=transfer_id,
@@ -447,9 +465,9 @@ class MoRIIOConnectorMetadata(KVConnectorMetadata):
remote_block_ids=kv_transfer_params["remote_block_ids"],
remote_engine_id=kv_transfer_params["remote_engine_id"],
remote_host=remote_host,
remote_port=remote_handshake_port,
remote_handshake_port=remote_handshake_port,
remote_notify_port=remote_notify_port,
remote_port=int(remote_handshake_port),
remote_handshake_port=int(remote_handshake_port),
remote_notify_port=int(remote_notify_port),
tp_size=kv_transfer_params.get("tp_size", 1),
remote_dp_size=kv_transfer_params.get("remote_dp_size", 1),
)
@@ -234,7 +234,13 @@ class MoRIIOConnector(KVConnectorBase_V1):
return None
def wait_for_save(self):
pass
if self.mode != MoRIIOMode.WRITE or get_role() != ROLE.PRODUCER:
return
assert self.connector_worker is not None
assert isinstance(self._connector_metadata, MoRIIOConnectorMetadata), (
"Connector metadata not initialized yet"
)
self.connector_worker.wait_for_save(self._connector_metadata)
def shutdown(self):
if self.connector_worker is not None:
@@ -390,6 +396,49 @@ class MoRIIOConnectorScheduler:
serialized_data = msgpack.dumps(data)
self.paths[path].send(serialized_data)
def _send_transfer_release(self, transfer_id: TransferId, host: str, port: int):
path = make_zmq_path("tcp", host, port)
if path not in self.paths:
ctx = zmq.Context.instance()
sock = make_zmq_socket(
ctx=ctx, path=path, socket_type=zmq.DEALER, bind=False
)
self.paths[path] = sock
self.paths[path].send(
msgpack.dumps({"type": "release", "transfer_id": transfer_id})
)
def _release_write_prefill_blocks(self, request_id: ReqId, params: dict[str, Any]):
transfer_id = params.get("transfer_id")
if transfer_id is None:
logger.warning(
"Cannot release WRITE prefill blocks for request %s: "
"missing transfer_id",
request_id,
)
return
remote_dp_rank = params.get("remote_dp_rank", 0)
remote_host = params.get("remote_host")
remote_notify_port = params.get("remote_notify_port")
if remote_host is None or remote_notify_port is None:
try:
peer_zmq = get_peer_zmq_from_request_id(request_id, is_producer=False)
remote_host, _, remote_notify_port = parse_moriio_zmq_address(peer_zmq)
except ValueError:
logger.warning(
"Cannot release WRITE prefill blocks for request %s: "
"missing remote notify address",
request_id,
)
return
remote_notify_port = int(remote_notify_port)
for tp_index in range(self.tp_size):
target_port = remote_notify_port + get_port_offset(remote_dp_rank, tp_index)
self._send_transfer_release(transfer_id, remote_host, target_port)
def update_state_after_alloc(
self,
request: "Request",
@@ -443,11 +492,20 @@ class MoRIIOConnectorScheduler:
)
remote_dp_rank = request.kv_transfer_params.get("remote_dp_rank", 0)
peer_zmq = get_peer_zmq_from_request_id(
request.request_id, is_producer=False
remote_host = request.kv_transfer_params.get("remote_host")
remote_notify_port = request.kv_transfer_params.get(
"remote_notify_port"
)
remote_host, _, remote_notify_port = parse_moriio_zmq_address(peer_zmq)
if remote_host is None or remote_notify_port is None:
peer_zmq = get_peer_zmq_from_request_id(
request.request_id, is_producer=False
)
remote_host, _, remote_notify_port = parse_moriio_zmq_address(
peer_zmq
)
remote_notify_port = int(remote_notify_port)
block_ids = blocks.get_block_ids()[0]
for tp_index in range(self.tp_size):
target_port = remote_notify_port + get_port_offset(
@@ -457,7 +515,7 @@ class MoRIIOConnectorScheduler:
self.send_notify_block(
req_id=request.request_id,
transfer_id=request.kv_transfer_params["transfer_id"],
block_notify_list=blocks.get_block_ids()[0],
block_notify_list=block_ids,
host=remote_host,
port=target_port,
)
@@ -473,60 +531,31 @@ class MoRIIOConnectorScheduler:
meta = MoRIIOConnectorMetadata()
meta.transfer_id_to_request_id = self.transfer_id_to_request_id
if self.mode == MoRIIOMode.WRITE:
# when async_load_kv finished,
# new reqs will be added to scheduler_output.scheduled_new_reqs
if self.mode == MoRIIOMode.WRITE and get_role() == ROLE.PRODUCER:
# This is the logic for checking against chunked prefill.
# When the last chunk is identified,
# It places the request metadata into the saving queue.
if get_role() == ROLE.CONSUMER:
for new_req in scheduler_output.scheduled_new_reqs:
red_id = new_req.req_id
local_block_ids = list(new_req.block_ids)[0]
assert new_req.sampling_params is not None, (
f"sampling_params is None for req {new_req.req_id}"
)
assert hasattr(new_req.sampling_params, "extra_args"), (
f"sampling_params missing extra_args for req {new_req.req_id}"
)
kv_transfer_params = (
new_req.sampling_params.extra_args.get("kv_transfer_params", {})
if new_req.sampling_params.extra_args
else {}
)
meta.add_new_req(
red_id,
local_block_ids,
kv_transfer_params,
)
if get_role() == ROLE.PRODUCER:
# This is the logic for checking against chunked prefill.
# When the last chunk is identified,
# It places the request metadata into the saving queue.
for i, req_id in enumerate(scheduler_output.scheduled_cached_reqs.req_ids):
new_block_ids = scheduler_output.scheduled_cached_reqs.new_block_ids[i]
for i, req_id in enumerate(
scheduler_output.scheduled_cached_reqs.req_ids
):
new_block_ids = (
scheduler_output.scheduled_cached_reqs.new_block_ids[i]
)
if new_block_ids is not None:
block_ids = new_block_ids[0]
# TODO : hybrid attn, etc
req, existing_blocks = self._reqs_need_pending_save[req_id]
updated_blocks = list(existing_blocks) + (block_ids)
self._reqs_need_pending_save[req_id] = (req, updated_blocks)
if (
len(self._reqs_need_pending_save[req_id][1])
* self.block_size
>= req.num_prompt_tokens
):
meta.add_new_req(
request_id=req_id,
local_block_ids=self._reqs_need_pending_save[req_id][1],
kv_transfer_params=req.kv_transfer_params or {},
write_mode=True,
)
del self._reqs_need_pending_save[req_id]
if new_block_ids is not None:
block_ids = new_block_ids[0]
# TODO : hybrid attn, etc
req, existing_blocks = self._reqs_need_pending_save[req_id]
updated_blocks = list(existing_blocks) + (block_ids)
self._reqs_need_pending_save[req_id] = (req, updated_blocks)
if (
len(self._reqs_need_pending_save[req_id][1]) * self.block_size
>= req.num_prompt_tokens
):
meta.add_new_req(
request_id=req_id,
local_block_ids=self._reqs_need_pending_save[req_id][1],
kv_transfer_params=req.kv_transfer_params or {},
write_mode=True,
)
del self._reqs_need_pending_save[req_id]
# Loop through scheduled reqs and convert to ReqMeta.
for req_id, (req, block_ids) in self._reqs_need_recv.items():
@@ -601,9 +630,15 @@ class MoRIIOConnectorScheduler:
# update_state_after_alloc must not have been called (the request
# must have been aborted before it was scheduled).
# To avoid stranding the prefill blocks in the prefill instance,
# we must add empty block_ids to _reqs_need_recv so that our
# worker side will notify and free blocks in the prefill instance.
self._reqs_need_recv[request.request_id] = (request, [])
# READ mode adds empty block_ids to _reqs_need_recv so the worker
# side notifies the prefill instance. WRITE mode should notify the
# producer directly: there is no decode allocation for the producer
# to write into, and a plain request_id may not contain router-
# embedded MoRIIO ZMQ addresses.
if self.mode == MoRIIOMode.WRITE:
self._release_write_prefill_blocks(request.request_id, params)
else:
self._reqs_need_recv[request.request_id] = (request, [])
params["do_remote_prefill"] = False
return False, None
@@ -635,6 +670,10 @@ class MoRIIOConnectorScheduler:
do_remote_decode=False,
remote_block_ids=computed_block_ids,
remote_engine_id=self.engine_id,
remote_host=self.host_ip,
remote_handshake_port=self.handshake_port,
remote_notify_port=self.side_notify_port,
remote_dp_size=self.vllm_config.parallel_config.data_parallel_size,
tp_size=self.vllm_config.parallel_config.tensor_parallel_size,
transfer_id=params["transfer_id"],
)
@@ -1480,7 +1519,7 @@ class MoRIIOConnectorWorker:
metadata: MoRIIOConnectorMetadata,
layer_name: str,
kv_layer: torch.Tensor,
attn_metadata: "AttentionMetadata",
attn_metadata: "AttentionMetadata | None",
**kwargs,
):
if not self.is_producer:
@@ -1604,6 +1643,12 @@ class MoRIIOConnectorWorker:
self._reqs_to_send.update(metadata.reqs_to_send)
def wait_for_save(self, metadata: MoRIIOConnectorMetadata):
if self.mode == MoRIIOMode.WRITE and self.is_producer:
for layer_name, kv_layer in self.kv_caches.items():
self.save_kv_layer(metadata, layer_name, kv_layer, None)
self._writer.seal_pending_transfers()
def _read_blocks_for_req(self, req_id: str, meta: ReqMeta):
logger.debug(
"Remote agent %s available, calling _read_blocks for req %s",
@@ -2,6 +2,8 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import threading
import time
from collections import OrderedDict, defaultdict
from queue import Empty, Queue
from typing import TYPE_CHECKING, Any
from weakref import ref as weakref_ref
@@ -18,8 +20,6 @@ from vllm.utils.network_utils import (
if TYPE_CHECKING:
from mori.io import BackendType
from queue import Empty, Queue
from vllm.distributed.kv_transfer.kv_connector.v1.moriio.moriio_common import (
ROLE,
HandshakeError,
@@ -61,10 +61,25 @@ except ImportError:
"""Write task execution logic for MoRIIO connector."""
_MAX_TERMINAL_TRANSFER_IDS = 4096
WriteGeometryKey = tuple[tuple[int, ...], tuple[int, ...], torch.dtype]
def _get_write_geometry_key(kv_cache: torch.Tensor) -> WriteGeometryKey:
return (tuple(kv_cache.shape), tuple(kv_cache.stride()), kv_cache.dtype)
class MoRIIOWriter:
"""Handles write operations for KV cache transfers.
Implements distributed KV cache transfer using the MoRIIO library
for RDMA-based communication between prefill and decode instances."""
WRITE mode state machine:
D sends destination block allocation, P schedules one write per layer
after the layer CUDA event, P seals the scheduled write count after
forward, then P notifies D and releases P blocks after all scheduled
writes complete.
"""
def __init__(self, worker: "MoRIIOConnectorWorker"):
"""Initialize the writer.
@@ -76,7 +91,11 @@ class MoRIIOWriter:
self._write_task_q: Queue[WriteTask] = Queue()
self._write_worker_started = False
self._write_worker_lock = threading.Lock()
self._write_state_lock = threading.Lock()
self._deferred_tasks: list[WriteTask] = []
self._scheduled_writes: dict[TransferId, int] = defaultdict(int)
self._scheduled_layers: dict[TransferId, set[str]] = defaultdict(set)
self._sealed_writes: dict[TransferId, int] = {}
self._defer_timeout = worker.moriio_config.defer_timeout
@property
@@ -106,14 +125,55 @@ class MoRIIOWriter:
thread.start()
logger.info("Started MoRIIO write worker thread")
def schedule_write(self, task: WriteTask) -> None:
def schedule_write(self, task: WriteTask) -> bool:
"""Schedule a write task.
Args:
task: The write task to schedule
"""
self.ensure_worker_started()
if self._is_transfer_terminal(task.transfer_id):
return False
with self._write_state_lock:
if self._is_transfer_terminal(task.transfer_id):
return False
if task.layer_name in self._scheduled_layers[task.transfer_id]:
return False
self._scheduled_layers[task.transfer_id].add(task.layer_name)
self._scheduled_writes[task.transfer_id] += 1
self._write_task_q.put(task)
return True
def is_scheduled(self, transfer_id: TransferId, layer_name: str) -> bool:
with self._write_state_lock:
return layer_name in self._scheduled_layers.get(transfer_id, set())
def seal_pending_transfers(self) -> None:
"""Seal expected WRITE counts after the model forward has run.
`save_kv_layer` is only invoked for attention layers whose backend uses
the standard KV connector hook. Hybrid models can register more KV
cache tensors than the number of hooks that fire in a forward, so WRITE
completion must be based on the tasks actually queued for the transfer.
"""
pending: list[tuple[TransferId, RemoteAllocInfo]] = []
with self._write_state_lock:
for transfer_id, write_count in self._scheduled_writes.items():
if transfer_id in self._sealed_writes:
continue
self._sealed_writes[transfer_id] = write_count
request_info = (
self.worker.moriio_wrapper.done_remote_allocate_req_dict.get(
transfer_id
)
)
if request_info is not None:
request_info.writes_expected = write_count
pending.append((transfer_id, request_info))
for transfer_id, request_info in pending:
self._finalize_if_complete(transfer_id, request_info)
def _write_worker_loop(self) -> None:
"""Main loop for the write worker thread."""
@@ -128,6 +188,9 @@ class MoRIIOWriter:
except Empty:
continue
if self._is_transfer_terminal(task.transfer_id):
continue
# Check if remote blocks are ready
if not self._is_remote_ready(task):
# task.retry_count += 1
@@ -158,6 +221,8 @@ class MoRIIOWriter:
still_deferred: list[WriteTask] = []
for task in self._deferred_tasks:
if self._is_transfer_terminal(task.transfer_id):
continue
if now - task.enqueue_time > defer_timeout:
logger.error(
"Deferred write task for request %s expired after %.1fs "
@@ -181,12 +246,25 @@ class MoRIIOWriter:
self._deferred_tasks = still_deferred
def _clear_transfer_state(self, transfer_id: TransferId) -> None:
with self._write_state_lock:
self._scheduled_writes.pop(transfer_id, None)
self._scheduled_layers.pop(transfer_id, None)
self._sealed_writes.pop(transfer_id, None)
def _is_transfer_terminal(self, transfer_id: TransferId) -> bool:
wrapper = self.worker.moriio_wrapper
with wrapper.lock:
return wrapper._is_transfer_terminal_locked(transfer_id)
def _mark_request_done(self, transfer_id: str) -> None:
"""Mark a request done so its blocks are freed, even on transfer failure."""
wrapper = self.worker.moriio_wrapper
with wrapper.lock:
wrapper.done_req_ids.append(transfer_id)
wrapper.done_remote_allocate_req_dict.pop(transfer_id, None)
wrapper.done_remote_allocate_req_dict.pop(transfer_id, None)
wrapper._mark_transfer_terminal_locked(transfer_id)
self._clear_transfer_state(transfer_id)
def _is_remote_ready(self, task: WriteTask) -> bool:
"""Check if remote blocks are allocated for this task.
@@ -229,6 +307,12 @@ class MoRIIOWriter:
"""
# Get remote allocation info
request_info = self._get_remote_alloc_info(task.transfer_id)
with self._write_state_lock:
request_info.completion_request_id = task.request_id
request_info.completion_remote_notify_port = task.remote_notify_port
request_info.completion_remote_ip = task.remote_ip
if task.transfer_id in self._sealed_writes:
request_info.writes_expected = self._sealed_writes[task.transfer_id]
if request_info.block_ids is None:
logger.debug(
@@ -259,10 +343,12 @@ class MoRIIOWriter:
plan = self._prepare_transfer_plan(task, request_info, remote_moriio_meta)
# Execute transfer
self._do_layer_write(plan, sessions)
transfer_statuses = self._do_layer_write(plan, sessions)
with self._write_state_lock:
request_info.transfer_statuses.extend(transfer_statuses)
# Finalize if all layers complete
self._finalize_if_complete(task, request_info)
self._mark_write_done(task.transfer_id, request_info)
def _prepare_transfer_plan(
self,
@@ -279,21 +365,23 @@ class MoRIIOWriter:
Returns:
The transfer plan
"""
# Compute offsets if not cached
if request_info.transfer_offset is None:
layer_cache = self.worker.kv_caches[task.layer_name]
geometry_key = _get_write_geometry_key(layer_cache)
offsets = request_info.transfer_offsets.get(geometry_key)
if offsets is None:
offsets = self.worker._compute_block_transfer_offsets(
task.layer_name,
task.local_block_ids,
request_info.block_ids,
remote_moriio_meta,
)
request_info.transfer_offset = offsets
request_info.transfer_offsets[geometry_key] = offsets
# Get session index
layer_names = list(self.worker.layer_name_to_local_kv_cache_metadata.keys())
sess_idx = layer_names.index(task.layer_name)
local_off, remote_off, sizes = request_info.transfer_offset
local_off, remote_off, sizes = offsets
return LayerTransferPlan(
request_id=task.request_id,
@@ -306,7 +394,7 @@ class MoRIIOWriter:
use_batch=True,
)
def _do_layer_write(self, plan: LayerTransferPlan, sessions: list) -> None:
def _do_layer_write(self, plan: LayerTransferPlan, sessions: list) -> list[Any]:
"""Perform the actual layer write.
Args:
@@ -314,59 +402,82 @@ class MoRIIOWriter:
sessions: List of transfer sessions
"""
if plan.use_batch:
self.worker.moriio_wrapper.write_remote_data(
plan.transfer_sizes,
plan.transfer_local_offsets,
plan.transfer_remote_offsets,
sessions[plan.sess_idx],
)
else:
for i in range(len(plan.transfer_local_offsets)):
return [
self.worker.moriio_wrapper.write_remote_data(
plan.transfer_sizes,
plan.transfer_local_offsets,
plan.transfer_remote_offsets,
sessions[plan.sess_idx],
)
]
transfer_statuses: list[Any] = []
for i in range(len(plan.transfer_local_offsets)):
transfer_statuses.append(
self.worker.moriio_wrapper.write_remote_data_single(
plan.transfer_sizes[i],
plan.transfer_local_offsets[i],
plan.transfer_remote_offsets[i],
plan.sess_idx,
)
)
return transfer_statuses
def _mark_write_done(
self, transfer_id: TransferId, request_info: RemoteAllocInfo
) -> None:
"""Record one completed WRITE task and finalize if sealed."""
with self._write_state_lock:
request_info.writes_done += 1
self._finalize_if_complete(transfer_id, request_info)
def _finalize_if_complete(
self, task: WriteTask, request_info: RemoteAllocInfo
self, transfer_id: TransferId, request_info: RemoteAllocInfo
) -> None:
"""Finalize transfer if all layers are complete.
"""Finalize transfer if all scheduled writes are complete."""
with self._write_state_lock:
expected = request_info.writes_expected
if expected is None or request_info.writes_done < expected:
return
if request_info.completion_notified:
return
request_id = request_info.completion_request_id
remote_notify_port = request_info.completion_remote_notify_port
remote_ip = request_info.completion_remote_ip
if request_id is None or remote_notify_port is None or remote_ip is None:
return
transfer_statuses = list(request_info.transfer_statuses)
request_info.transfer_statuses.clear()
request_info.completion_notified = True
Args:
task: The write task
request_info: Remote allocation information
"""
request_info.writes_done += 1
# Wait for this request's transfers to complete.
self.worker.moriio_wrapper.waiting_for_transfer_complete(transfer_statuses)
if request_info.writes_done >= self.worker.num_layers:
# Wait for transfer to complete
self.worker.moriio_wrapper.waiting_for_transfer_complete()
remote_port = remote_notify_port + get_port_offset(
request_info.decode_dp_rank, self.worker.tp_rank
)
# Consider using RDMA immediate data in decode side
# to eliminate the need for this notification.
# Consider including the first gen token from prefill in the notification
remote_port = task.remote_notify_port + get_port_offset(
request_info.decode_dp_rank, self.worker.tp_rank
)
# Consider using RDMA immediate data in decode side
# to eliminate the need for this notification.
# Consider including the first gen token from prefill in the notification
# Send completion notification
self.worker.moriio_wrapper.send_notify(
task.transfer_id, task.remote_ip, remote_port
)
# mark request as done, then we can free the blocks
with self.worker.moriio_wrapper.lock:
self.worker.moriio_wrapper.done_req_ids.append(task.transfer_id)
del self.worker.moriio_wrapper.done_remote_allocate_req_dict[
task.transfer_id
]
logger.debug(
"Completed transfer for (request, transfer) %s, %s, notified port %d",
task.request_id,
task.transfer_id,
remote_port,
# Send completion notification
self.worker.moriio_wrapper.send_notify(
transfer_id, remote_ip, remote_port, message_type="write_done"
)
# mark request as done, then we can free the blocks
with self.worker.moriio_wrapper.lock:
self.worker.moriio_wrapper.done_req_ids.append(transfer_id)
self.worker.moriio_wrapper.done_remote_allocate_req_dict.pop(
transfer_id, None
)
self.worker.moriio_wrapper._mark_transfer_terminal_locked(transfer_id)
self._clear_transfer_state(transfer_id)
logger.debug(
"Completed transfer for (request, transfer) %s, %s, notified port %d",
request_id,
transfer_id,
remote_port,
)
class MoRIIOWrapper:
@@ -400,6 +511,7 @@ class MoRIIOWrapper:
self.done_req_ids: list[str] = []
self.done_remote_allocate_req_dict: dict[TransferId, RemoteAllocInfo] = {}
self.done_write_cache_req_ids: list[str] = []
self._terminal_transfer_ids: OrderedDict[TransferId, None] = OrderedDict()
self._transfer_timeout = transfer_timeout
self.notify_thread: threading.Thread | None = None
self.sessions: list[IOEngine.Session] = []
@@ -506,8 +618,7 @@ class MoRIIOWrapper:
transfer_status = session.batch_write(
local_offset, remote_offset, transfer_size_byte, write_uid
)
with self.lock:
self.transfer_status.append(transfer_status)
return transfer_status
def write_remote_data_single(
self, transfer_size_byte, local_offset=0, remote_offset=0, sess_idx=0
@@ -520,17 +631,19 @@ class MoRIIOWrapper:
transfer_size_byte,
self.moriio_engine.allocate_transfer_uid(),
)
with self.lock:
self.transfer_status.append(transfer_status)
return transfer_status
def waiting_for_transfer_complete(self):
if not self.transfer_status:
def waiting_for_transfer_complete(self, transfer_statuses: list[Any] | None = None):
if transfer_statuses is None:
with self.lock:
transfers_to_wait = self.transfer_status[:]
self.transfer_status.clear()
else:
transfers_to_wait = list(transfer_statuses)
if not transfers_to_wait:
return
with self.lock:
transfers_to_wait = self.transfer_status[:]
self.transfer_status.clear()
timeout = self._transfer_timeout
deadline = time.monotonic() + timeout
remaining = list(transfers_to_wait)
@@ -598,49 +711,105 @@ class MoRIIOWrapper:
# [read] mode: receives block release messages from decode side
# Decode Role:
# [write] mode: receives KV cache write completion notifications
msg_str = repr(msg)
handled = False
try:
data = msgpack.loads(msg)
if isinstance(data, dict) and "req_id" in data:
if isinstance(data, dict):
self._handle_structured_message(data)
return
except (msgpack.exceptions.ExtraData, msgpack.exceptions.UnpackException):
except (
msgpack.exceptions.ExtraData,
msgpack.exceptions.UnpackException,
ValueError,
):
logger.debug("Failed to decode msgpack message, will try as string")
pass
try:
msg_str = msg.decode("UTF-8")
if msg_str.startswith(MoRIIOConstants.TRANSFER_PREFIX):
if msg_str:
self._handle_completion_message(msg_str)
handled = True
except UnicodeDecodeError:
logger.warning("Received non-UTF8 message: %s", msg_str)
logger.warning("Received non-UTF8 message: %r", msg)
if not handled:
raise MoRIIOError(f"Unhandled message format: {msg_str}")
def _handle_structured_message(self, data: dict):
message_type = data.get("type")
if message_type is None and "req_id" in data:
message_type = "remote_blocks"
if message_type == "remote_blocks":
self._handle_remote_blocks_message(data)
elif message_type == "write_done":
self._handle_write_done_message(data)
elif message_type == "release":
self._handle_release_message(data)
else:
raise MoRIIOError(f"Unhandled structured message type: {message_type}")
def _handle_remote_blocks_message(self, data: dict):
assert get_role() == ROLE.PRODUCER, "Only prefill can get block messages"
transfer_id = data["transfer_id"]
block_notify_list = data.get("block_notify_list", [])
decode_dp_rank = data.get("decode_rank", 0)
assert len(block_notify_list) > 0, (
"block_notify_list cannot be empty in remote allocate message"
)
if not block_notify_list:
raise MoRIIOError(
"block_notify_list cannot be empty in remote allocate message"
)
with self.lock:
if self._is_transfer_terminal_locked(transfer_id):
logger.debug(
"Ignoring remote allocation for terminal transfer %s",
transfer_id,
)
return
self.done_remote_allocate_req_dict[transfer_id] = RemoteAllocInfo(
block_ids=block_notify_list, decode_dp_rank=decode_dp_rank
)
def _handle_write_done_message(self, data: dict):
assert get_role() != ROLE.PRODUCER, (
"Only decode can get WRITE completion messages"
)
transfer_id = data["transfer_id"]
with self.lock:
self.done_write_cache_req_ids.append(transfer_id)
def _handle_release_message(self, data: dict):
assert get_role() == ROLE.PRODUCER, (
"Only prefill can get transfer release messages"
)
transfer_id = data["transfer_id"]
with self.lock:
self.done_req_ids.append(transfer_id)
self.done_remote_allocate_req_dict.pop(transfer_id, None)
self._mark_transfer_terminal_locked(transfer_id)
def _handle_completion_message(self, msg: str):
with self.lock:
if get_role() == ROLE.PRODUCER:
self.done_req_ids.append(msg)
self.done_remote_allocate_req_dict.pop(msg, None)
self._mark_transfer_terminal_locked(msg)
else:
self.done_write_cache_req_ids.append(msg)
def send_notify(self, req_ids, remote_ip, remote_port):
def _is_transfer_terminal_locked(self, transfer_id: TransferId) -> bool:
return transfer_id in self._terminal_transfer_ids
def _mark_transfer_terminal_locked(self, transfer_id: TransferId) -> None:
self._terminal_transfer_ids[transfer_id] = None
self._terminal_transfer_ids.move_to_end(transfer_id)
while len(self._terminal_transfer_ids) > _MAX_TERMINAL_TRANSFER_IDS:
self._terminal_transfer_ids.popitem(last=False)
def send_notify(
self, req_ids, remote_ip, remote_port, message_type: str | None = None
):
if not remote_ip or not remote_port:
logger.warning("Missing remote_ip or remote_port for notification")
return
@@ -664,7 +833,12 @@ class MoRIIOWrapper:
"Invalid req_id type: %s, expected str", type(req_id)
)
continue
sock.send(req_id.encode("utf-8"))
if message_type is None:
sock.send(req_id.encode("utf-8"))
else:
sock.send(
msgpack.dumps({"type": message_type, "transfer_id": req_id})
)
except Exception as e:
logger.error("Failed to send notification to %s: %s", path, e)
self.paths.pop(path, None)
@@ -56,6 +56,42 @@ def is_mla_cache_layer(
return isinstance(spec, (MLAAttentionSpec, SlidingWindowMLASpec))
def _spec_dim_matches(value: int, expected: int | None) -> bool:
return expected is None or value == expected
def _kernel_layout_matches(
spec: KVCacheSpec, kernel_block_size: int, num_kv_heads: int, head_dim: int
) -> bool:
if kernel_block_size <= 0 or spec.block_size % kernel_block_size != 0:
return False
return _spec_dim_matches(
num_kv_heads, getattr(spec, "num_kv_heads", None)
) and _spec_dim_matches(head_dim, getattr(spec, "head_size", None))
def _select_kernel_block_layout(
layer_name: str, shape: torch.Size, spec: KVCacheSpec
) -> tuple[int, int, int]:
axis2_matches = _kernel_layout_matches(spec, shape[2], shape[3], shape[4])
axis3_matches = _kernel_layout_matches(spec, shape[3], shape[2], shape[4])
if axis2_matches and axis3_matches and shape[2] != shape[3]:
raise ValueError(
f"Ambiguous MoRIIO kernel-block K/V cache shape for layer "
f"{layer_name}: {tuple(shape)}"
)
if axis2_matches:
return shape[2], shape[3], shape[4]
if axis3_matches:
return shape[3], shape[2], shape[4]
raise ValueError(
f"Unsupported MoRIIO K/V cache shape for layer {layer_name}: "
f"{tuple(shape)} does not contain block size {spec.block_size}"
)
def get_layer_transfer_geometry(
layer_name: str,
kv_cache: torch.Tensor,
@@ -65,6 +101,7 @@ def get_layer_transfer_geometry(
shape = kv_cache.shape
stride = kv_cache.stride()
element_size = kv_cache.element_size()
spec = layer_to_spec[layer_name]
is_mla_cache = is_mla_cache_layer(layer_to_spec, layer_name)
if is_mla_cache and len(shape) == 3:
@@ -85,25 +122,92 @@ def get_layer_transfer_geometry(
)
if not is_mla_cache and len(shape) == 5 and shape[0] == 2:
_, num_blocks, block_size, num_kv_heads, head_dim = shape
_, num_blocks = shape[:2]
kernel_blocks_per_block = 1
if shape[2] == spec.block_size:
block_size, num_kv_heads, head_dim = shape[2:]
elif shape[3] == spec.block_size:
num_kv_heads, block_size, head_dim = shape[2:]
else:
kernel_num_blocks = num_blocks
kernel_block_size, num_kv_heads, head_dim = _select_kernel_block_layout(
layer_name, shape, spec
)
kernel_blocks_per_block = spec.block_size // kernel_block_size
if kernel_num_blocks % kernel_blocks_per_block != 0:
raise ValueError(
f"Unsupported MoRIIO K/V cache shape for layer {layer_name}: "
f"{tuple(shape)} has {kernel_num_blocks} kernel blocks, "
f"not divisible by {kernel_blocks_per_block}"
)
num_blocks = kernel_num_blocks // kernel_blocks_per_block
block_size = spec.block_size
slot_size_bytes = num_kv_heads * head_dim * element_size
block_len = block_size * slot_size_bytes
remote_kv_stride = stride[1] * (remote_num_blocks or num_blocks)
return LayerTransferGeometry(
num_blocks=num_blocks,
block_size=block_size,
block_len=block_len,
slot_size_bytes=slot_size_bytes,
block_stride=stride[1],
block_stride=stride[1] * kernel_blocks_per_block,
local_kv_stride=stride[0],
remote_kv_stride=remote_kv_stride,
remote_kv_stride=(
stride[1] * kernel_blocks_per_block * (remote_num_blocks or num_blocks)
),
transfers_per_block=2,
regions_per_block=1,
split_kv_regions=True,
)
if not is_mla_cache and len(shape) == 5 and shape[1] == 2:
num_blocks, _, block_size, num_kv_heads, head_dim = shape
num_blocks = shape[0]
if shape[2] == spec.block_size:
block_size, num_kv_heads, head_dim = shape[2:]
slot_size_bytes = num_kv_heads * head_dim * element_size
block_len = block_size * slot_size_bytes
return LayerTransferGeometry(
num_blocks=num_blocks,
block_size=block_size,
block_len=block_len,
slot_size_bytes=slot_size_bytes,
block_stride=stride[0],
local_kv_stride=stride[1],
remote_kv_stride=stride[1],
transfers_per_block=2,
regions_per_block=2,
split_kv_regions=False,
)
elif shape[3] == spec.block_size:
num_kv_heads, block_size, head_dim = shape[2:]
else:
kernel_num_blocks = num_blocks
kernel_block_size, _, _ = _select_kernel_block_layout(
layer_name, shape, spec
)
kernel_blocks_per_block = spec.block_size // kernel_block_size
if kernel_num_blocks % kernel_blocks_per_block != 0:
raise ValueError(
f"Unsupported MoRIIO K/V cache shape for layer {layer_name}: "
f"{tuple(shape)} has {kernel_num_blocks} kernel blocks, "
f"not divisible by {kernel_blocks_per_block}"
)
num_blocks = kernel_num_blocks // kernel_blocks_per_block
block_size = spec.block_size
block_stride = stride[0] * kernel_blocks_per_block
block_len = block_stride * element_size
slot_size_bytes = block_len // block_size
return LayerTransferGeometry(
num_blocks=num_blocks,
block_size=block_size,
block_len=block_len,
slot_size_bytes=slot_size_bytes,
block_stride=block_stride,
local_kv_stride=None,
remote_kv_stride=None,
transfers_per_block=1,
regions_per_block=1,
split_kv_regions=False,
)
slot_size_bytes = num_kv_heads * head_dim * element_size
block_len = block_size * slot_size_bytes
return LayerTransferGeometry(