[ROCm][P/D] Fix MoRIIO WRITE mode for mixed KV layouts (#46290)
Signed-off-by: Tan Pin Siang <tanpinsiang@gmail.com> Co-authored-by: vllmellm <vllm.ellm@embeddedllm.com> Co-authored-by: Hongxia Yang <hongxia.yang@amd.com> Co-authored-by: Jun Kang Chow <junkangchow@gmail.com> Co-authored-by: Chun Fang <chun.fang@amd.com> Co-authored-by: TianDi101 <ditian12@amd.com> Co-authored-by: functionstackx <47992694+functionstackx@users.noreply.github.com>
This commit is contained in:
co-authored by
vllmellm
Hongxia Yang
Jun Kang Chow
Chun Fang
TianDi101
functionstackx
parent
a8481be7a9
commit
7e47fb72b5
@@ -78,8 +78,17 @@ class RemoteAllocInfo:
|
||||
|
||||
block_ids: list[int]
|
||||
writes_done: int = 0
|
||||
writes_expected: int | None = None
|
||||
decode_dp_rank: int = 0
|
||||
transfer_offset: tuple[list[int], list[int], list[int]] | None = None
|
||||
completion_request_id: str | None = None
|
||||
completion_remote_notify_port: int | None = None
|
||||
completion_remote_ip: str | None = None
|
||||
completion_notified: bool = False
|
||||
transfer_statuses: list[Any] = field(default_factory=list)
|
||||
transfer_offsets: dict[
|
||||
tuple[tuple[int, ...], tuple[int, ...], torch.dtype],
|
||||
tuple[list[int], list[int], list[int]],
|
||||
] = field(default_factory=dict)
|
||||
|
||||
|
||||
class ROLE(Enum):
|
||||
@@ -434,12 +443,21 @@ class MoRIIOConnectorMetadata(KVConnectorMetadata):
|
||||
):
|
||||
transfer_id = kv_transfer_params["transfer_id"]
|
||||
|
||||
# Parse host/ports from the request_id. The router embeds both zmq_addresses
|
||||
# in the request_id
|
||||
peer_zmq = get_peer_zmq_from_request_id(request_id, is_producer=write_mode)
|
||||
remote_host, remote_handshake_port, remote_notify_port = (
|
||||
parse_moriio_zmq_address(peer_zmq)
|
||||
)
|
||||
remote_host = kv_transfer_params.get("remote_host")
|
||||
remote_handshake_port = kv_transfer_params.get("remote_handshake_port")
|
||||
remote_notify_port = kv_transfer_params.get("remote_notify_port")
|
||||
if (
|
||||
remote_host is None
|
||||
or remote_handshake_port is None
|
||||
or remote_notify_port is None
|
||||
):
|
||||
# Parse host/ports from the request_id. The router embeds both
|
||||
# zmq_addresses in PD request IDs, but WRITE decode requests may carry
|
||||
# a plain request ID and get the remote address via kv_transfer_params.
|
||||
peer_zmq = get_peer_zmq_from_request_id(request_id, is_producer=write_mode)
|
||||
remote_host, remote_handshake_port, remote_notify_port = (
|
||||
parse_moriio_zmq_address(peer_zmq)
|
||||
)
|
||||
|
||||
_req = ReqMeta(
|
||||
transfer_id=transfer_id,
|
||||
@@ -447,9 +465,9 @@ class MoRIIOConnectorMetadata(KVConnectorMetadata):
|
||||
remote_block_ids=kv_transfer_params["remote_block_ids"],
|
||||
remote_engine_id=kv_transfer_params["remote_engine_id"],
|
||||
remote_host=remote_host,
|
||||
remote_port=remote_handshake_port,
|
||||
remote_handshake_port=remote_handshake_port,
|
||||
remote_notify_port=remote_notify_port,
|
||||
remote_port=int(remote_handshake_port),
|
||||
remote_handshake_port=int(remote_handshake_port),
|
||||
remote_notify_port=int(remote_notify_port),
|
||||
tp_size=kv_transfer_params.get("tp_size", 1),
|
||||
remote_dp_size=kv_transfer_params.get("remote_dp_size", 1),
|
||||
)
|
||||
|
||||
@@ -234,7 +234,13 @@ class MoRIIOConnector(KVConnectorBase_V1):
|
||||
return None
|
||||
|
||||
def wait_for_save(self):
|
||||
pass
|
||||
if self.mode != MoRIIOMode.WRITE or get_role() != ROLE.PRODUCER:
|
||||
return
|
||||
assert self.connector_worker is not None
|
||||
assert isinstance(self._connector_metadata, MoRIIOConnectorMetadata), (
|
||||
"Connector metadata not initialized yet"
|
||||
)
|
||||
self.connector_worker.wait_for_save(self._connector_metadata)
|
||||
|
||||
def shutdown(self):
|
||||
if self.connector_worker is not None:
|
||||
@@ -390,6 +396,49 @@ class MoRIIOConnectorScheduler:
|
||||
serialized_data = msgpack.dumps(data)
|
||||
self.paths[path].send(serialized_data)
|
||||
|
||||
def _send_transfer_release(self, transfer_id: TransferId, host: str, port: int):
|
||||
path = make_zmq_path("tcp", host, port)
|
||||
if path not in self.paths:
|
||||
ctx = zmq.Context.instance()
|
||||
sock = make_zmq_socket(
|
||||
ctx=ctx, path=path, socket_type=zmq.DEALER, bind=False
|
||||
)
|
||||
self.paths[path] = sock
|
||||
|
||||
self.paths[path].send(
|
||||
msgpack.dumps({"type": "release", "transfer_id": transfer_id})
|
||||
)
|
||||
|
||||
def _release_write_prefill_blocks(self, request_id: ReqId, params: dict[str, Any]):
|
||||
transfer_id = params.get("transfer_id")
|
||||
if transfer_id is None:
|
||||
logger.warning(
|
||||
"Cannot release WRITE prefill blocks for request %s: "
|
||||
"missing transfer_id",
|
||||
request_id,
|
||||
)
|
||||
return
|
||||
|
||||
remote_dp_rank = params.get("remote_dp_rank", 0)
|
||||
remote_host = params.get("remote_host")
|
||||
remote_notify_port = params.get("remote_notify_port")
|
||||
if remote_host is None or remote_notify_port is None:
|
||||
try:
|
||||
peer_zmq = get_peer_zmq_from_request_id(request_id, is_producer=False)
|
||||
remote_host, _, remote_notify_port = parse_moriio_zmq_address(peer_zmq)
|
||||
except ValueError:
|
||||
logger.warning(
|
||||
"Cannot release WRITE prefill blocks for request %s: "
|
||||
"missing remote notify address",
|
||||
request_id,
|
||||
)
|
||||
return
|
||||
|
||||
remote_notify_port = int(remote_notify_port)
|
||||
for tp_index in range(self.tp_size):
|
||||
target_port = remote_notify_port + get_port_offset(remote_dp_rank, tp_index)
|
||||
self._send_transfer_release(transfer_id, remote_host, target_port)
|
||||
|
||||
def update_state_after_alloc(
|
||||
self,
|
||||
request: "Request",
|
||||
@@ -443,11 +492,20 @@ class MoRIIOConnectorScheduler:
|
||||
)
|
||||
|
||||
remote_dp_rank = request.kv_transfer_params.get("remote_dp_rank", 0)
|
||||
|
||||
peer_zmq = get_peer_zmq_from_request_id(
|
||||
request.request_id, is_producer=False
|
||||
remote_host = request.kv_transfer_params.get("remote_host")
|
||||
remote_notify_port = request.kv_transfer_params.get(
|
||||
"remote_notify_port"
|
||||
)
|
||||
remote_host, _, remote_notify_port = parse_moriio_zmq_address(peer_zmq)
|
||||
if remote_host is None or remote_notify_port is None:
|
||||
peer_zmq = get_peer_zmq_from_request_id(
|
||||
request.request_id, is_producer=False
|
||||
)
|
||||
remote_host, _, remote_notify_port = parse_moriio_zmq_address(
|
||||
peer_zmq
|
||||
)
|
||||
remote_notify_port = int(remote_notify_port)
|
||||
|
||||
block_ids = blocks.get_block_ids()[0]
|
||||
|
||||
for tp_index in range(self.tp_size):
|
||||
target_port = remote_notify_port + get_port_offset(
|
||||
@@ -457,7 +515,7 @@ class MoRIIOConnectorScheduler:
|
||||
self.send_notify_block(
|
||||
req_id=request.request_id,
|
||||
transfer_id=request.kv_transfer_params["transfer_id"],
|
||||
block_notify_list=blocks.get_block_ids()[0],
|
||||
block_notify_list=block_ids,
|
||||
host=remote_host,
|
||||
port=target_port,
|
||||
)
|
||||
@@ -473,60 +531,31 @@ class MoRIIOConnectorScheduler:
|
||||
meta = MoRIIOConnectorMetadata()
|
||||
meta.transfer_id_to_request_id = self.transfer_id_to_request_id
|
||||
|
||||
if self.mode == MoRIIOMode.WRITE:
|
||||
# when async_load_kv finished,
|
||||
# new reqs will be added to scheduler_output.scheduled_new_reqs
|
||||
if self.mode == MoRIIOMode.WRITE and get_role() == ROLE.PRODUCER:
|
||||
# This is the logic for checking against chunked prefill.
|
||||
# When the last chunk is identified,
|
||||
# It places the request metadata into the saving queue.
|
||||
|
||||
if get_role() == ROLE.CONSUMER:
|
||||
for new_req in scheduler_output.scheduled_new_reqs:
|
||||
red_id = new_req.req_id
|
||||
local_block_ids = list(new_req.block_ids)[0]
|
||||
assert new_req.sampling_params is not None, (
|
||||
f"sampling_params is None for req {new_req.req_id}"
|
||||
)
|
||||
assert hasattr(new_req.sampling_params, "extra_args"), (
|
||||
f"sampling_params missing extra_args for req {new_req.req_id}"
|
||||
)
|
||||
kv_transfer_params = (
|
||||
new_req.sampling_params.extra_args.get("kv_transfer_params", {})
|
||||
if new_req.sampling_params.extra_args
|
||||
else {}
|
||||
)
|
||||
meta.add_new_req(
|
||||
red_id,
|
||||
local_block_ids,
|
||||
kv_transfer_params,
|
||||
)
|
||||
if get_role() == ROLE.PRODUCER:
|
||||
# This is the logic for checking against chunked prefill.
|
||||
# When the last chunk is identified,
|
||||
# It places the request metadata into the saving queue.
|
||||
for i, req_id in enumerate(scheduler_output.scheduled_cached_reqs.req_ids):
|
||||
new_block_ids = scheduler_output.scheduled_cached_reqs.new_block_ids[i]
|
||||
|
||||
for i, req_id in enumerate(
|
||||
scheduler_output.scheduled_cached_reqs.req_ids
|
||||
):
|
||||
new_block_ids = (
|
||||
scheduler_output.scheduled_cached_reqs.new_block_ids[i]
|
||||
)
|
||||
|
||||
if new_block_ids is not None:
|
||||
block_ids = new_block_ids[0]
|
||||
# TODO : hybrid attn, etc
|
||||
req, existing_blocks = self._reqs_need_pending_save[req_id]
|
||||
updated_blocks = list(existing_blocks) + (block_ids)
|
||||
self._reqs_need_pending_save[req_id] = (req, updated_blocks)
|
||||
if (
|
||||
len(self._reqs_need_pending_save[req_id][1])
|
||||
* self.block_size
|
||||
>= req.num_prompt_tokens
|
||||
):
|
||||
meta.add_new_req(
|
||||
request_id=req_id,
|
||||
local_block_ids=self._reqs_need_pending_save[req_id][1],
|
||||
kv_transfer_params=req.kv_transfer_params or {},
|
||||
write_mode=True,
|
||||
)
|
||||
del self._reqs_need_pending_save[req_id]
|
||||
if new_block_ids is not None:
|
||||
block_ids = new_block_ids[0]
|
||||
# TODO : hybrid attn, etc
|
||||
req, existing_blocks = self._reqs_need_pending_save[req_id]
|
||||
updated_blocks = list(existing_blocks) + (block_ids)
|
||||
self._reqs_need_pending_save[req_id] = (req, updated_blocks)
|
||||
if (
|
||||
len(self._reqs_need_pending_save[req_id][1]) * self.block_size
|
||||
>= req.num_prompt_tokens
|
||||
):
|
||||
meta.add_new_req(
|
||||
request_id=req_id,
|
||||
local_block_ids=self._reqs_need_pending_save[req_id][1],
|
||||
kv_transfer_params=req.kv_transfer_params or {},
|
||||
write_mode=True,
|
||||
)
|
||||
del self._reqs_need_pending_save[req_id]
|
||||
|
||||
# Loop through scheduled reqs and convert to ReqMeta.
|
||||
for req_id, (req, block_ids) in self._reqs_need_recv.items():
|
||||
@@ -601,9 +630,15 @@ class MoRIIOConnectorScheduler:
|
||||
# update_state_after_alloc must not have been called (the request
|
||||
# must have been aborted before it was scheduled).
|
||||
# To avoid stranding the prefill blocks in the prefill instance,
|
||||
# we must add empty block_ids to _reqs_need_recv so that our
|
||||
# worker side will notify and free blocks in the prefill instance.
|
||||
self._reqs_need_recv[request.request_id] = (request, [])
|
||||
# READ mode adds empty block_ids to _reqs_need_recv so the worker
|
||||
# side notifies the prefill instance. WRITE mode should notify the
|
||||
# producer directly: there is no decode allocation for the producer
|
||||
# to write into, and a plain request_id may not contain router-
|
||||
# embedded MoRIIO ZMQ addresses.
|
||||
if self.mode == MoRIIOMode.WRITE:
|
||||
self._release_write_prefill_blocks(request.request_id, params)
|
||||
else:
|
||||
self._reqs_need_recv[request.request_id] = (request, [])
|
||||
params["do_remote_prefill"] = False
|
||||
return False, None
|
||||
|
||||
@@ -635,6 +670,10 @@ class MoRIIOConnectorScheduler:
|
||||
do_remote_decode=False,
|
||||
remote_block_ids=computed_block_ids,
|
||||
remote_engine_id=self.engine_id,
|
||||
remote_host=self.host_ip,
|
||||
remote_handshake_port=self.handshake_port,
|
||||
remote_notify_port=self.side_notify_port,
|
||||
remote_dp_size=self.vllm_config.parallel_config.data_parallel_size,
|
||||
tp_size=self.vllm_config.parallel_config.tensor_parallel_size,
|
||||
transfer_id=params["transfer_id"],
|
||||
)
|
||||
@@ -1480,7 +1519,7 @@ class MoRIIOConnectorWorker:
|
||||
metadata: MoRIIOConnectorMetadata,
|
||||
layer_name: str,
|
||||
kv_layer: torch.Tensor,
|
||||
attn_metadata: "AttentionMetadata",
|
||||
attn_metadata: "AttentionMetadata | None",
|
||||
**kwargs,
|
||||
):
|
||||
if not self.is_producer:
|
||||
@@ -1604,6 +1643,12 @@ class MoRIIOConnectorWorker:
|
||||
|
||||
self._reqs_to_send.update(metadata.reqs_to_send)
|
||||
|
||||
def wait_for_save(self, metadata: MoRIIOConnectorMetadata):
|
||||
if self.mode == MoRIIOMode.WRITE and self.is_producer:
|
||||
for layer_name, kv_layer in self.kv_caches.items():
|
||||
self.save_kv_layer(metadata, layer_name, kv_layer, None)
|
||||
self._writer.seal_pending_transfers()
|
||||
|
||||
def _read_blocks_for_req(self, req_id: str, meta: ReqMeta):
|
||||
logger.debug(
|
||||
"Remote agent %s available, calling _read_blocks for req %s",
|
||||
|
||||
@@ -2,6 +2,8 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import threading
|
||||
import time
|
||||
from collections import OrderedDict, defaultdict
|
||||
from queue import Empty, Queue
|
||||
from typing import TYPE_CHECKING, Any
|
||||
from weakref import ref as weakref_ref
|
||||
|
||||
@@ -18,8 +20,6 @@ from vllm.utils.network_utils import (
|
||||
if TYPE_CHECKING:
|
||||
from mori.io import BackendType
|
||||
|
||||
from queue import Empty, Queue
|
||||
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.moriio.moriio_common import (
|
||||
ROLE,
|
||||
HandshakeError,
|
||||
@@ -61,10 +61,25 @@ except ImportError:
|
||||
"""Write task execution logic for MoRIIO connector."""
|
||||
|
||||
|
||||
_MAX_TERMINAL_TRANSFER_IDS = 4096
|
||||
|
||||
|
||||
WriteGeometryKey = tuple[tuple[int, ...], tuple[int, ...], torch.dtype]
|
||||
|
||||
|
||||
def _get_write_geometry_key(kv_cache: torch.Tensor) -> WriteGeometryKey:
|
||||
return (tuple(kv_cache.shape), tuple(kv_cache.stride()), kv_cache.dtype)
|
||||
|
||||
|
||||
class MoRIIOWriter:
|
||||
"""Handles write operations for KV cache transfers.
|
||||
Implements distributed KV cache transfer using the MoRIIO library
|
||||
for RDMA-based communication between prefill and decode instances."""
|
||||
|
||||
WRITE mode state machine:
|
||||
D sends destination block allocation, P schedules one write per layer
|
||||
after the layer CUDA event, P seals the scheduled write count after
|
||||
forward, then P notifies D and releases P blocks after all scheduled
|
||||
writes complete.
|
||||
"""
|
||||
|
||||
def __init__(self, worker: "MoRIIOConnectorWorker"):
|
||||
"""Initialize the writer.
|
||||
@@ -76,7 +91,11 @@ class MoRIIOWriter:
|
||||
self._write_task_q: Queue[WriteTask] = Queue()
|
||||
self._write_worker_started = False
|
||||
self._write_worker_lock = threading.Lock()
|
||||
self._write_state_lock = threading.Lock()
|
||||
self._deferred_tasks: list[WriteTask] = []
|
||||
self._scheduled_writes: dict[TransferId, int] = defaultdict(int)
|
||||
self._scheduled_layers: dict[TransferId, set[str]] = defaultdict(set)
|
||||
self._sealed_writes: dict[TransferId, int] = {}
|
||||
self._defer_timeout = worker.moriio_config.defer_timeout
|
||||
|
||||
@property
|
||||
@@ -106,14 +125,55 @@ class MoRIIOWriter:
|
||||
thread.start()
|
||||
logger.info("Started MoRIIO write worker thread")
|
||||
|
||||
def schedule_write(self, task: WriteTask) -> None:
|
||||
def schedule_write(self, task: WriteTask) -> bool:
|
||||
"""Schedule a write task.
|
||||
|
||||
Args:
|
||||
task: The write task to schedule
|
||||
"""
|
||||
self.ensure_worker_started()
|
||||
if self._is_transfer_terminal(task.transfer_id):
|
||||
return False
|
||||
|
||||
with self._write_state_lock:
|
||||
if self._is_transfer_terminal(task.transfer_id):
|
||||
return False
|
||||
if task.layer_name in self._scheduled_layers[task.transfer_id]:
|
||||
return False
|
||||
self._scheduled_layers[task.transfer_id].add(task.layer_name)
|
||||
self._scheduled_writes[task.transfer_id] += 1
|
||||
self._write_task_q.put(task)
|
||||
return True
|
||||
|
||||
def is_scheduled(self, transfer_id: TransferId, layer_name: str) -> bool:
|
||||
with self._write_state_lock:
|
||||
return layer_name in self._scheduled_layers.get(transfer_id, set())
|
||||
|
||||
def seal_pending_transfers(self) -> None:
|
||||
"""Seal expected WRITE counts after the model forward has run.
|
||||
|
||||
`save_kv_layer` is only invoked for attention layers whose backend uses
|
||||
the standard KV connector hook. Hybrid models can register more KV
|
||||
cache tensors than the number of hooks that fire in a forward, so WRITE
|
||||
completion must be based on the tasks actually queued for the transfer.
|
||||
"""
|
||||
pending: list[tuple[TransferId, RemoteAllocInfo]] = []
|
||||
with self._write_state_lock:
|
||||
for transfer_id, write_count in self._scheduled_writes.items():
|
||||
if transfer_id in self._sealed_writes:
|
||||
continue
|
||||
self._sealed_writes[transfer_id] = write_count
|
||||
request_info = (
|
||||
self.worker.moriio_wrapper.done_remote_allocate_req_dict.get(
|
||||
transfer_id
|
||||
)
|
||||
)
|
||||
if request_info is not None:
|
||||
request_info.writes_expected = write_count
|
||||
pending.append((transfer_id, request_info))
|
||||
|
||||
for transfer_id, request_info in pending:
|
||||
self._finalize_if_complete(transfer_id, request_info)
|
||||
|
||||
def _write_worker_loop(self) -> None:
|
||||
"""Main loop for the write worker thread."""
|
||||
@@ -128,6 +188,9 @@ class MoRIIOWriter:
|
||||
except Empty:
|
||||
continue
|
||||
|
||||
if self._is_transfer_terminal(task.transfer_id):
|
||||
continue
|
||||
|
||||
# Check if remote blocks are ready
|
||||
if not self._is_remote_ready(task):
|
||||
# task.retry_count += 1
|
||||
@@ -158,6 +221,8 @@ class MoRIIOWriter:
|
||||
still_deferred: list[WriteTask] = []
|
||||
|
||||
for task in self._deferred_tasks:
|
||||
if self._is_transfer_terminal(task.transfer_id):
|
||||
continue
|
||||
if now - task.enqueue_time > defer_timeout:
|
||||
logger.error(
|
||||
"Deferred write task for request %s expired after %.1fs "
|
||||
@@ -181,12 +246,25 @@ class MoRIIOWriter:
|
||||
|
||||
self._deferred_tasks = still_deferred
|
||||
|
||||
def _clear_transfer_state(self, transfer_id: TransferId) -> None:
|
||||
with self._write_state_lock:
|
||||
self._scheduled_writes.pop(transfer_id, None)
|
||||
self._scheduled_layers.pop(transfer_id, None)
|
||||
self._sealed_writes.pop(transfer_id, None)
|
||||
|
||||
def _is_transfer_terminal(self, transfer_id: TransferId) -> bool:
|
||||
wrapper = self.worker.moriio_wrapper
|
||||
with wrapper.lock:
|
||||
return wrapper._is_transfer_terminal_locked(transfer_id)
|
||||
|
||||
def _mark_request_done(self, transfer_id: str) -> None:
|
||||
"""Mark a request done so its blocks are freed, even on transfer failure."""
|
||||
wrapper = self.worker.moriio_wrapper
|
||||
with wrapper.lock:
|
||||
wrapper.done_req_ids.append(transfer_id)
|
||||
wrapper.done_remote_allocate_req_dict.pop(transfer_id, None)
|
||||
wrapper.done_remote_allocate_req_dict.pop(transfer_id, None)
|
||||
wrapper._mark_transfer_terminal_locked(transfer_id)
|
||||
self._clear_transfer_state(transfer_id)
|
||||
|
||||
def _is_remote_ready(self, task: WriteTask) -> bool:
|
||||
"""Check if remote blocks are allocated for this task.
|
||||
@@ -229,6 +307,12 @@ class MoRIIOWriter:
|
||||
"""
|
||||
# Get remote allocation info
|
||||
request_info = self._get_remote_alloc_info(task.transfer_id)
|
||||
with self._write_state_lock:
|
||||
request_info.completion_request_id = task.request_id
|
||||
request_info.completion_remote_notify_port = task.remote_notify_port
|
||||
request_info.completion_remote_ip = task.remote_ip
|
||||
if task.transfer_id in self._sealed_writes:
|
||||
request_info.writes_expected = self._sealed_writes[task.transfer_id]
|
||||
|
||||
if request_info.block_ids is None:
|
||||
logger.debug(
|
||||
@@ -259,10 +343,12 @@ class MoRIIOWriter:
|
||||
plan = self._prepare_transfer_plan(task, request_info, remote_moriio_meta)
|
||||
|
||||
# Execute transfer
|
||||
self._do_layer_write(plan, sessions)
|
||||
transfer_statuses = self._do_layer_write(plan, sessions)
|
||||
with self._write_state_lock:
|
||||
request_info.transfer_statuses.extend(transfer_statuses)
|
||||
|
||||
# Finalize if all layers complete
|
||||
self._finalize_if_complete(task, request_info)
|
||||
self._mark_write_done(task.transfer_id, request_info)
|
||||
|
||||
def _prepare_transfer_plan(
|
||||
self,
|
||||
@@ -279,21 +365,23 @@ class MoRIIOWriter:
|
||||
Returns:
|
||||
The transfer plan
|
||||
"""
|
||||
# Compute offsets if not cached
|
||||
if request_info.transfer_offset is None:
|
||||
layer_cache = self.worker.kv_caches[task.layer_name]
|
||||
geometry_key = _get_write_geometry_key(layer_cache)
|
||||
offsets = request_info.transfer_offsets.get(geometry_key)
|
||||
if offsets is None:
|
||||
offsets = self.worker._compute_block_transfer_offsets(
|
||||
task.layer_name,
|
||||
task.local_block_ids,
|
||||
request_info.block_ids,
|
||||
remote_moriio_meta,
|
||||
)
|
||||
request_info.transfer_offset = offsets
|
||||
request_info.transfer_offsets[geometry_key] = offsets
|
||||
|
||||
# Get session index
|
||||
layer_names = list(self.worker.layer_name_to_local_kv_cache_metadata.keys())
|
||||
sess_idx = layer_names.index(task.layer_name)
|
||||
|
||||
local_off, remote_off, sizes = request_info.transfer_offset
|
||||
local_off, remote_off, sizes = offsets
|
||||
|
||||
return LayerTransferPlan(
|
||||
request_id=task.request_id,
|
||||
@@ -306,7 +394,7 @@ class MoRIIOWriter:
|
||||
use_batch=True,
|
||||
)
|
||||
|
||||
def _do_layer_write(self, plan: LayerTransferPlan, sessions: list) -> None:
|
||||
def _do_layer_write(self, plan: LayerTransferPlan, sessions: list) -> list[Any]:
|
||||
"""Perform the actual layer write.
|
||||
|
||||
Args:
|
||||
@@ -314,59 +402,82 @@ class MoRIIOWriter:
|
||||
sessions: List of transfer sessions
|
||||
"""
|
||||
if plan.use_batch:
|
||||
self.worker.moriio_wrapper.write_remote_data(
|
||||
plan.transfer_sizes,
|
||||
plan.transfer_local_offsets,
|
||||
plan.transfer_remote_offsets,
|
||||
sessions[plan.sess_idx],
|
||||
)
|
||||
else:
|
||||
for i in range(len(plan.transfer_local_offsets)):
|
||||
return [
|
||||
self.worker.moriio_wrapper.write_remote_data(
|
||||
plan.transfer_sizes,
|
||||
plan.transfer_local_offsets,
|
||||
plan.transfer_remote_offsets,
|
||||
sessions[plan.sess_idx],
|
||||
)
|
||||
]
|
||||
|
||||
transfer_statuses: list[Any] = []
|
||||
for i in range(len(plan.transfer_local_offsets)):
|
||||
transfer_statuses.append(
|
||||
self.worker.moriio_wrapper.write_remote_data_single(
|
||||
plan.transfer_sizes[i],
|
||||
plan.transfer_local_offsets[i],
|
||||
plan.transfer_remote_offsets[i],
|
||||
plan.sess_idx,
|
||||
)
|
||||
)
|
||||
return transfer_statuses
|
||||
|
||||
def _mark_write_done(
|
||||
self, transfer_id: TransferId, request_info: RemoteAllocInfo
|
||||
) -> None:
|
||||
"""Record one completed WRITE task and finalize if sealed."""
|
||||
with self._write_state_lock:
|
||||
request_info.writes_done += 1
|
||||
self._finalize_if_complete(transfer_id, request_info)
|
||||
|
||||
def _finalize_if_complete(
|
||||
self, task: WriteTask, request_info: RemoteAllocInfo
|
||||
self, transfer_id: TransferId, request_info: RemoteAllocInfo
|
||||
) -> None:
|
||||
"""Finalize transfer if all layers are complete.
|
||||
"""Finalize transfer if all scheduled writes are complete."""
|
||||
with self._write_state_lock:
|
||||
expected = request_info.writes_expected
|
||||
if expected is None or request_info.writes_done < expected:
|
||||
return
|
||||
if request_info.completion_notified:
|
||||
return
|
||||
request_id = request_info.completion_request_id
|
||||
remote_notify_port = request_info.completion_remote_notify_port
|
||||
remote_ip = request_info.completion_remote_ip
|
||||
if request_id is None or remote_notify_port is None or remote_ip is None:
|
||||
return
|
||||
transfer_statuses = list(request_info.transfer_statuses)
|
||||
request_info.transfer_statuses.clear()
|
||||
request_info.completion_notified = True
|
||||
|
||||
Args:
|
||||
task: The write task
|
||||
request_info: Remote allocation information
|
||||
"""
|
||||
request_info.writes_done += 1
|
||||
# Wait for this request's transfers to complete.
|
||||
self.worker.moriio_wrapper.waiting_for_transfer_complete(transfer_statuses)
|
||||
|
||||
if request_info.writes_done >= self.worker.num_layers:
|
||||
# Wait for transfer to complete
|
||||
self.worker.moriio_wrapper.waiting_for_transfer_complete()
|
||||
remote_port = remote_notify_port + get_port_offset(
|
||||
request_info.decode_dp_rank, self.worker.tp_rank
|
||||
)
|
||||
# Consider using RDMA immediate data in decode side
|
||||
# to eliminate the need for this notification.
|
||||
# Consider including the first gen token from prefill in the notification
|
||||
|
||||
remote_port = task.remote_notify_port + get_port_offset(
|
||||
request_info.decode_dp_rank, self.worker.tp_rank
|
||||
)
|
||||
# Consider using RDMA immediate data in decode side
|
||||
# to eliminate the need for this notification.
|
||||
# Consider including the first gen token from prefill in the notification
|
||||
|
||||
# Send completion notification
|
||||
self.worker.moriio_wrapper.send_notify(
|
||||
task.transfer_id, task.remote_ip, remote_port
|
||||
)
|
||||
# mark request as done, then we can free the blocks
|
||||
with self.worker.moriio_wrapper.lock:
|
||||
self.worker.moriio_wrapper.done_req_ids.append(task.transfer_id)
|
||||
del self.worker.moriio_wrapper.done_remote_allocate_req_dict[
|
||||
task.transfer_id
|
||||
]
|
||||
logger.debug(
|
||||
"Completed transfer for (request, transfer) %s, %s, notified port %d",
|
||||
task.request_id,
|
||||
task.transfer_id,
|
||||
remote_port,
|
||||
# Send completion notification
|
||||
self.worker.moriio_wrapper.send_notify(
|
||||
transfer_id, remote_ip, remote_port, message_type="write_done"
|
||||
)
|
||||
# mark request as done, then we can free the blocks
|
||||
with self.worker.moriio_wrapper.lock:
|
||||
self.worker.moriio_wrapper.done_req_ids.append(transfer_id)
|
||||
self.worker.moriio_wrapper.done_remote_allocate_req_dict.pop(
|
||||
transfer_id, None
|
||||
)
|
||||
self.worker.moriio_wrapper._mark_transfer_terminal_locked(transfer_id)
|
||||
self._clear_transfer_state(transfer_id)
|
||||
logger.debug(
|
||||
"Completed transfer for (request, transfer) %s, %s, notified port %d",
|
||||
request_id,
|
||||
transfer_id,
|
||||
remote_port,
|
||||
)
|
||||
|
||||
|
||||
class MoRIIOWrapper:
|
||||
@@ -400,6 +511,7 @@ class MoRIIOWrapper:
|
||||
self.done_req_ids: list[str] = []
|
||||
self.done_remote_allocate_req_dict: dict[TransferId, RemoteAllocInfo] = {}
|
||||
self.done_write_cache_req_ids: list[str] = []
|
||||
self._terminal_transfer_ids: OrderedDict[TransferId, None] = OrderedDict()
|
||||
self._transfer_timeout = transfer_timeout
|
||||
self.notify_thread: threading.Thread | None = None
|
||||
self.sessions: list[IOEngine.Session] = []
|
||||
@@ -506,8 +618,7 @@ class MoRIIOWrapper:
|
||||
transfer_status = session.batch_write(
|
||||
local_offset, remote_offset, transfer_size_byte, write_uid
|
||||
)
|
||||
with self.lock:
|
||||
self.transfer_status.append(transfer_status)
|
||||
return transfer_status
|
||||
|
||||
def write_remote_data_single(
|
||||
self, transfer_size_byte, local_offset=0, remote_offset=0, sess_idx=0
|
||||
@@ -520,17 +631,19 @@ class MoRIIOWrapper:
|
||||
transfer_size_byte,
|
||||
self.moriio_engine.allocate_transfer_uid(),
|
||||
)
|
||||
with self.lock:
|
||||
self.transfer_status.append(transfer_status)
|
||||
return transfer_status
|
||||
|
||||
def waiting_for_transfer_complete(self):
|
||||
if not self.transfer_status:
|
||||
def waiting_for_transfer_complete(self, transfer_statuses: list[Any] | None = None):
|
||||
if transfer_statuses is None:
|
||||
with self.lock:
|
||||
transfers_to_wait = self.transfer_status[:]
|
||||
self.transfer_status.clear()
|
||||
else:
|
||||
transfers_to_wait = list(transfer_statuses)
|
||||
|
||||
if not transfers_to_wait:
|
||||
return
|
||||
|
||||
with self.lock:
|
||||
transfers_to_wait = self.transfer_status[:]
|
||||
self.transfer_status.clear()
|
||||
|
||||
timeout = self._transfer_timeout
|
||||
deadline = time.monotonic() + timeout
|
||||
remaining = list(transfers_to_wait)
|
||||
@@ -598,49 +711,105 @@ class MoRIIOWrapper:
|
||||
# [read] mode: receives block release messages from decode side
|
||||
# Decode Role:
|
||||
# [write] mode: receives KV cache write completion notifications
|
||||
msg_str = repr(msg)
|
||||
handled = False
|
||||
try:
|
||||
data = msgpack.loads(msg)
|
||||
if isinstance(data, dict) and "req_id" in data:
|
||||
if isinstance(data, dict):
|
||||
self._handle_structured_message(data)
|
||||
|
||||
return
|
||||
except (msgpack.exceptions.ExtraData, msgpack.exceptions.UnpackException):
|
||||
except (
|
||||
msgpack.exceptions.ExtraData,
|
||||
msgpack.exceptions.UnpackException,
|
||||
ValueError,
|
||||
):
|
||||
logger.debug("Failed to decode msgpack message, will try as string")
|
||||
pass
|
||||
|
||||
try:
|
||||
msg_str = msg.decode("UTF-8")
|
||||
if msg_str.startswith(MoRIIOConstants.TRANSFER_PREFIX):
|
||||
if msg_str:
|
||||
self._handle_completion_message(msg_str)
|
||||
handled = True
|
||||
except UnicodeDecodeError:
|
||||
logger.warning("Received non-UTF8 message: %s", msg_str)
|
||||
logger.warning("Received non-UTF8 message: %r", msg)
|
||||
if not handled:
|
||||
raise MoRIIOError(f"Unhandled message format: {msg_str}")
|
||||
|
||||
def _handle_structured_message(self, data: dict):
|
||||
message_type = data.get("type")
|
||||
if message_type is None and "req_id" in data:
|
||||
message_type = "remote_blocks"
|
||||
|
||||
if message_type == "remote_blocks":
|
||||
self._handle_remote_blocks_message(data)
|
||||
elif message_type == "write_done":
|
||||
self._handle_write_done_message(data)
|
||||
elif message_type == "release":
|
||||
self._handle_release_message(data)
|
||||
else:
|
||||
raise MoRIIOError(f"Unhandled structured message type: {message_type}")
|
||||
|
||||
def _handle_remote_blocks_message(self, data: dict):
|
||||
assert get_role() == ROLE.PRODUCER, "Only prefill can get block messages"
|
||||
transfer_id = data["transfer_id"]
|
||||
block_notify_list = data.get("block_notify_list", [])
|
||||
decode_dp_rank = data.get("decode_rank", 0)
|
||||
assert len(block_notify_list) > 0, (
|
||||
"block_notify_list cannot be empty in remote allocate message"
|
||||
)
|
||||
if not block_notify_list:
|
||||
raise MoRIIOError(
|
||||
"block_notify_list cannot be empty in remote allocate message"
|
||||
)
|
||||
|
||||
with self.lock:
|
||||
if self._is_transfer_terminal_locked(transfer_id):
|
||||
logger.debug(
|
||||
"Ignoring remote allocation for terminal transfer %s",
|
||||
transfer_id,
|
||||
)
|
||||
return
|
||||
self.done_remote_allocate_req_dict[transfer_id] = RemoteAllocInfo(
|
||||
block_ids=block_notify_list, decode_dp_rank=decode_dp_rank
|
||||
)
|
||||
|
||||
def _handle_write_done_message(self, data: dict):
|
||||
assert get_role() != ROLE.PRODUCER, (
|
||||
"Only decode can get WRITE completion messages"
|
||||
)
|
||||
transfer_id = data["transfer_id"]
|
||||
with self.lock:
|
||||
self.done_write_cache_req_ids.append(transfer_id)
|
||||
|
||||
def _handle_release_message(self, data: dict):
|
||||
assert get_role() == ROLE.PRODUCER, (
|
||||
"Only prefill can get transfer release messages"
|
||||
)
|
||||
transfer_id = data["transfer_id"]
|
||||
with self.lock:
|
||||
self.done_req_ids.append(transfer_id)
|
||||
self.done_remote_allocate_req_dict.pop(transfer_id, None)
|
||||
self._mark_transfer_terminal_locked(transfer_id)
|
||||
|
||||
def _handle_completion_message(self, msg: str):
|
||||
with self.lock:
|
||||
if get_role() == ROLE.PRODUCER:
|
||||
self.done_req_ids.append(msg)
|
||||
self.done_remote_allocate_req_dict.pop(msg, None)
|
||||
self._mark_transfer_terminal_locked(msg)
|
||||
else:
|
||||
self.done_write_cache_req_ids.append(msg)
|
||||
|
||||
def send_notify(self, req_ids, remote_ip, remote_port):
|
||||
def _is_transfer_terminal_locked(self, transfer_id: TransferId) -> bool:
|
||||
return transfer_id in self._terminal_transfer_ids
|
||||
|
||||
def _mark_transfer_terminal_locked(self, transfer_id: TransferId) -> None:
|
||||
self._terminal_transfer_ids[transfer_id] = None
|
||||
self._terminal_transfer_ids.move_to_end(transfer_id)
|
||||
while len(self._terminal_transfer_ids) > _MAX_TERMINAL_TRANSFER_IDS:
|
||||
self._terminal_transfer_ids.popitem(last=False)
|
||||
|
||||
def send_notify(
|
||||
self, req_ids, remote_ip, remote_port, message_type: str | None = None
|
||||
):
|
||||
if not remote_ip or not remote_port:
|
||||
logger.warning("Missing remote_ip or remote_port for notification")
|
||||
return
|
||||
@@ -664,7 +833,12 @@ class MoRIIOWrapper:
|
||||
"Invalid req_id type: %s, expected str", type(req_id)
|
||||
)
|
||||
continue
|
||||
sock.send(req_id.encode("utf-8"))
|
||||
if message_type is None:
|
||||
sock.send(req_id.encode("utf-8"))
|
||||
else:
|
||||
sock.send(
|
||||
msgpack.dumps({"type": message_type, "transfer_id": req_id})
|
||||
)
|
||||
except Exception as e:
|
||||
logger.error("Failed to send notification to %s: %s", path, e)
|
||||
self.paths.pop(path, None)
|
||||
|
||||
@@ -56,6 +56,42 @@ def is_mla_cache_layer(
|
||||
return isinstance(spec, (MLAAttentionSpec, SlidingWindowMLASpec))
|
||||
|
||||
|
||||
def _spec_dim_matches(value: int, expected: int | None) -> bool:
|
||||
return expected is None or value == expected
|
||||
|
||||
|
||||
def _kernel_layout_matches(
|
||||
spec: KVCacheSpec, kernel_block_size: int, num_kv_heads: int, head_dim: int
|
||||
) -> bool:
|
||||
if kernel_block_size <= 0 or spec.block_size % kernel_block_size != 0:
|
||||
return False
|
||||
return _spec_dim_matches(
|
||||
num_kv_heads, getattr(spec, "num_kv_heads", None)
|
||||
) and _spec_dim_matches(head_dim, getattr(spec, "head_size", None))
|
||||
|
||||
|
||||
def _select_kernel_block_layout(
|
||||
layer_name: str, shape: torch.Size, spec: KVCacheSpec
|
||||
) -> tuple[int, int, int]:
|
||||
axis2_matches = _kernel_layout_matches(spec, shape[2], shape[3], shape[4])
|
||||
axis3_matches = _kernel_layout_matches(spec, shape[3], shape[2], shape[4])
|
||||
|
||||
if axis2_matches and axis3_matches and shape[2] != shape[3]:
|
||||
raise ValueError(
|
||||
f"Ambiguous MoRIIO kernel-block K/V cache shape for layer "
|
||||
f"{layer_name}: {tuple(shape)}"
|
||||
)
|
||||
if axis2_matches:
|
||||
return shape[2], shape[3], shape[4]
|
||||
if axis3_matches:
|
||||
return shape[3], shape[2], shape[4]
|
||||
|
||||
raise ValueError(
|
||||
f"Unsupported MoRIIO K/V cache shape for layer {layer_name}: "
|
||||
f"{tuple(shape)} does not contain block size {spec.block_size}"
|
||||
)
|
||||
|
||||
|
||||
def get_layer_transfer_geometry(
|
||||
layer_name: str,
|
||||
kv_cache: torch.Tensor,
|
||||
@@ -65,6 +101,7 @@ def get_layer_transfer_geometry(
|
||||
shape = kv_cache.shape
|
||||
stride = kv_cache.stride()
|
||||
element_size = kv_cache.element_size()
|
||||
spec = layer_to_spec[layer_name]
|
||||
is_mla_cache = is_mla_cache_layer(layer_to_spec, layer_name)
|
||||
|
||||
if is_mla_cache and len(shape) == 3:
|
||||
@@ -85,25 +122,92 @@ def get_layer_transfer_geometry(
|
||||
)
|
||||
|
||||
if not is_mla_cache and len(shape) == 5 and shape[0] == 2:
|
||||
_, num_blocks, block_size, num_kv_heads, head_dim = shape
|
||||
_, num_blocks = shape[:2]
|
||||
kernel_blocks_per_block = 1
|
||||
if shape[2] == spec.block_size:
|
||||
block_size, num_kv_heads, head_dim = shape[2:]
|
||||
elif shape[3] == spec.block_size:
|
||||
num_kv_heads, block_size, head_dim = shape[2:]
|
||||
else:
|
||||
kernel_num_blocks = num_blocks
|
||||
kernel_block_size, num_kv_heads, head_dim = _select_kernel_block_layout(
|
||||
layer_name, shape, spec
|
||||
)
|
||||
kernel_blocks_per_block = spec.block_size // kernel_block_size
|
||||
if kernel_num_blocks % kernel_blocks_per_block != 0:
|
||||
raise ValueError(
|
||||
f"Unsupported MoRIIO K/V cache shape for layer {layer_name}: "
|
||||
f"{tuple(shape)} has {kernel_num_blocks} kernel blocks, "
|
||||
f"not divisible by {kernel_blocks_per_block}"
|
||||
)
|
||||
num_blocks = kernel_num_blocks // kernel_blocks_per_block
|
||||
block_size = spec.block_size
|
||||
slot_size_bytes = num_kv_heads * head_dim * element_size
|
||||
block_len = block_size * slot_size_bytes
|
||||
remote_kv_stride = stride[1] * (remote_num_blocks or num_blocks)
|
||||
return LayerTransferGeometry(
|
||||
num_blocks=num_blocks,
|
||||
block_size=block_size,
|
||||
block_len=block_len,
|
||||
slot_size_bytes=slot_size_bytes,
|
||||
block_stride=stride[1],
|
||||
block_stride=stride[1] * kernel_blocks_per_block,
|
||||
local_kv_stride=stride[0],
|
||||
remote_kv_stride=remote_kv_stride,
|
||||
remote_kv_stride=(
|
||||
stride[1] * kernel_blocks_per_block * (remote_num_blocks or num_blocks)
|
||||
),
|
||||
transfers_per_block=2,
|
||||
regions_per_block=1,
|
||||
split_kv_regions=True,
|
||||
)
|
||||
|
||||
if not is_mla_cache and len(shape) == 5 and shape[1] == 2:
|
||||
num_blocks, _, block_size, num_kv_heads, head_dim = shape
|
||||
num_blocks = shape[0]
|
||||
if shape[2] == spec.block_size:
|
||||
block_size, num_kv_heads, head_dim = shape[2:]
|
||||
slot_size_bytes = num_kv_heads * head_dim * element_size
|
||||
block_len = block_size * slot_size_bytes
|
||||
return LayerTransferGeometry(
|
||||
num_blocks=num_blocks,
|
||||
block_size=block_size,
|
||||
block_len=block_len,
|
||||
slot_size_bytes=slot_size_bytes,
|
||||
block_stride=stride[0],
|
||||
local_kv_stride=stride[1],
|
||||
remote_kv_stride=stride[1],
|
||||
transfers_per_block=2,
|
||||
regions_per_block=2,
|
||||
split_kv_regions=False,
|
||||
)
|
||||
elif shape[3] == spec.block_size:
|
||||
num_kv_heads, block_size, head_dim = shape[2:]
|
||||
else:
|
||||
kernel_num_blocks = num_blocks
|
||||
kernel_block_size, _, _ = _select_kernel_block_layout(
|
||||
layer_name, shape, spec
|
||||
)
|
||||
kernel_blocks_per_block = spec.block_size // kernel_block_size
|
||||
if kernel_num_blocks % kernel_blocks_per_block != 0:
|
||||
raise ValueError(
|
||||
f"Unsupported MoRIIO K/V cache shape for layer {layer_name}: "
|
||||
f"{tuple(shape)} has {kernel_num_blocks} kernel blocks, "
|
||||
f"not divisible by {kernel_blocks_per_block}"
|
||||
)
|
||||
num_blocks = kernel_num_blocks // kernel_blocks_per_block
|
||||
block_size = spec.block_size
|
||||
block_stride = stride[0] * kernel_blocks_per_block
|
||||
block_len = block_stride * element_size
|
||||
slot_size_bytes = block_len // block_size
|
||||
return LayerTransferGeometry(
|
||||
num_blocks=num_blocks,
|
||||
block_size=block_size,
|
||||
block_len=block_len,
|
||||
slot_size_bytes=slot_size_bytes,
|
||||
block_stride=block_stride,
|
||||
local_kv_stride=None,
|
||||
remote_kv_stride=None,
|
||||
transfers_per_block=1,
|
||||
regions_per_block=1,
|
||||
split_kv_regions=False,
|
||||
)
|
||||
slot_size_bytes = num_kv_heads * head_dim * element_size
|
||||
block_len = block_size * slot_size_bytes
|
||||
return LayerTransferGeometry(
|
||||
|
||||
Reference in New Issue
Block a user