[P/D] [NixlConnector] kv load recovery integration (#26171)
Signed-off-by: Will Eaton <weaton@redhat.com>
This commit is contained in:
@@ -68,6 +68,7 @@ except ImportError:
|
||||
NixlWrapper = None
|
||||
nixlXferTelemetry = None
|
||||
|
||||
|
||||
try:
|
||||
from nixl._api import nixl_agent_config
|
||||
except ImportError:
|
||||
@@ -234,6 +235,11 @@ class NixlConnector(KVConnectorBase_V1):
|
||||
assert self.connector_worker is not None
|
||||
return self.connector_worker.get_finished()
|
||||
|
||||
def get_block_ids_with_load_errors(self) -> set[int]:
|
||||
"""Get block IDs that failed to load via NIXL."""
|
||||
assert self.connector_worker is not None
|
||||
return self.connector_worker.get_block_ids_with_load_errors()
|
||||
|
||||
def get_kv_connector_stats(self) -> KVConnectorStats | None:
|
||||
assert self.connector_worker is not None
|
||||
return self.connector_worker.get_kv_connector_stats()
|
||||
@@ -614,6 +620,11 @@ class NixlConnectorWorker:
|
||||
# Set of requests that have been part of a batch, regardless of status.
|
||||
self._reqs_to_process: set[ReqId] = set()
|
||||
|
||||
# invalid blocks from failed NIXL operations
|
||||
self._invalid_block_ids: set[int] = set()
|
||||
# requests that skipped transfer (handshake or transfer failures)
|
||||
self._failed_recv_reqs: set[ReqId] = set()
|
||||
|
||||
# Background thread for handling new handshake requests.
|
||||
self._nixl_handshake_listener_t: threading.Thread | None = None
|
||||
# Background thread for initializing new NIXL handshakes.
|
||||
@@ -713,6 +724,8 @@ class NixlConnectorWorker:
|
||||
|
||||
# Send query for the request.
|
||||
with zmq_ctx(zmq.REQ, path) as sock:
|
||||
# Set receive timeout to 5 seconds to avoid hanging on dead server
|
||||
sock.setsockopt(zmq.RCVTIMEO, 5000) # milliseconds
|
||||
sock.send(GET_META_MSG)
|
||||
metadata_bytes = sock.recv()
|
||||
decoder = msgspec.msgpack.Decoder(NixlAgentMetadata)
|
||||
@@ -795,10 +808,20 @@ class NixlConnectorWorker:
|
||||
|
||||
fut.add_done_callback(done_callback)
|
||||
|
||||
# TODO: handle failure state of future in the
|
||||
# callback, we want to fail the request in this case.
|
||||
def request_ready(_f: Future[Any], entry=(req_id, meta)):
|
||||
self._ready_requests.put(entry)
|
||||
# check handshake success before proceeding with request
|
||||
def request_ready(f: Future[Any], entry=(req_id, meta)):
|
||||
try:
|
||||
# check if handshake succeeded
|
||||
f.result()
|
||||
self._ready_requests.put(entry)
|
||||
except Exception:
|
||||
# handshake failed - mark blocks as invalid
|
||||
logger.exception(
|
||||
"Handshake failed for request %s, marking blocks as invalid", req_id
|
||||
)
|
||||
if req_meta := self._recving_metadata.get(req_id):
|
||||
self._invalid_block_ids.update(req_meta.local_block_ids)
|
||||
self._failed_recv_reqs.add(req_id)
|
||||
|
||||
fut.add_done_callback(request_ready)
|
||||
|
||||
@@ -1205,6 +1228,11 @@ class NixlConnectorWorker:
|
||||
"""
|
||||
done_sending = self._get_new_notifs()
|
||||
done_recving = self._pop_done_transfers(self._recving_transfers)
|
||||
|
||||
# add requests that skipped transfer to done_recving
|
||||
done_recving.update(self._failed_recv_reqs)
|
||||
self._failed_recv_reqs.clear()
|
||||
|
||||
if len(done_sending) > 0 or len(done_recving) > 0:
|
||||
logger.debug(
|
||||
"Rank %s, get_finished: %s requests done sending "
|
||||
@@ -1214,10 +1242,10 @@ class NixlConnectorWorker:
|
||||
len(done_recving),
|
||||
)
|
||||
|
||||
if self.use_host_buffer:
|
||||
for req_id in done_recving:
|
||||
meta = self._recving_metadata.pop(req_id)
|
||||
assert meta, f"{req_id} not found in recving_metadata list"
|
||||
# clean up metadata for completed requests
|
||||
for req_id in done_recving:
|
||||
meta = self._recving_metadata.pop(req_id, None)
|
||||
if self.use_host_buffer and meta:
|
||||
self.sync_recved_kv_to_device(req_id, meta)
|
||||
|
||||
# Handle timeout to avoid stranding blocks on remote.
|
||||
@@ -1296,7 +1324,19 @@ class NixlConnectorWorker:
|
||||
in_progress = True
|
||||
continue
|
||||
else:
|
||||
raise RuntimeError("Transfer failed with state %s", xfer_state)
|
||||
# transfer failed - mark blocks as invalid
|
||||
logger.error(
|
||||
"NIXL transfer failed for request %s with state %s. "
|
||||
"Marking blocks as invalid.",
|
||||
req_id,
|
||||
xfer_state,
|
||||
)
|
||||
# mark all blocks for this request as invalid
|
||||
if meta := self._recving_metadata.pop(req_id, None):
|
||||
self._invalid_block_ids.update(meta.local_block_ids)
|
||||
self._recving_metadata.pop(req_id, None)
|
||||
self.nixl_wrapper.release_xfer_handle(handle)
|
||||
self.xfer_stats.record_failed_transfer()
|
||||
if not in_progress:
|
||||
done_req_ids.add(req_id)
|
||||
del transfers[req_id]
|
||||
@@ -1317,8 +1357,8 @@ class NixlConnectorWorker:
|
||||
len(meta.local_block_ids),
|
||||
len(meta.remote_block_ids),
|
||||
)
|
||||
if self.use_host_buffer:
|
||||
self._recving_metadata[req_id] = meta
|
||||
# always store metadata for failure recovery
|
||||
self._recving_metadata[req_id] = meta
|
||||
if remote_engine_id not in self._remote_agents:
|
||||
# Initiate handshake with remote engine to exchange metadata.
|
||||
with self._handshake_lock:
|
||||
@@ -1394,7 +1434,16 @@ class NixlConnectorWorker:
|
||||
if num_local_blocks == 0:
|
||||
remote_rank = self.tp_rank // tp_ratio
|
||||
agent_name = self._remote_agents[dst_engine_id][remote_rank]
|
||||
self.nixl_wrapper.send_notif(agent_name, notif_msg=notif_id)
|
||||
try:
|
||||
self.nixl_wrapper.send_notif(agent_name, notif_msg=notif_id)
|
||||
except Exception:
|
||||
logger.exception(
|
||||
"NIXL send_notif failed for request %s: "
|
||||
"P worker blocks will be freed after timeout. "
|
||||
"This may indicate network issues.",
|
||||
request_id,
|
||||
)
|
||||
self.xfer_stats.record_failed_notification()
|
||||
return
|
||||
|
||||
# Partial prefix cache hit: just read uncomputed blocks.
|
||||
@@ -1456,20 +1505,35 @@ class NixlConnectorWorker:
|
||||
assert len(local_block_descs_ids) == len(remote_block_descs_ids)
|
||||
|
||||
# Prepare transfer with Nixl.
|
||||
handle = self.nixl_wrapper.make_prepped_xfer(
|
||||
"READ",
|
||||
local_xfer_side_handle,
|
||||
local_block_descs_ids,
|
||||
remote_xfer_side_handle,
|
||||
remote_block_descs_ids,
|
||||
notif_msg=notif_id,
|
||||
)
|
||||
handle = None
|
||||
try:
|
||||
handle = self.nixl_wrapper.make_prepped_xfer(
|
||||
"READ",
|
||||
local_xfer_side_handle,
|
||||
local_block_descs_ids,
|
||||
remote_xfer_side_handle,
|
||||
remote_block_descs_ids,
|
||||
notif_msg=notif_id,
|
||||
)
|
||||
|
||||
# Begin async xfer.
|
||||
self.nixl_wrapper.transfer(handle)
|
||||
# Begin async xfer.
|
||||
self.nixl_wrapper.transfer(handle)
|
||||
|
||||
# Use handle to check completion in future step().
|
||||
self._recving_transfers[request_id].append((handle, time.perf_counter()))
|
||||
# Use handle to check completion in future step().
|
||||
self._recving_transfers[request_id].append((handle, time.perf_counter()))
|
||||
except Exception:
|
||||
logger.exception(
|
||||
"NIXL transfer setup/initiation failed for request %s. "
|
||||
"Marking blocks as invalid.",
|
||||
request_id,
|
||||
)
|
||||
# mark all blocks for this request as invalid
|
||||
if meta := self._recving_metadata.get(request_id):
|
||||
self._invalid_block_ids.update(meta.local_block_ids)
|
||||
self.xfer_stats.record_failed_transfer()
|
||||
if handle is not None:
|
||||
self.nixl_wrapper.release_xfer_handle(handle)
|
||||
self._failed_recv_reqs.add(request_id)
|
||||
|
||||
def _get_block_descs_ids(
|
||||
self, engine_id: str, block_ids: list[int], layer_idx: int | None = None
|
||||
@@ -1527,6 +1591,17 @@ class NixlConnectorWorker:
|
||||
return self.xfer_stats.clone_and_reset()
|
||||
return None
|
||||
|
||||
def get_block_ids_with_load_errors(self) -> set[int]:
|
||||
"""
|
||||
Return and clear the set of block IDs that failed to load.
|
||||
|
||||
This is called by the scheduler to identify blocks that need
|
||||
to be retried after a NIXL transfer failure.
|
||||
"""
|
||||
result = self._invalid_block_ids
|
||||
self._invalid_block_ids = set()
|
||||
return result
|
||||
|
||||
def shutdown(self):
|
||||
"""Shutdown the connector worker."""
|
||||
self._handshake_initiation_executor.shutdown(wait=False)
|
||||
@@ -1586,6 +1661,8 @@ class NixlKVConnectorStats(KVConnectorStats):
|
||||
"post_duration": [],
|
||||
"bytes_transferred": [],
|
||||
"num_descriptors": [],
|
||||
"num_failed_transfers": [],
|
||||
"num_failed_notifications": [],
|
||||
}
|
||||
|
||||
def record_transfer(self, res: nixlXferTelemetry):
|
||||
@@ -1595,6 +1672,14 @@ class NixlKVConnectorStats(KVConnectorStats):
|
||||
self.data["bytes_transferred"].append(res.totalBytes)
|
||||
self.data["num_descriptors"].append(res.descCount)
|
||||
|
||||
def record_failed_transfer(self):
|
||||
"""Record a failed NIXL transfer operation."""
|
||||
self.data["num_failed_transfers"].append(1.0)
|
||||
|
||||
def record_failed_notification(self):
|
||||
"""Record a failed NIXL notification (send_notif)."""
|
||||
self.data["num_failed_notifications"].append(1.0)
|
||||
|
||||
def clone_and_reset(self) -> "NixlKVConnectorStats":
|
||||
old = copy.copy(self)
|
||||
self.reset()
|
||||
|
||||
Reference in New Issue
Block a user