[Bugfix][Mooncake] Report failed remote KV loads to the scheduler (#50984)
Signed-off-by: liuqihao <liuqihao@myhexin.com> Co-authored-by: liuqihao <liuqihao@myhexin.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
liuqihao
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
5236bef721
commit
2c7ee87223
@@ -2,9 +2,11 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import asyncio
|
||||
import logging
|
||||
import queue
|
||||
import threading
|
||||
import time
|
||||
from collections import defaultdict
|
||||
from collections.abc import Collection
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from dataclasses import dataclass
|
||||
from enum import IntEnum
|
||||
@@ -433,6 +435,9 @@ class PullReqMeta:
|
||||
expire_time: float = float("inf")
|
||||
# Designed for one D pairing to multiple P
|
||||
pull_tasks_count: int = 0
|
||||
# Set once any worker reports a failure, so a success from another worker
|
||||
# for the same request is not counted afterwards.
|
||||
failed: bool = False
|
||||
|
||||
|
||||
@dataclass
|
||||
@@ -582,6 +587,11 @@ class MooncakeConnector(KVConnectorBase_V1, SupportsHMA):
|
||||
assert self.connector_worker is not None
|
||||
return self.connector_worker.get_finished()
|
||||
|
||||
def get_block_ids_with_load_errors(self) -> set[int]:
|
||||
"""Get block IDs whose remote KV load failed."""
|
||||
assert self.connector_worker is not None
|
||||
return self.connector_worker.get_block_ids_with_load_errors()
|
||||
|
||||
def start_load_kv(self, forward_context: "ForwardContext", **kwargs) -> None:
|
||||
assert self.connector_worker is not None
|
||||
assert isinstance(self._connector_metadata, MooncakeConnectorMetadata)
|
||||
@@ -1034,6 +1044,8 @@ class MooncakeConnectorWorker:
|
||||
|
||||
self.finished_sending_reqs: set[ReqId] = set()
|
||||
self.finished_recving_reqs: set[ReqId] = set()
|
||||
# Written from the receiver loop, drained from the worker thread.
|
||||
self._invalid_block_ids: queue.Queue[set[int]] = queue.Queue()
|
||||
|
||||
self.xfer_stats = MooncakeKVConnectorStats()
|
||||
|
||||
@@ -1907,12 +1919,11 @@ class MooncakeConnectorWorker:
|
||||
ret_msg = await sock.recv()
|
||||
response = self._xfer_resp_decoder.decode(ret_msg)
|
||||
if response.status == MooncakeXferResponseStatus.ERROR:
|
||||
logger.error(
|
||||
"Error happens during transferring kvcache for %s: %s",
|
||||
self._handle_failed_recv(
|
||||
pull_metas,
|
||||
req_ids,
|
||||
response.err_msg,
|
||||
response.err_msg or "transfer error",
|
||||
)
|
||||
self.xfer_stats.record_failed_recv()
|
||||
return
|
||||
self.process_pulling_result(response, pull_metas)
|
||||
if response.status == MooncakeXferResponseStatus.FINISH:
|
||||
@@ -1920,10 +1931,50 @@ class MooncakeConnectorWorker:
|
||||
except zmq.ContextTerminated:
|
||||
logger.debug("ZMQ context terminated, exiting Mooncake receiver thread.")
|
||||
except Exception as e:
|
||||
logger.error("MooncakeXferMetadata transfer failed for %s: %s", req_ids, e)
|
||||
self.xfer_stats.record_failed_recv()
|
||||
self._handle_failed_recv(pull_metas, req_ids, f"transfer failed: {e}")
|
||||
return
|
||||
|
||||
def _handle_failed_recv(
|
||||
self,
|
||||
pull_metas: dict[ReqId, PullReqMeta],
|
||||
req_ids: Collection[ReqId],
|
||||
reason: str,
|
||||
) -> None:
|
||||
"""Report a failed remote KV load so the scheduler can fail or recompute it."""
|
||||
failed: list[ReqId] = []
|
||||
for req_id in req_ids:
|
||||
pull_meta = pull_metas.get(req_id)
|
||||
if pull_meta is None or pull_meta.failed:
|
||||
continue
|
||||
pull_meta.failed = True
|
||||
failed.append(req_id)
|
||||
self.xfer_stats.record_failed_recv()
|
||||
|
||||
invalid = {b for group in pull_meta.local_block_ids for b in group}
|
||||
if not invalid:
|
||||
# A pull with no local blocks only asks P to release its blocks
|
||||
# for a request that never reached the scheduler (see
|
||||
# AsyncLLM.notify_kv_transfer_request_rejected, which submits an
|
||||
# abort_immediately request just to run request_finished). No D
|
||||
# request is waiting on a load, and reporting one here would trip
|
||||
# the scheduler's `assert req_id in self.requests`.
|
||||
continue
|
||||
self._invalid_block_ids.put(invalid)
|
||||
self.finished_recving_reqs.add(pull_meta.d_req_id)
|
||||
|
||||
if failed:
|
||||
logger.error("pulling kv_caches for %s failed: %s", failed, reason)
|
||||
|
||||
def get_block_ids_with_load_errors(self) -> set[int]:
|
||||
"""Drain the blocks whose remote KV load failed since the last call."""
|
||||
result: set[int] = set()
|
||||
while True:
|
||||
try:
|
||||
result.update(self._invalid_block_ids.get_nowait())
|
||||
except queue.Empty:
|
||||
break
|
||||
return result
|
||||
|
||||
def process_pulling_result(
|
||||
self,
|
||||
response: MooncakeXferResponse,
|
||||
@@ -1933,6 +1984,8 @@ class MooncakeConnectorWorker:
|
||||
|
||||
for req_id in ok_reqs:
|
||||
pull_meta = pull_metas[req_id]
|
||||
if pull_meta.failed:
|
||||
continue
|
||||
# No race because we are in async loop.
|
||||
pull_meta.pull_tasks_count -= 1
|
||||
if pull_meta.pull_tasks_count == 0:
|
||||
@@ -1942,10 +1995,8 @@ class MooncakeConnectorWorker:
|
||||
logger.debug("pulling kv_caches for %s finished", ok_reqs)
|
||||
|
||||
if response.err_reqs:
|
||||
logger.error(
|
||||
"pulling kv_caches for %s failed: %s",
|
||||
response.err_reqs,
|
||||
response.err_msg,
|
||||
self._handle_failed_recv(
|
||||
pull_metas, response.err_reqs, response.err_msg or "unknown error"
|
||||
)
|
||||
|
||||
async def _connect_to_prefiller_bootstrap(self, remote_bootstrap_addr: str):
|
||||
@@ -2023,10 +2074,11 @@ class MooncakeConnectorWorker:
|
||||
await self._pending_bootstrap_queries[remote_bootstrap_addr].wait()
|
||||
|
||||
if remote_engine_id not in self._remote_agents:
|
||||
logger.error(
|
||||
"Failed to find remote engine_id %s from bootstrap server %s",
|
||||
remote_engine_id,
|
||||
remote_bootstrap_addr,
|
||||
self._handle_failed_recv(
|
||||
pull_metas,
|
||||
list(pull_metas),
|
||||
f"remote engine_id {remote_engine_id} not found from bootstrap "
|
||||
f"server {remote_bootstrap_addr}",
|
||||
)
|
||||
return
|
||||
|
||||
|
||||
Reference in New Issue
Block a user