[Bugfix][Mooncake] Report request-level KV load failures under HMA (#56855)
Signed-off-by: NickLucche <nicolo.lucchesi@mistral.ai>
This commit is contained in:
@@ -30,6 +30,7 @@ from vllm.distributed.kv_transfer.kv_connector.v1.base import (
|
||||
KVConnectorBase_V1,
|
||||
KVConnectorMetadata,
|
||||
KVConnectorRole,
|
||||
KVConnectorTransferResults,
|
||||
SupportsHMA,
|
||||
)
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.metrics import KVConnectorStats
|
||||
@@ -587,6 +588,13 @@ class MooncakeConnector(KVConnectorBase_V1, SupportsHMA):
|
||||
assert self.connector_worker is not None
|
||||
return self.connector_worker.get_finished()
|
||||
|
||||
def get_transfer_results(
|
||||
self, finished_req_ids: set[str]
|
||||
) -> KVConnectorTransferResults:
|
||||
"""Get finished and failed sends/recvs from the worker."""
|
||||
assert self.connector_worker is not None
|
||||
return self.connector_worker.get_transfer_results()
|
||||
|
||||
def get_block_ids_with_load_errors(self) -> set[int]:
|
||||
"""Get block IDs whose remote KV load failed."""
|
||||
assert self.connector_worker is not None
|
||||
@@ -1046,6 +1054,16 @@ class MooncakeConnectorWorker:
|
||||
self.finished_recving_reqs: set[ReqId] = set()
|
||||
# Written from the receiver loop, drained from the worker thread.
|
||||
self._invalid_block_ids: queue.Queue[set[int]] = queue.Queue()
|
||||
self._is_hma_required = (
|
||||
not vllm_config.scheduler_config.disable_hybrid_kv_cache_manager
|
||||
and any(
|
||||
not isinstance(g.kv_cache_spec, FullAttentionSpec)
|
||||
for g in kv_cache_config.transfer_groups
|
||||
)
|
||||
)
|
||||
# Block IDs are only unique within a group; with HMA the scheduler
|
||||
# tracks a single merged group, so failures are reported per request.
|
||||
self._failed_recv_reqs: queue.Queue[ReqId] = queue.Queue()
|
||||
|
||||
self.xfer_stats = MooncakeKVConnectorStats()
|
||||
|
||||
@@ -1866,6 +1884,29 @@ class MooncakeConnectorWorker:
|
||||
|
||||
return finished_sending_reqs or None, finished_recving_reqs or None
|
||||
|
||||
def get_transfer_results(self) -> KVConnectorTransferResults:
|
||||
"""
|
||||
Get transfers that completed on this specific worker, including
|
||||
requests whose remote KV load failed.
|
||||
|
||||
The scheduler process (via the MultiprocExecutor) will use this output
|
||||
to track which workers are done.
|
||||
"""
|
||||
finished_sending_reqs, finished_recving_reqs = self.get_finished()
|
||||
|
||||
failed_recving_reqs: set[ReqId] = set()
|
||||
while True:
|
||||
try:
|
||||
failed_recving_reqs.add(self._failed_recv_reqs.get_nowait())
|
||||
except queue.Empty:
|
||||
break
|
||||
|
||||
return KVConnectorTransferResults(
|
||||
finished_sending=set(finished_sending_reqs or ()),
|
||||
finished_recving=set(finished_recving_reqs or ()),
|
||||
failed_recving=failed_recving_reqs,
|
||||
)
|
||||
|
||||
def get_kv_connector_stats(self) -> KVConnectorStats | None:
|
||||
"""Return transfer stats collected since the last call, or None
|
||||
if nothing has been recorded in this interval."""
|
||||
@@ -1959,7 +2000,10 @@ class MooncakeConnectorWorker:
|
||||
# request is waiting on a load, and reporting one here would trip
|
||||
# the scheduler's `assert req_id in self.requests`.
|
||||
continue
|
||||
self._invalid_block_ids.put(invalid)
|
||||
if self._is_hma_required:
|
||||
self._failed_recv_reqs.put(pull_meta.d_req_id)
|
||||
else:
|
||||
self._invalid_block_ids.put(invalid)
|
||||
self.finished_recving_reqs.add(pull_meta.d_req_id)
|
||||
|
||||
if failed:
|
||||
|
||||
@@ -27,6 +27,7 @@ from vllm.distributed.kv_transfer.kv_connector.v1.base import (
|
||||
KVConnectorBase_V1,
|
||||
KVConnectorMetadata,
|
||||
KVConnectorRole,
|
||||
KVConnectorTransferResults,
|
||||
KVConnectorWorkerMetadata,
|
||||
SupportsHMA,
|
||||
)
|
||||
@@ -359,6 +360,14 @@ class MooncakeStoreConnector(KVConnectorBase_V1, SupportsHMA):
|
||||
assert isinstance(metadata, MooncakeStoreConnectorMetadata)
|
||||
return self.connector_worker.get_finished(finished_req_ids, metadata)
|
||||
|
||||
def get_transfer_results(
|
||||
self, finished_req_ids: set[str]
|
||||
) -> KVConnectorTransferResults:
|
||||
assert self.connector_worker is not None
|
||||
metadata = self._get_connector_metadata()
|
||||
assert isinstance(metadata, MooncakeStoreConnectorMetadata)
|
||||
return self.connector_worker.get_transfer_results(finished_req_ids, metadata)
|
||||
|
||||
def get_block_ids_with_load_errors(self) -> set[int]:
|
||||
assert self.connector_worker is not None
|
||||
return self.connector_worker.get_block_ids_with_load_errors()
|
||||
|
||||
@@ -36,6 +36,9 @@ from vllm.distributed import (
|
||||
get_tensor_model_parallel_world_size,
|
||||
)
|
||||
from vllm.distributed.kv_events import BlockStored
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.base import (
|
||||
KVConnectorTransferResults,
|
||||
)
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.mooncake import rdma_utils
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.coordinator import ( # noqa: E501
|
||||
ExternalCachedBlockPool,
|
||||
@@ -1265,6 +1268,7 @@ class KVCacheStoreRecvingThread(KVTransferThread):
|
||||
record_operation: Callable[..., None] | None = None,
|
||||
request_queue: queue.Queue[Any] | None = None,
|
||||
group_participates: Sequence[bool] | None = None,
|
||||
is_hma_required: bool = False,
|
||||
):
|
||||
super().__init__(
|
||||
store,
|
||||
@@ -1284,6 +1288,10 @@ class KVCacheStoreRecvingThread(KVTransferThread):
|
||||
# _invalid_block_ids can be access by both the Worker and RecvingThread
|
||||
self._invalid_block_ids_lock = threading.Lock()
|
||||
self._invalid_block_ids: set[int] = set()
|
||||
# With HMA the scheduler tracks a single merged group while block IDs
|
||||
# are only unique within a group, so failures are reported per request.
|
||||
self._is_hma_required = is_hma_required
|
||||
self._failed_requests: set[str] = set()
|
||||
self.disk_offload_buffer_budget_bytes = disk_offload_buffer_budget_bytes
|
||||
self.usable_disk_offload_buffer_budget_bytes = (
|
||||
None
|
||||
@@ -1298,6 +1306,22 @@ class KVCacheStoreRecvingThread(KVTransferThread):
|
||||
with self._invalid_block_ids_lock:
|
||||
self._invalid_block_ids.update(block_ids)
|
||||
|
||||
def set_failed_request(self, req_id: str):
|
||||
with self.done_task_lock:
|
||||
self._failed_requests.add(req_id)
|
||||
|
||||
def get_and_clear_failed_requests(self) -> set[str]:
|
||||
with self.done_task_lock:
|
||||
failed = self._failed_requests.copy()
|
||||
self._failed_requests.clear()
|
||||
return failed
|
||||
|
||||
def _report_load_error(self, req_id: str, block_ids: list[int]) -> None:
|
||||
if self._is_hma_required:
|
||||
self.set_failed_request(req_id)
|
||||
else:
|
||||
self._add_load_error_block_ids(block_ids)
|
||||
|
||||
def get_and_clear_block_ids_with_load_errors(self) -> set[int]:
|
||||
with self._invalid_block_ids_lock:
|
||||
invalid_block_ids = self._invalid_block_ids.copy()
|
||||
@@ -1392,7 +1416,7 @@ class KVCacheStoreRecvingThread(KVTransferThread):
|
||||
# Mark every block: we skip the whole request, and the
|
||||
# tp_rank rotation means oversized_key isn't necessarily
|
||||
# the first block in the request's original order.
|
||||
self._add_load_error_block_ids(block_id_list_c)
|
||||
self._report_load_error(req_id, block_id_list_c)
|
||||
oversized_key_bytes = _estimate_disk_offload_staging_bytes(
|
||||
size_list_c[oversized_key_index]
|
||||
)
|
||||
@@ -1465,8 +1489,8 @@ class KVCacheStoreRecvingThread(KVTransferThread):
|
||||
num_failed_keys=len(failed),
|
||||
)
|
||||
if failed:
|
||||
self._add_load_error_block_ids(
|
||||
[block_id for _, _, block_id in failed]
|
||||
self._report_load_error(
|
||||
req_id, [block_id for _, _, block_id in failed]
|
||||
)
|
||||
logger.warning(
|
||||
"Failed to get %d Mooncake keys from sub-batch "
|
||||
@@ -1477,7 +1501,7 @@ class KVCacheStoreRecvingThread(KVTransferThread):
|
||||
)
|
||||
break
|
||||
except Exception as e:
|
||||
self._add_load_error_block_ids(current_batch_block_ids)
|
||||
self._report_load_error(req_id, current_batch_block_ids)
|
||||
self._record_operation(
|
||||
"load_get",
|
||||
load_get_start,
|
||||
@@ -1551,6 +1575,13 @@ class MooncakeStoreWorker:
|
||||
and not self.can_put
|
||||
)
|
||||
self.cache_config = vllm_config.cache_config
|
||||
self._is_hma_required = (
|
||||
not vllm_config.scheduler_config.disable_hybrid_kv_cache_manager
|
||||
and any(
|
||||
not isinstance(g.kv_cache_spec, FullAttentionSpec)
|
||||
for g in kv_cache_config.transfer_groups
|
||||
)
|
||||
)
|
||||
self.block_size, self.hash_block_size = resolve_kv_cache_block_sizes(
|
||||
kv_cache_config, vllm_config
|
||||
)
|
||||
@@ -2096,6 +2127,7 @@ class MooncakeStoreWorker:
|
||||
group.kv_cache_spec.prefix_cacheable
|
||||
for group in self._kv_cache_groups
|
||||
],
|
||||
is_hma_required=self._is_hma_required,
|
||||
)
|
||||
recv_thread.name = f"KVCacheStoreRecvingThread-{i}"
|
||||
recv_thread.start()
|
||||
@@ -2185,6 +2217,26 @@ class MooncakeStoreWorker:
|
||||
block_ids |= recv_thread.get_and_clear_block_ids_with_load_errors()
|
||||
return block_ids
|
||||
|
||||
def get_transfer_results(
|
||||
self, finished_req_ids: set[str], meta: MooncakeStoreConnectorMetadata
|
||||
) -> KVConnectorTransferResults:
|
||||
"""Get completed sends/recvs plus requests whose remote KV load failed."""
|
||||
done_sending, done_recving = self.get_finished(finished_req_ids, meta)
|
||||
|
||||
if self._capacity_only:
|
||||
return KVConnectorTransferResults(done_sending, done_recving)
|
||||
|
||||
failed_recving: set[str] = set()
|
||||
if self.load_async:
|
||||
for recv_thread in self.kv_recv_threads:
|
||||
failed_recving |= recv_thread.get_and_clear_failed_requests()
|
||||
|
||||
return KVConnectorTransferResults(
|
||||
finished_sending=done_sending,
|
||||
finished_recving=done_recving,
|
||||
failed_recving=failed_recving,
|
||||
)
|
||||
|
||||
def _record_kv_connector_operation(
|
||||
self,
|
||||
operation: str,
|
||||
|
||||
Reference in New Issue
Block a user