[Bugfix][Mooncake] Report request-level KV load failures under HMA (#56855)

Signed-off-by: NickLucche <nicolo.lucchesi@mistral.ai>
This commit is contained in:
Nicolò Lucchesi
2026-09-16 14:55:40 +02:00
committed by GitHub
parent 903285fbcc
commit 8b1d188046
9 changed files with 329 additions and 377 deletions
@@ -30,6 +30,7 @@ from vllm.distributed.kv_transfer.kv_connector.v1.base import (
KVConnectorBase_V1,
KVConnectorMetadata,
KVConnectorRole,
KVConnectorTransferResults,
SupportsHMA,
)
from vllm.distributed.kv_transfer.kv_connector.v1.metrics import KVConnectorStats
@@ -587,6 +588,13 @@ class MooncakeConnector(KVConnectorBase_V1, SupportsHMA):
assert self.connector_worker is not None
return self.connector_worker.get_finished()
def get_transfer_results(
self, finished_req_ids: set[str]
) -> KVConnectorTransferResults:
"""Get finished and failed sends/recvs from the worker."""
assert self.connector_worker is not None
return self.connector_worker.get_transfer_results()
def get_block_ids_with_load_errors(self) -> set[int]:
"""Get block IDs whose remote KV load failed."""
assert self.connector_worker is not None
@@ -1046,6 +1054,16 @@ class MooncakeConnectorWorker:
self.finished_recving_reqs: set[ReqId] = set()
# Written from the receiver loop, drained from the worker thread.
self._invalid_block_ids: queue.Queue[set[int]] = queue.Queue()
self._is_hma_required = (
not vllm_config.scheduler_config.disable_hybrid_kv_cache_manager
and any(
not isinstance(g.kv_cache_spec, FullAttentionSpec)
for g in kv_cache_config.transfer_groups
)
)
# Block IDs are only unique within a group; with HMA the scheduler
# tracks a single merged group, so failures are reported per request.
self._failed_recv_reqs: queue.Queue[ReqId] = queue.Queue()
self.xfer_stats = MooncakeKVConnectorStats()
@@ -1866,6 +1884,29 @@ class MooncakeConnectorWorker:
return finished_sending_reqs or None, finished_recving_reqs or None
def get_transfer_results(self) -> KVConnectorTransferResults:
"""
Get transfers that completed on this specific worker, including
requests whose remote KV load failed.
The scheduler process (via the MultiprocExecutor) will use this output
to track which workers are done.
"""
finished_sending_reqs, finished_recving_reqs = self.get_finished()
failed_recving_reqs: set[ReqId] = set()
while True:
try:
failed_recving_reqs.add(self._failed_recv_reqs.get_nowait())
except queue.Empty:
break
return KVConnectorTransferResults(
finished_sending=set(finished_sending_reqs or ()),
finished_recving=set(finished_recving_reqs or ()),
failed_recving=failed_recving_reqs,
)
def get_kv_connector_stats(self) -> KVConnectorStats | None:
"""Return transfer stats collected since the last call, or None
if nothing has been recorded in this interval."""
@@ -1959,7 +2000,10 @@ class MooncakeConnectorWorker:
# request is waiting on a load, and reporting one here would trip
# the scheduler's `assert req_id in self.requests`.
continue
self._invalid_block_ids.put(invalid)
if self._is_hma_required:
self._failed_recv_reqs.put(pull_meta.d_req_id)
else:
self._invalid_block_ids.put(invalid)
self.finished_recving_reqs.add(pull_meta.d_req_id)
if failed:
@@ -27,6 +27,7 @@ from vllm.distributed.kv_transfer.kv_connector.v1.base import (
KVConnectorBase_V1,
KVConnectorMetadata,
KVConnectorRole,
KVConnectorTransferResults,
KVConnectorWorkerMetadata,
SupportsHMA,
)
@@ -359,6 +360,14 @@ class MooncakeStoreConnector(KVConnectorBase_V1, SupportsHMA):
assert isinstance(metadata, MooncakeStoreConnectorMetadata)
return self.connector_worker.get_finished(finished_req_ids, metadata)
def get_transfer_results(
self, finished_req_ids: set[str]
) -> KVConnectorTransferResults:
assert self.connector_worker is not None
metadata = self._get_connector_metadata()
assert isinstance(metadata, MooncakeStoreConnectorMetadata)
return self.connector_worker.get_transfer_results(finished_req_ids, metadata)
def get_block_ids_with_load_errors(self) -> set[int]:
assert self.connector_worker is not None
return self.connector_worker.get_block_ids_with_load_errors()
@@ -36,6 +36,9 @@ from vllm.distributed import (
get_tensor_model_parallel_world_size,
)
from vllm.distributed.kv_events import BlockStored
from vllm.distributed.kv_transfer.kv_connector.v1.base import (
KVConnectorTransferResults,
)
from vllm.distributed.kv_transfer.kv_connector.v1.mooncake import rdma_utils
from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.coordinator import ( # noqa: E501
ExternalCachedBlockPool,
@@ -1265,6 +1268,7 @@ class KVCacheStoreRecvingThread(KVTransferThread):
record_operation: Callable[..., None] | None = None,
request_queue: queue.Queue[Any] | None = None,
group_participates: Sequence[bool] | None = None,
is_hma_required: bool = False,
):
super().__init__(
store,
@@ -1284,6 +1288,10 @@ class KVCacheStoreRecvingThread(KVTransferThread):
# _invalid_block_ids can be access by both the Worker and RecvingThread
self._invalid_block_ids_lock = threading.Lock()
self._invalid_block_ids: set[int] = set()
# With HMA the scheduler tracks a single merged group while block IDs
# are only unique within a group, so failures are reported per request.
self._is_hma_required = is_hma_required
self._failed_requests: set[str] = set()
self.disk_offload_buffer_budget_bytes = disk_offload_buffer_budget_bytes
self.usable_disk_offload_buffer_budget_bytes = (
None
@@ -1298,6 +1306,22 @@ class KVCacheStoreRecvingThread(KVTransferThread):
with self._invalid_block_ids_lock:
self._invalid_block_ids.update(block_ids)
def set_failed_request(self, req_id: str):
with self.done_task_lock:
self._failed_requests.add(req_id)
def get_and_clear_failed_requests(self) -> set[str]:
with self.done_task_lock:
failed = self._failed_requests.copy()
self._failed_requests.clear()
return failed
def _report_load_error(self, req_id: str, block_ids: list[int]) -> None:
if self._is_hma_required:
self.set_failed_request(req_id)
else:
self._add_load_error_block_ids(block_ids)
def get_and_clear_block_ids_with_load_errors(self) -> set[int]:
with self._invalid_block_ids_lock:
invalid_block_ids = self._invalid_block_ids.copy()
@@ -1392,7 +1416,7 @@ class KVCacheStoreRecvingThread(KVTransferThread):
# Mark every block: we skip the whole request, and the
# tp_rank rotation means oversized_key isn't necessarily
# the first block in the request's original order.
self._add_load_error_block_ids(block_id_list_c)
self._report_load_error(req_id, block_id_list_c)
oversized_key_bytes = _estimate_disk_offload_staging_bytes(
size_list_c[oversized_key_index]
)
@@ -1465,8 +1489,8 @@ class KVCacheStoreRecvingThread(KVTransferThread):
num_failed_keys=len(failed),
)
if failed:
self._add_load_error_block_ids(
[block_id for _, _, block_id in failed]
self._report_load_error(
req_id, [block_id for _, _, block_id in failed]
)
logger.warning(
"Failed to get %d Mooncake keys from sub-batch "
@@ -1477,7 +1501,7 @@ class KVCacheStoreRecvingThread(KVTransferThread):
)
break
except Exception as e:
self._add_load_error_block_ids(current_batch_block_ids)
self._report_load_error(req_id, current_batch_block_ids)
self._record_operation(
"load_get",
load_get_start,
@@ -1551,6 +1575,13 @@ class MooncakeStoreWorker:
and not self.can_put
)
self.cache_config = vllm_config.cache_config
self._is_hma_required = (
not vllm_config.scheduler_config.disable_hybrid_kv_cache_manager
and any(
not isinstance(g.kv_cache_spec, FullAttentionSpec)
for g in kv_cache_config.transfer_groups
)
)
self.block_size, self.hash_block_size = resolve_kv_cache_block_sizes(
kv_cache_config, vllm_config
)
@@ -2096,6 +2127,7 @@ class MooncakeStoreWorker:
group.kv_cache_spec.prefix_cacheable
for group in self._kv_cache_groups
],
is_hma_required=self._is_hma_required,
)
recv_thread.name = f"KVCacheStoreRecvingThread-{i}"
recv_thread.start()
@@ -2185,6 +2217,26 @@ class MooncakeStoreWorker:
block_ids |= recv_thread.get_and_clear_block_ids_with_load_errors()
return block_ids
def get_transfer_results(
self, finished_req_ids: set[str], meta: MooncakeStoreConnectorMetadata
) -> KVConnectorTransferResults:
"""Get completed sends/recvs plus requests whose remote KV load failed."""
done_sending, done_recving = self.get_finished(finished_req_ids, meta)
if self._capacity_only:
return KVConnectorTransferResults(done_sending, done_recving)
failed_recving: set[str] = set()
if self.load_async:
for recv_thread in self.kv_recv_threads:
failed_recving |= recv_thread.get_and_clear_failed_requests()
return KVConnectorTransferResults(
finished_sending=done_sending,
finished_recving=done_recving,
failed_recving=failed_recving,
)
def _record_kv_connector_operation(
self,
operation: str,