[Bugfix][Mooncake] Reference GPU blocks for in-flight store jobs and key the store ledger by store_job_id (#52372)

Signed-off-by: Guanyi Chen <939416532@qq.com>
This commit is contained in:
Guanyi Chen
2026-08-16 16:09:46 -07:00
committed by GitHub
parent eee538d5da
commit dc9ae4b8ac
7 changed files with 477 additions and 253 deletions
@@ -26,6 +26,7 @@ from vllm.distributed.kv_transfer.kv_connector.v1.base import (
KVConnectorBase_V1,
KVConnectorMetadata,
KVConnectorRole,
KVConnectorWorkerMetadata,
SupportsHMA,
)
from vllm.distributed.kv_transfer.kv_connector.v1.metrics import (
@@ -37,6 +38,7 @@ from vllm.distributed.kv_transfer.kv_connector.v1.metrics import (
from vllm.forward_context import ForwardContext
from vllm.logger import init_logger
from vllm.v1.attention.backend import AttentionMetadata
from vllm.v1.core.block_pool import BlockPool
from vllm.v1.core.kv_cache_manager import KVCacheBlocks
from vllm.v1.core.sched.output import SchedulerOutput
from vllm.v1.kv_cache_interface import KVCacheConfig
@@ -201,6 +203,14 @@ class MooncakeStoreConnector(KVConnectorBase_V1, SupportsHMA):
request, blocks, num_external_tokens
)
def bind_gpu_block_pool(self, gpu_block_pool: BlockPool) -> None:
assert self.connector_scheduler is not None
self.connector_scheduler.bind_gpu_block_pool(gpu_block_pool)
def has_pending_push_work(self) -> bool:
assert self.connector_scheduler is not None
return self.connector_scheduler.has_pending_push_work()
def build_connector_meta(
self,
scheduler_output: SchedulerOutput,
@@ -208,6 +218,10 @@ class MooncakeStoreConnector(KVConnectorBase_V1, SupportsHMA):
assert self.connector_scheduler is not None
return self.connector_scheduler.build_connector_meta(scheduler_output)
def build_connector_worker_meta(self) -> KVConnectorWorkerMetadata | None:
assert self.connector_worker is not None
return self.connector_worker.build_connector_worker_meta()
def request_finished(
self,
request: Request,
@@ -220,8 +234,9 @@ class MooncakeStoreConnector(KVConnectorBase_V1, SupportsHMA):
request: Request,
block_ids: tuple[list[int], ...],
) -> tuple[bool, dict[str, Any] | None]:
assert self.connector_scheduler is not None
return self.connector_scheduler.request_finished(request, block_ids)
# An in-flight store job holds its own reference on the blocks it reads,
# so a finishing request never has to defer freeing them.
return False, None
def reset_cache(self) -> bool | None:
"""Reset the external Mooncake store on prefix-cache reset.
@@ -241,6 +256,9 @@ class MooncakeStoreConnector(KVConnectorBase_V1, SupportsHMA):
return None
def update_connector_output(self, connector_output: KVConnectorOutput):
assert self.connector_scheduler is not None
self.connector_scheduler.update_connector_output(connector_output)
kv_cache_events = connector_output.kv_cache_events
if not kv_cache_events or not isinstance(
kv_cache_events, MooncakeStoreKVEvents
@@ -6,7 +6,7 @@
"""Data classes for MooncakeStoreConnector."""
from collections.abc import Iterable, Sequence
from dataclasses import dataclass
from dataclasses import dataclass, field
from typing import cast
import numpy as np
@@ -14,6 +14,7 @@ import torch
from vllm.distributed.kv_transfer.kv_connector.v1.base import (
KVConnectorMetadata,
KVConnectorWorkerMetadata,
)
from vllm.logger import init_logger
from vllm.utils.math_utils import cdiv
@@ -366,6 +367,10 @@ class ReqMeta:
token_ids: list[int] | None = None
num_prompt_tokens: int | None = None
# Identifies this store job for the engine's lifetime. A request id cannot
# serve that purpose: it is reused once a preempted request resumes, so it
# would release the wrong job's blocks.
store_job_id: int | None = None
# Core-provided per-mamba-group
# (group_id, cow_block_id, boundary_tokens) for this request's partial tail.
# Present only on the producer's CoW step; drives the connector's offload
@@ -434,6 +439,23 @@ class ReqMeta:
)
@dataclass
class MooncakeStoreWorkerMetadata(KVConnectorWorkerMetadata):
"""Maps ``ReqMeta.store_job_id`` to the number of ranks done with that job."""
completed_saves: dict[int, int] = field(default_factory=dict)
def aggregate(
self, other: "KVConnectorWorkerMetadata"
) -> "MooncakeStoreWorkerMetadata":
assert isinstance(other, MooncakeStoreWorkerMetadata)
for store_job_id, count in other.completed_saves.items():
self.completed_saves[store_job_id] = (
self.completed_saves.get(store_job_id, 0) + count
)
return self
class MooncakeStoreConnectorMetadata(KVConnectorMetadata):
"""Metadata passed from scheduler to worker."""
@@ -5,8 +5,6 @@
# (vllm_ascend/distributed/kv_transfer/kv_pool/ascend_store/).
"""Scheduler-side logic for MooncakeStoreConnector."""
from typing import Any
from vllm.config import VllmConfig
from vllm.distributed.kv_transfer.kv_connector.v1.base import (
KVConnectorMetadata,
@@ -17,6 +15,7 @@ from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.coordinator imp
from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.data import ( # noqa: E501
LoadSpec,
MooncakeStoreConnectorMetadata,
MooncakeStoreWorkerMetadata,
ReqMeta,
RequestTracker,
)
@@ -24,10 +23,12 @@ from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.worker import (
LookupKeyClient,
)
from vllm.logger import init_logger
from vllm.v1.core.block_pool import BlockPool
from vllm.v1.core.kv_cache_manager import KVCacheBlocks
from vllm.v1.core.kv_cache_utils import resolve_kv_cache_block_sizes
from vllm.v1.core.sched.output import NewRequestData, SchedulerOutput
from vllm.v1.kv_cache_interface import KVCacheConfig
from vllm.v1.outputs import KVConnectorOutput
from vllm.v1.request import Request
logger = init_logger(__name__)
@@ -78,6 +79,15 @@ class MooncakeStoreScheduler:
self._unfinished_requests: dict[str, tuple[Request, tuple[list[int], ...]]] = {}
self._unfinished_request_ids: set[str] = set()
self._gpu_block_pool: BlockPool | None = None
self._num_workers = vllm_config.parallel_config.world_size
self._next_store_job_id = 0
# store_job_id -> (referenced block ids, ranks yet to report completion)
self._pinned_saves: dict[int, tuple[list[int], int]] = {}
def bind_gpu_block_pool(self, gpu_block_pool: BlockPool) -> None:
self._gpu_block_pool = gpu_block_pool
def get_num_new_matched_tokens(
self,
request: Request,
@@ -392,33 +402,75 @@ class MooncakeStoreScheduler:
)
)
self._reference_save_blocks(meta)
return meta
def request_finished(
self,
request: Request,
block_ids: tuple[list[int], ...],
) -> tuple[bool, dict[str, Any] | None]:
"""Determine whether to delay freeing blocks for async save."""
if self.kv_role == "kv_consumer":
return False, None
tracker = self._request_trackers.get(request.request_id)
# Missing tracker can happen when the request is aborted before the
# connector observes the normal finished lifecycle or is preempted
# before finishing.
if tracker is None or (
tracker.num_saved_tokens <= 0 and not tracker.has_pending_offload
):
return False, None
total_blocks = sum(len(g) for g in block_ids)
delay_free_blocks = total_blocks > 0
if delay_free_blocks:
logger.debug(
"Delaying free of %d blocks for request %s",
total_blocks,
request.request_id,
def _reference_save_blocks(self, meta: MooncakeStoreConnectorMetadata) -> None:
"""Take a GPU block reference for every store job this step emits.
The worker DMAs out of these blocks after the step that scheduled them,
so a reference keeps them out of the free queue even once the request
itself is freed, until every rank reports the job done.
"""
pool = self._gpu_block_pool
for req_meta in meta.requests:
if not req_meta.can_save:
continue
assert pool is not None, (
"GPU block pool must be bound before any store job is emitted"
)
return delay_free_blocks, None
req_meta.store_job_id = store_job_id = self._next_store_job_id
self._next_store_job_id += 1
block_ids: list[int] = []
if req_meta.partial_tail_offloads:
# A partial-tail CoW block is deliberately kept out of the
# request's block table, so it is absent from `block_ids` even
# though the worker DMAs out of it just as asynchronously.
# It leads the list, as in `pop_blocks_for_free`.
block_ids += [bid for _, bid, _ in req_meta.partial_tail_offloads]
# Every allocated block is referenced, not just the ones covering
# this job's token range: a rank resumes from its own last
# successful offset, which lags the scheduler's whenever a save was
# skipped or failed, so it may read anywhere below the range.
block_ids += [bid for group in req_meta.block_ids for bid in group]
if not block_ids:
continue
self._pinned_saves[store_job_id] = (block_ids, self._num_workers)
pool.touch([pool.blocks[bid] for bid in block_ids])
def update_connector_output(self, connector_output: KVConnectorOutput) -> None:
"""Drop the block references of store jobs every rank has finished."""
meta = connector_output.kv_connector_worker_meta
if not isinstance(meta, MooncakeStoreWorkerMetadata):
return
pool = self._gpu_block_pool
assert pool is not None
for store_job_id, count in meta.completed_saves.items():
pinned = self._pinned_saves.get(store_job_id)
if pinned is None:
# The job referenced no blocks, so nothing was recorded for it.
continue
block_ids, remaining = pinned
remaining -= count
if remaining > 0:
self._pinned_saves[store_job_id] = (block_ids, remaining)
continue
assert remaining == 0, (
f"store job {store_job_id} reported by too many ranks"
)
del self._pinned_saves[store_job_id]
# Tail-first, as elsewhere, so the shared prefix is evicted last.
pool.free_blocks(pool.blocks[bid] for bid in reversed(block_ids))
def has_pending_push_work(self) -> bool:
"""Keep the engine stepping while any store job still holds block refs.
Completions only reach the scheduler as worker metadata on a step, so an
engine that quiesced with jobs in flight would leave those references
held indefinitely. Nothing else keeps it alive now that a finishing
request no longer defers its own free.
"""
return bool(self._pinned_saves)
def reset_store(self) -> bool:
"""Trigger a global ``remove_all(force=True)`` on the Mooncake master.
@@ -17,7 +17,6 @@ import queue
import socket
import threading
import time
from collections import defaultdict
from collections.abc import Callable, Sequence
from concurrent.futures import Future, ThreadPoolExecutor
from dataclasses import dataclass
@@ -46,6 +45,7 @@ from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.data import (
ChunkedTokenDatabase,
KeyMetadata,
MooncakeStoreConnectorMetadata,
MooncakeStoreWorkerMetadata,
PoolKey,
ReqMeta,
)
@@ -506,7 +506,16 @@ class KVCacheStoreSendingThread(KVTransferThread):
self.group_put_steps = group_put_steps
self.coord = coord
self.kv_role = kv_role
self.stored_requests: defaultdict[str, int] = defaultdict(int)
# req_id -> ids of its store jobs that are still queued or running.
# Keying by store_job_id, which never repeats for the engine's lifetime,
# rather than counting jobs per request id makes the ledger immune to id
# reuse across preemption: a job left over from a retired generation is
# missing from the set its resumed generation builds, so it can no longer
# retire that generation, rewind its resume offset, or mark it skipped.
self.stored_requests: dict[str, set[int]] = {}
# store_job_id -> times this rank finished with it, drained every step
# so the scheduler can release the blocks it referenced for those jobs.
self._completed_saves: dict[int, int] = {}
self.enable_kv_event = enable_kv_event
# Caller always passes a non-None ReplicateConfig — see
# MooncakeStoreWorker.__init__ where store_replicate_config is built.
@@ -522,14 +531,20 @@ class KVCacheStoreSendingThread(KVTransferThread):
# batch resumes here, so pressure-skipped or failed ranges are retried.
self._saved_offset: dict[str, int] = {}
def add_stored_request(self, req_id: str):
def add_request(self, request: ReqMeta) -> None:
# Register before enqueueing so a job is never picked up unledgered.
assert request.store_job_id is not None
with self.done_task_lock:
self.stored_requests[req_id] += 1
self.stored_requests.setdefault(request.req_id, set()).add(
request.store_job_id
)
super().add_request(request)
def dec_stored_request(self, req_id: str):
def is_live_store_job(self, req_meta: ReqMeta) -> bool:
with self.done_task_lock:
if req_id in self.stored_requests:
self.stored_requests[req_id] -= 1
return req_meta.store_job_id in self.stored_requests.get(
req_meta.req_id, ()
)
def delete_finished_stored_request(self, req_id: str):
with self.done_task_lock:
@@ -538,21 +553,51 @@ class KVCacheStoreSendingThread(KVTransferThread):
self._skip_store_requests.discard(req_id)
self._saved_offset.pop(req_id, None)
def _record_saved(self, req_id: str, token_len: int) -> None:
# Guard on liveness so a concurrent finish/preempt pop isn't recreated.
def finish_store_job(self, req_meta: ReqMeta) -> None:
"""Retire a job from the ledger and report its blocks as no longer read.
Every path out of a job must reach this, skips and failures included: a
job that never reports leaves its blocks referenced for the rest of the
run. The discard is a no-op for a job whose generation already retired.
"""
store_job_id = req_meta.store_job_id
assert store_job_id is not None, (
"a queued store job always carries a store_job_id"
)
with self.done_task_lock:
if req_id in self.stored_requests:
self._saved_offset[req_id] = token_len
live = self.stored_requests.get(req_meta.req_id)
if live is not None:
live.discard(store_job_id)
self._completed_saves[store_job_id] = (
self._completed_saves.get(store_job_id, 0) + 1
)
def take_completed_saves(self) -> dict[int, int]:
with self.done_task_lock:
completed = self._completed_saves
self._completed_saves = {}
return completed
def _record_saved(self, req_meta: ReqMeta, token_len: int) -> None:
# Guard on job liveness so neither a concurrent finish/preempt pop nor a
# stale job's offset is written back over the live generation's.
with self.done_task_lock:
if req_meta.store_job_id in self.stored_requests.get(req_meta.req_id, ()):
self._saved_offset[req_meta.req_id] = token_len
def _should_skip_request(self, req_id: str) -> bool:
with self.done_task_lock:
return self._store_pressure_active and req_id in self._skip_store_requests
def _mark_request_skipped_for_pressure(self, req_id: str) -> bool:
def _mark_request_skipped_for_pressure(self, req_meta: ReqMeta) -> bool:
req_id = req_meta.req_id
with self.done_task_lock:
already_skipped = req_id in self._skip_store_requests
self._store_pressure_active = True
self._skip_store_requests.add(req_id)
# The pressure itself is global, but only a live job may sentence its
# own request to being skipped.
if req_meta.store_job_id in self.stored_requests.get(req_id, ()):
self._skip_store_requests.add(req_id)
return already_skipped
def _clear_store_pressure(self) -> bool:
@@ -727,7 +772,7 @@ class KVCacheStoreSendingThread(KVTransferThread):
failed_codes,
)
if MOONCAKE_NO_AVAILABLE_HANDLE in failed_codes:
self._mark_request_skipped_for_pressure(req_meta.req_id)
self._mark_request_skipped_for_pressure(req_meta)
return False
if self._clear_store_pressure():
@@ -738,22 +783,20 @@ class KVCacheStoreSendingThread(KVTransferThread):
return True
def _handle_request(self, req_meta: ReqMeta):
# Cache hits are always a multiple of ``lcm_block_size`` tokens, which
# is also ``store_mask``'s precondition.
lcm_block_size = self.coord.lcm_block_size
token_len = req_meta.token_len_chunk // lcm_block_size * lcm_block_size
block_ids_per_group = req_meta.block_ids
req_id = req_meta.req_id
current_event = req_meta.current_event
if req_id not in self.stored_requests:
self.request_queue.task_done()
return
# Decrement the in-flight counter and signal task_done() in `finally`
# so the scheduler can release the GPU blocks it pinned for this
# request (via `delay_free_blocks`) even when the store path raises.
# The single `finally` is the only way out, so the scheduler releases
# this job's GPU block references however the job ends.
try:
# Cache hits are always a multiple of ``lcm_block_size`` tokens,
# which is also ``store_mask``'s precondition.
lcm_block_size = self.coord.lcm_block_size
token_len = req_meta.token_len_chunk // lcm_block_size * lcm_block_size
block_ids_per_group = req_meta.block_ids
req_id = req_meta.req_id
current_event = req_meta.current_event
if not self.is_live_store_job(req_meta):
return
if self._should_skip_request(req_id):
logger.debug(
"Skipping Mooncake store for request %s while CPU/disk "
@@ -808,7 +851,7 @@ class KVCacheStoreSendingThread(KVTransferThread):
group_indices.append(g_idx)
if not keys:
self._record_saved(req_id, token_len)
self._record_saved(req_meta, token_len)
return
# Check which blocks already exist (dedup)
@@ -834,7 +877,7 @@ class KVCacheStoreSendingThread(KVTransferThread):
]
if not missing_indices:
self._record_saved(req_id, token_len)
self._record_saved(req_meta, token_len)
return
if len(missing_indices) != len(keys):
@@ -954,7 +997,7 @@ class KVCacheStoreSendingThread(KVTransferThread):
)
if (
MOONCAKE_NO_AVAILABLE_HANDLE in failed_codes
and not self._mark_request_skipped_for_pressure(req_id)
and not self._mark_request_skipped_for_pressure(req_meta)
):
logger.warning(
"Detected Mooncake CPU/disk offloading pressure "
@@ -964,7 +1007,7 @@ class KVCacheStoreSendingThread(KVTransferThread):
req_id,
)
else:
self._record_saved(req_id, token_len)
self._record_saved(req_meta, token_len)
if self._clear_store_pressure():
logger.info(
"Mooncake CPU/disk offloading pressure cleared "
@@ -984,7 +1027,7 @@ class KVCacheStoreSendingThread(KVTransferThread):
if self.enable_kv_event and stored_events:
self.update_kv_event(stored_events)
finally:
self.dec_stored_request(req_id)
self.finish_store_job(req_meta)
self.request_queue.task_done()
@@ -1670,16 +1713,15 @@ class MooncakeStoreWorker:
continue
request.current_event = current_event
assert self.kv_send_thread is not None
self.kv_send_thread.add_stored_request(request.req_id)
self.kv_send_thread.add_request(request)
# Check completion of previously queued transfers
done_sending = (
self._get_and_clear_finished_sending(finished_req_ids, meta)
if self.kv_role in ["kv_producer", "kv_both"]
else set()
)
if self.kv_role in ["kv_producer", "kv_both"]:
self._close_ended_store_requests(finished_req_ids, meta)
# Blocks read by a store job are released by the scheduler when the job
# reports back (see build_connector_worker_meta), so no request ever waits
# on a `finished_sending` signal to get its blocks back.
done_sending: set[str] = set()
done_recving: set[str] = set()
if self.load_async:
for recv_thread in self.kv_recv_threads:
@@ -1727,35 +1769,37 @@ class MooncakeStoreWorker:
self.kv_connector_stats = MooncakeStoreConnectorStats()
return kv_connector_stats
def _get_and_clear_finished_sending(
def _close_ended_store_requests(
self,
finished_req_ids: set[str],
meta: MooncakeStoreConnectorMetadata,
) -> set[str]:
) -> None:
"""Retire the ledger entries of requests that finished or were preempted.
An entry may only go once its jobs have drained, because they still read
the resume offset it owns; a request that comes back after preemption
then saves from the start rather than from where the last attempt got to.
"""
assert self.kv_send_thread is not None
finished_sending: set[str] = set()
for req_id in meta.preempted_req_ids:
self.kv_send_thread.delete_finished_stored_request(req_id)
for req_id in self.kv_send_thread.stored_requests.copy():
if (
self.kv_send_thread.stored_requests[req_id] == 0
and req_id in self.finished_store_req
):
self.finished_store_req.remove(req_id)
finished_sending.add(req_id)
self.kv_send_thread.delete_finished_stored_request(req_id)
for req_id in finished_req_ids:
req_remain_jobs = self.kv_send_thread.stored_requests.get(req_id)
if req_remain_jobs == 0:
finished_sending.add(req_id)
self.kv_send_thread.delete_finished_stored_request(req_id)
elif req_remain_jobs is not None:
for req_id in finished_req_ids | self.finished_store_req:
if self.kv_send_thread.stored_requests.get(req_id):
# Queued jobs still need the resume offset; retire on a later step.
self.finished_store_req.add(req_id)
else:
self.finished_store_req.discard(req_id)
self.kv_send_thread.delete_finished_stored_request(req_id)
return finished_sending
def build_connector_worker_meta(self) -> MooncakeStoreWorkerMetadata | None:
if self.kv_send_thread is None:
return None
completed_saves = self.kv_send_thread.take_completed_saves()
if not completed_saves:
return None
return MooncakeStoreWorkerMetadata(completed_saves=completed_saves)
def lookup(self, num_tokens: int, block_hashes: Sequence[BlockHash]) -> int:
"""Check how many prefix tokens exist in the store.