[KV Connector] Add decode offloading to Mooncake Store consumers (#52466)

Signed-off-by: z-zanez <zhouzh93@mail2.sysu.edu.cn>
Signed-off-by: Guanyi Chen <939416532@qq.com>
Co-authored-by: z-zanez <zhouzh93@mail2.sysu.edu.cn>
This commit is contained in:
Guanyi Chen
2026-08-19 17:40:35 -07:00
committed by GitHub
co-authored by z-zanez
parent d4f4d3f40f
commit bf2866f8bf
7 changed files with 518 additions and 72 deletions
@@ -141,12 +141,14 @@ class MooncakeStoreConnector(KVConnectorBase_V1, SupportsHMA):
assert vllm_config.kv_transfer_config is not None
assert kv_cache_config is not None, "kv_cache_config is required"
self.kv_role = vllm_config.kv_transfer_config.kv_role
extra_config = vllm_config.kv_transfer_config.kv_connector_extra_config
save_decode_cache = extra_config.get("save_decode_cache", False)
# Capacity-only: contributes its segment to the store pool but transfers
# no KV, so the KV-cache-shape invariants below cannot be reached.
self._capacity_only = self.kv_role == "kv_consumer" and not (
vllm_config.kv_transfer_config.kv_connector_extra_config.get(
"enable_lookup", True
)
self._capacity_only = (
self.kv_role == "kv_consumer"
and not extra_config.get("enable_lookup", True)
and not save_decode_cache
)
if not self._capacity_only:
self._validate_kv_cache_config(vllm_config, kv_cache_config)
@@ -362,10 +362,11 @@ class ReqMeta:
can_save: bool | None = None
load_spec: LoadSpec | None = None
is_last_chunk: bool | None = None
current_event: torch.cuda.Event | None = None
token_ids: list[int] | None = None
# Absolute request offset represented by token_ids[0].
token_ids_start: int = 0
num_prompt_tokens: int | None = None
# Identifies this store job for the engine's lifetime. A request id cannot
# serve that purpose: it is reused once a preempted request resumes, so it
@@ -384,14 +385,14 @@ class ReqMeta:
load_spec: LoadSpec | None = None,
skip_save: bool | None = False,
block_hashes: list[BlockHash] | None = None,
is_last_chunk: bool | None = None,
) -> "ReqMeta | None":
"""Create ReqMeta from a RequestTracker."""
if block_hashes is None:
block_hashes = []
input_token_len = tracker.token_len
chunk_boundary = cdiv(tracker.num_saved_tokens + 1, block_size) * block_size
token_ids_start = tracker.num_saved_tokens
chunk_boundary = cdiv(token_ids_start + 1, block_size) * block_size
num_tokens_to_save = input_token_len // block_size * block_size
skip_save = skip_save or num_tokens_to_save < chunk_boundary
@@ -408,8 +409,10 @@ class ReqMeta:
tracker.num_saved_tokens = num_tokens_to_save
token_ids = None
if tracker.token_ids:
token_ids = tracker.token_ids
if tracker.token_ids and not skip_save:
# Scheduler tracking continues while this job is handled by an
# asynchronous worker, so metadata must own a stable snapshot.
token_ids = tracker.token_ids[token_ids_start:num_tokens_to_save]
if load_spec is not None and load_spec.can_load:
logger.debug(
@@ -433,8 +436,8 @@ class ReqMeta:
can_save=not skip_save,
load_spec=load_spec,
block_hashes=block_hashes,
is_last_chunk=is_last_chunk,
token_ids=token_ids,
token_ids_start=token_ids_start,
num_prompt_tokens=tracker.prefill_end_tokens,
)
@@ -63,6 +63,11 @@ class MooncakeStoreScheduler:
self.lookup_async = kvc_extra_config.get("lookup_async", False)
# Skips lookup CPU cost on instances that never load KV from the store.
self.enable_lookup = kvc_extra_config.get("enable_lookup", True)
self.save_decode_cache = kvc_extra_config.get("save_decode_cache", False)
kv_event_config = vllm_config.kv_events_config
self.enable_kv_events = bool(
kv_event_config and kv_event_config.enable_kv_cache_events
)
self.client = LookupKeyClient(vllm_config)
# Align with the engine's own scheduler_block_size and hash_block_size.
@@ -182,7 +187,8 @@ class MooncakeStoreScheduler:
self, scheduler_output: SchedulerOutput
) -> KVConnectorMetadata:
"""Build connector metadata for this scheduler step."""
force_skip_save = self.kv_role == "kv_consumer"
is_consumer = self.kv_role == "kv_consumer"
can_process_cached = not is_consumer or self.save_decode_cache
for finished_req_id in scheduler_output.finished_req_ids:
self.client.discard(finished_req_id)
@@ -227,37 +233,38 @@ class MooncakeStoreScheduler:
token_len=num_tokens_to_compute,
allocated_block_ids=unfolded_block_ids,
num_saved_tokens=0,
token_ids=prefill_tokens[:num_tokens_to_compute],
token_ids=(
prefill_tokens[:num_tokens_to_compute]
if self.enable_kv_events
else None
),
prefill_end_tokens=len(prefill_tokens),
)
self._request_trackers[request.req_id] = request_tracker
last_chunk_tokens_num = (
len(prefill_tokens) // self._block_size * self._block_size
)
req_meta = ReqMeta.from_request_tracker(
request_tracker,
self._block_size,
load_spec=load_spec,
skip_save=force_skip_save,
# A consumer may write decode KV without becoming a prefill
# producer. Loads are still carried by the same metadata.
skip_save=is_consumer,
block_hashes=request_real.block_hashes,
is_last_chunk=(request_tracker.token_len >= last_chunk_tokens_num),
)
if req_meta is not None:
meta.add_request(req_meta)
# Handle cached (running, or MRV1 resumed-from-preemption) requests
cached_reqs = scheduler_output.scheduled_cached_reqs
if not force_skip_save:
if can_process_cached:
for i, req_id in enumerate(cached_reqs.req_ids):
new_block_ids = cached_reqs.new_block_ids[i]
if not new_block_ids:
continue
req_meta = None
if req_id in cached_reqs.resumed_req_ids:
# Resumed after preemption
if not new_block_ids:
continue
if isinstance(new_block_ids, tuple):
new_block_ids = tuple(b.copy() for b in new_block_ids)
else:
@@ -277,27 +284,33 @@ class MooncakeStoreScheduler:
token_len=num_tokens_to_compute,
allocated_block_ids=new_block_ids,
num_saved_tokens=0,
token_ids=prefill_tokens[:num_tokens_to_compute].copy(),
token_ids=(
prefill_tokens[:num_tokens_to_compute]
if self.enable_kv_events
else None
),
prefill_end_tokens=len(prefill_tokens),
)
self._request_trackers[req_id] = request_tracker
last_chunk_tokens_num = (
len(prefill_tokens) // self._block_size * self._block_size
)
req_meta = ReqMeta.from_request_tracker(
request_tracker,
self._block_size,
load_spec=load_spec,
skip_save=force_skip_save,
skip_save=is_consumer,
block_hashes=request_real.block_hashes,
is_last_chunk=(
request_tracker.token_len >= last_chunk_tokens_num
),
)
else:
# Decode/chunked request
request_tracker = self._request_trackers[req_id]
num_computed_token = cached_reqs.num_computed_tokens[i]
# Use the tracker's snapshot of the prefill range so resumed
# requests keep saving past the original prompt boundary.
prefill_end = request_tracker.prefill_end_tokens
is_decode = num_computed_token >= prefill_end
if is_decode and not self.save_decode_cache:
continue
num_new_tokens = scheduler_output.num_scheduled_tokens[req_id]
req_tuple = self._unfinished_requests.get(req_id)
if req_tuple:
@@ -307,30 +320,26 @@ class MooncakeStoreScheduler:
num_current_tokens : num_current_tokens + num_new_tokens
]
request_tracker.token_len += len(new_token_ids)
if request_tracker.token_ids is not None:
request_tracker.token_ids.extend(new_token_ids)
else:
raise ValueError(
f"Request {req_id} is not in _unfinished_requests"
)
num_computed_token = cached_reqs.num_computed_tokens[i]
# Use the tracker's snapshot of the prefill range so resumed
# requests keep saving past the original prompt boundary.
prefill_end = request_tracker.prefill_end_tokens
if num_computed_token >= prefill_end:
# A block is usually allocated before the step that fills
# it, so reaching a save boundary does not imply that this
# step has new block ids.
if new_block_ids:
request_tracker.update(new_block_ids)
if is_consumer and not is_decode:
continue
request_tracker.update(new_block_ids)
last_chunk_tokens_num = (
prefill_end // self._block_size * self._block_size
)
req_meta = ReqMeta.from_request_tracker(
request_tracker,
self._block_size,
load_spec=None,
skip_save=force_skip_save,
skip_save=False,
block_hashes=unfinished_req.block_hashes,
is_last_chunk=(
request_tracker.token_len >= last_chunk_tokens_num
),
)
if req_meta is not None:
@@ -370,7 +379,7 @@ class MooncakeStoreScheduler:
# emit an offload-only ReqMeta (token_len_chunk=0 skips the normal
# save; can_save=True takes the normal enqueue path).
step_partial_tails = getattr(scheduler_output, "partial_tail_offloads", None)
if step_partial_tails and not force_skip_save:
if step_partial_tails and not is_consumer:
pending = dict(step_partial_tails)
for req_meta in meta.requests:
if req_meta.can_save:
@@ -530,6 +530,9 @@ class KVCacheStoreSendingThread(KVTransferThread):
# Per-request high-water mark of tokens actually persisted; the next
# batch resumes here, so pressure-skipped or failed ranges are retried.
self._saved_offset: dict[str, int] = {}
# Retained only after a failed store so retry events can recover the
# token suffix without full snapshots on the normal path.
self._retry_token_ids: dict[str, tuple[int, list[int]]] = {}
def add_request(self, request: ReqMeta) -> None:
# Register before enqueueing so a job is never picked up unledgered.
@@ -552,6 +555,7 @@ class KVCacheStoreSendingThread(KVTransferThread):
del self.stored_requests[req_id]
self._skip_store_requests.discard(req_id)
self._saved_offset.pop(req_id, None)
self._retry_token_ids.pop(req_id, None)
def finish_store_job(self, req_meta: ReqMeta) -> None:
"""Retire a job from the ledger and report its blocks as no longer read.
@@ -585,6 +589,36 @@ class KVCacheStoreSendingThread(KVTransferThread):
if req_meta.store_job_id in self.stored_requests.get(req_meta.req_id, ()):
self._saved_offset[req_meta.req_id] = token_len
def _get_retry_token_ids(self, req_meta: ReqMeta) -> tuple[int, list[int]] | None:
"""Return retry state only if this store job is still live."""
with self.done_task_lock:
if req_meta.store_job_id not in self.stored_requests.get(
req_meta.req_id, ()
):
return None
return self._retry_token_ids.get(req_meta.req_id)
def _update_retry_token_ids(
self,
req_meta: ReqMeta,
save_completed: bool,
token_ids_start: int,
event_token_ids: list[int] | None,
) -> None:
"""Update retry state without letting a stale job touch a reused ID."""
with self.done_task_lock:
if req_meta.store_job_id not in self.stored_requests.get(
req_meta.req_id, ()
):
return
if save_completed:
self._retry_token_ids.pop(req_meta.req_id, None)
elif event_token_ids is not None:
self._retry_token_ids[req_meta.req_id] = (
token_ids_start,
event_token_ids,
)
def _should_skip_request(self, req_id: str) -> bool:
with self.done_task_lock:
return self._store_pressure_active and req_id in self._skip_store_requests
@@ -785,18 +819,30 @@ class KVCacheStoreSendingThread(KVTransferThread):
def _handle_request(self, req_meta: ReqMeta):
# The single `finally` is the only way out, so the scheduler releases
# this job's GPU block references however the job ends.
save_completed = False
token_len = 0
req_id = req_meta.req_id
event_token_ids = req_meta.token_ids
token_ids_start = req_meta.token_ids_start
try:
# Cache hits are always a multiple of ``lcm_block_size`` tokens,
# which is also ``store_mask``'s precondition.
lcm_block_size = self.coord.lcm_block_size
token_len = req_meta.token_len_chunk // lcm_block_size * lcm_block_size
block_ids_per_group = req_meta.block_ids
req_id = req_meta.req_id
current_event = req_meta.current_event
if not self.is_live_store_job(req_meta):
return
if self.enable_kv_event:
retry_token_ids = self._get_retry_token_ids(req_meta)
if retry_token_ids is not None and event_token_ids is not None:
retry_start, retry_ids = retry_token_ids
if retry_start + len(retry_ids) == token_ids_start:
event_token_ids = retry_ids + event_token_ids
token_ids_start = retry_start
if self._should_skip_request(req_id):
logger.debug(
"Skipping Mooncake store for request %s while CPU/disk "
@@ -852,6 +898,7 @@ class KVCacheStoreSendingThread(KVTransferThread):
if not keys:
self._record_saved(req_meta, token_len)
save_completed = True
return
# Check which blocks already exist (dedup)
@@ -878,6 +925,7 @@ class KVCacheStoreSendingThread(KVTransferThread):
if not missing_indices:
self._record_saved(req_meta, token_len)
save_completed = True
return
if len(missing_indices) != len(keys):
@@ -927,12 +975,11 @@ class KVCacheStoreSendingThread(KVTransferThread):
addrs.extend(group_addrs)
sizes.extend(group_sizes)
# parent_block_hash chains live within a group, not across.
if self.enable_kv_event:
prev_key_per_group: dict[int, Any] = {}
new_block_hashes = [
maybe_convert_block_hash(bh) for bh in kv_event_block_hashes
]
token_ids_end = token_ids_start + len(event_token_ids or ())
for idx, (s, e, g_idx) in enumerate(
zip(starts, ends, group_indices, strict=True)
@@ -940,13 +987,24 @@ class KVCacheStoreSendingThread(KVTransferThread):
db = self.token_databases[g_idx]
if self.enable_kv_event:
token_ids = (
req_meta.token_ids[s:e]
if req_meta.token_ids is not None
else None
event_token_ids[s - token_ids_start : e - token_ids_start]
if event_token_ids is not None
and token_ids_start <= s
and e <= token_ids_end
else []
)
stored_event = BlockStored(
block_hashes=[new_block_hashes[idx]],
parent_block_hash=prev_key_per_group.get(g_idx),
# Derive the direct predecessor from the unfiltered
# request chain. Adjacent PUTs need not be adjacent in
# that chain after Store dedup, masks, or TP striding.
parent_block_hash=(
maybe_convert_block_hash(
req_meta.block_hashes[s // db.hash_block_size - 1]
)
if s > 0
else None
),
token_ids=token_ids,
block_size=db.block_size,
lora_id=None,
@@ -955,7 +1013,6 @@ class KVCacheStoreSendingThread(KVTransferThread):
group_idx=g_idx,
)
stored_events.append(stored_event)
prev_key_per_group[g_idx] = new_block_hashes[idx]
if current_event is not None:
current_event.synchronize()
@@ -984,6 +1041,13 @@ class KVCacheStoreSendingThread(KVTransferThread):
)
if failed:
failed_codes = set(res[i] for i in failed)
if self.enable_kv_event:
failed_indices = set(failed)
stored_events = [
event
for i, event in enumerate(stored_events)
if i not in failed_indices
]
logger.warning(
"batch_put failed: %d/%d keys failed "
"(codes=%s, batch_bytes=%d, num_keys=%d), "
@@ -1008,6 +1072,7 @@ class KVCacheStoreSendingThread(KVTransferThread):
)
else:
self._record_saved(req_meta, token_len)
save_completed = True
if self._clear_store_pressure():
logger.info(
"Mooncake CPU/disk offloading pressure cleared "
@@ -1023,10 +1088,18 @@ class KVCacheStoreSendingThread(KVTransferThread):
num_failed_keys=len(keys),
)
logger.error("Failed to put key %s, error: %s", keys, e)
stored_events.clear()
if self.enable_kv_event and stored_events:
self.update_kv_event(stored_events)
finally:
if self.enable_kv_event and token_len:
self._update_retry_token_ids(
req_meta,
save_completed,
token_ids_start,
event_token_ids,
)
self.finish_store_job(req_meta)
self.request_queue.task_done()
@@ -1274,15 +1347,19 @@ class MooncakeStoreWorker:
self.dcp_rank = get_dcp_group().rank_in_group if self.dcp_size > 1 else 0
assert vllm_config.kv_transfer_config is not None
self.kv_role = vllm_config.kv_transfer_config.kv_role
self.load_async = vllm_config.kv_transfer_config.kv_connector_extra_config.get(
"load_async", True
kv_role = vllm_config.kv_transfer_config.kv_role
assert kv_role is not None
self.kv_role = kv_role
extra_config = vllm_config.kv_transfer_config.kv_connector_extra_config
self.can_put = self.kv_role in ("kv_producer", "kv_both") or (
extra_config.get("save_decode_cache", False)
)
self.load_async = extra_config.get("load_async", True)
# Mirrors MooncakeStoreConnector._capacity_only.
self._capacity_only = self.kv_role == "kv_consumer" and not (
vllm_config.kv_transfer_config.kv_connector_extra_config.get(
"enable_lookup", True
)
self._capacity_only = (
self.kv_role == "kv_consumer"
and not extra_config.get("enable_lookup", True)
and not self.can_put
)
self.cache_config = vllm_config.cache_config
self.block_size, self.hash_block_size = resolve_kv_cache_block_sizes(
@@ -1294,11 +1371,6 @@ class MooncakeStoreWorker:
# Initialize MooncakeDistributedStore with its own TransferEngine
store_config = MooncakeStoreConfig.load_from_config()
extra_config = (
vllm_config.kv_transfer_config.kv_connector_extra_config
if vllm_config.kv_transfer_config
else {}
)
self.store = MooncakeDistributedStore()
local_ip = get_ip()
local_hostname = rdma_utils.get_requester_local_hostname(local_ip)
@@ -1617,7 +1689,7 @@ class MooncakeStoreWorker:
db.set_block_len(block_lens)
# Start transfer threads
if self.kv_role in ["kv_producer", "kv_both"]:
if self.can_put:
ready_event_sending = threading.Event()
self.kv_send_thread = KVCacheStoreSendingThread(
self.store,
@@ -1700,7 +1772,7 @@ class MooncakeStoreWorker:
assert self.load_async, "load_async must be True for better performance."
# Issue stores with CUDA event synchronization.
if self.kv_role in ["kv_producer", "kv_both"]:
if self.can_put:
current_event = None
for request in meta.requests:
if request.can_save:
@@ -1714,8 +1786,6 @@ class MooncakeStoreWorker:
request.current_event = current_event
assert self.kv_send_thread is not None
self.kv_send_thread.add_request(request)
if self.kv_role in ["kv_producer", "kv_both"]:
self._close_ended_store_requests(finished_req_ids, meta)
# Blocks read by a store job are released by the scheduler when the job