[Bugfix][Mooncake] Reference GPU blocks for in-flight store jobs and key the store ledger by store_job_id (#52372)
Signed-off-by: Guanyi Chen <939416532@qq.com>
This commit is contained in:
@@ -26,6 +26,7 @@ from vllm.distributed.kv_transfer.kv_connector.v1.base import (
|
||||
KVConnectorBase_V1,
|
||||
KVConnectorMetadata,
|
||||
KVConnectorRole,
|
||||
KVConnectorWorkerMetadata,
|
||||
SupportsHMA,
|
||||
)
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.metrics import (
|
||||
@@ -37,6 +38,7 @@ from vllm.distributed.kv_transfer.kv_connector.v1.metrics import (
|
||||
from vllm.forward_context import ForwardContext
|
||||
from vllm.logger import init_logger
|
||||
from vllm.v1.attention.backend import AttentionMetadata
|
||||
from vllm.v1.core.block_pool import BlockPool
|
||||
from vllm.v1.core.kv_cache_manager import KVCacheBlocks
|
||||
from vllm.v1.core.sched.output import SchedulerOutput
|
||||
from vllm.v1.kv_cache_interface import KVCacheConfig
|
||||
@@ -201,6 +203,14 @@ class MooncakeStoreConnector(KVConnectorBase_V1, SupportsHMA):
|
||||
request, blocks, num_external_tokens
|
||||
)
|
||||
|
||||
def bind_gpu_block_pool(self, gpu_block_pool: BlockPool) -> None:
|
||||
assert self.connector_scheduler is not None
|
||||
self.connector_scheduler.bind_gpu_block_pool(gpu_block_pool)
|
||||
|
||||
def has_pending_push_work(self) -> bool:
|
||||
assert self.connector_scheduler is not None
|
||||
return self.connector_scheduler.has_pending_push_work()
|
||||
|
||||
def build_connector_meta(
|
||||
self,
|
||||
scheduler_output: SchedulerOutput,
|
||||
@@ -208,6 +218,10 @@ class MooncakeStoreConnector(KVConnectorBase_V1, SupportsHMA):
|
||||
assert self.connector_scheduler is not None
|
||||
return self.connector_scheduler.build_connector_meta(scheduler_output)
|
||||
|
||||
def build_connector_worker_meta(self) -> KVConnectorWorkerMetadata | None:
|
||||
assert self.connector_worker is not None
|
||||
return self.connector_worker.build_connector_worker_meta()
|
||||
|
||||
def request_finished(
|
||||
self,
|
||||
request: Request,
|
||||
@@ -220,8 +234,9 @@ class MooncakeStoreConnector(KVConnectorBase_V1, SupportsHMA):
|
||||
request: Request,
|
||||
block_ids: tuple[list[int], ...],
|
||||
) -> tuple[bool, dict[str, Any] | None]:
|
||||
assert self.connector_scheduler is not None
|
||||
return self.connector_scheduler.request_finished(request, block_ids)
|
||||
# An in-flight store job holds its own reference on the blocks it reads,
|
||||
# so a finishing request never has to defer freeing them.
|
||||
return False, None
|
||||
|
||||
def reset_cache(self) -> bool | None:
|
||||
"""Reset the external Mooncake store on prefix-cache reset.
|
||||
@@ -241,6 +256,9 @@ class MooncakeStoreConnector(KVConnectorBase_V1, SupportsHMA):
|
||||
return None
|
||||
|
||||
def update_connector_output(self, connector_output: KVConnectorOutput):
|
||||
assert self.connector_scheduler is not None
|
||||
self.connector_scheduler.update_connector_output(connector_output)
|
||||
|
||||
kv_cache_events = connector_output.kv_cache_events
|
||||
if not kv_cache_events or not isinstance(
|
||||
kv_cache_events, MooncakeStoreKVEvents
|
||||
|
||||
@@ -6,7 +6,7 @@
|
||||
"""Data classes for MooncakeStoreConnector."""
|
||||
|
||||
from collections.abc import Iterable, Sequence
|
||||
from dataclasses import dataclass
|
||||
from dataclasses import dataclass, field
|
||||
from typing import cast
|
||||
|
||||
import numpy as np
|
||||
@@ -14,6 +14,7 @@ import torch
|
||||
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.base import (
|
||||
KVConnectorMetadata,
|
||||
KVConnectorWorkerMetadata,
|
||||
)
|
||||
from vllm.logger import init_logger
|
||||
from vllm.utils.math_utils import cdiv
|
||||
@@ -366,6 +367,10 @@ class ReqMeta:
|
||||
|
||||
token_ids: list[int] | None = None
|
||||
num_prompt_tokens: int | None = None
|
||||
# Identifies this store job for the engine's lifetime. A request id cannot
|
||||
# serve that purpose: it is reused once a preempted request resumes, so it
|
||||
# would release the wrong job's blocks.
|
||||
store_job_id: int | None = None
|
||||
# Core-provided per-mamba-group
|
||||
# (group_id, cow_block_id, boundary_tokens) for this request's partial tail.
|
||||
# Present only on the producer's CoW step; drives the connector's offload
|
||||
@@ -434,6 +439,23 @@ class ReqMeta:
|
||||
)
|
||||
|
||||
|
||||
@dataclass
|
||||
class MooncakeStoreWorkerMetadata(KVConnectorWorkerMetadata):
|
||||
"""Maps ``ReqMeta.store_job_id`` to the number of ranks done with that job."""
|
||||
|
||||
completed_saves: dict[int, int] = field(default_factory=dict)
|
||||
|
||||
def aggregate(
|
||||
self, other: "KVConnectorWorkerMetadata"
|
||||
) -> "MooncakeStoreWorkerMetadata":
|
||||
assert isinstance(other, MooncakeStoreWorkerMetadata)
|
||||
for store_job_id, count in other.completed_saves.items():
|
||||
self.completed_saves[store_job_id] = (
|
||||
self.completed_saves.get(store_job_id, 0) + count
|
||||
)
|
||||
return self
|
||||
|
||||
|
||||
class MooncakeStoreConnectorMetadata(KVConnectorMetadata):
|
||||
"""Metadata passed from scheduler to worker."""
|
||||
|
||||
|
||||
@@ -5,8 +5,6 @@
|
||||
# (vllm_ascend/distributed/kv_transfer/kv_pool/ascend_store/).
|
||||
"""Scheduler-side logic for MooncakeStoreConnector."""
|
||||
|
||||
from typing import Any
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.base import (
|
||||
KVConnectorMetadata,
|
||||
@@ -17,6 +15,7 @@ from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.coordinator imp
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.data import ( # noqa: E501
|
||||
LoadSpec,
|
||||
MooncakeStoreConnectorMetadata,
|
||||
MooncakeStoreWorkerMetadata,
|
||||
ReqMeta,
|
||||
RequestTracker,
|
||||
)
|
||||
@@ -24,10 +23,12 @@ from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.worker import (
|
||||
LookupKeyClient,
|
||||
)
|
||||
from vllm.logger import init_logger
|
||||
from vllm.v1.core.block_pool import BlockPool
|
||||
from vllm.v1.core.kv_cache_manager import KVCacheBlocks
|
||||
from vllm.v1.core.kv_cache_utils import resolve_kv_cache_block_sizes
|
||||
from vllm.v1.core.sched.output import NewRequestData, SchedulerOutput
|
||||
from vllm.v1.kv_cache_interface import KVCacheConfig
|
||||
from vllm.v1.outputs import KVConnectorOutput
|
||||
from vllm.v1.request import Request
|
||||
|
||||
logger = init_logger(__name__)
|
||||
@@ -78,6 +79,15 @@ class MooncakeStoreScheduler:
|
||||
self._unfinished_requests: dict[str, tuple[Request, tuple[list[int], ...]]] = {}
|
||||
self._unfinished_request_ids: set[str] = set()
|
||||
|
||||
self._gpu_block_pool: BlockPool | None = None
|
||||
self._num_workers = vllm_config.parallel_config.world_size
|
||||
self._next_store_job_id = 0
|
||||
# store_job_id -> (referenced block ids, ranks yet to report completion)
|
||||
self._pinned_saves: dict[int, tuple[list[int], int]] = {}
|
||||
|
||||
def bind_gpu_block_pool(self, gpu_block_pool: BlockPool) -> None:
|
||||
self._gpu_block_pool = gpu_block_pool
|
||||
|
||||
def get_num_new_matched_tokens(
|
||||
self,
|
||||
request: Request,
|
||||
@@ -392,33 +402,75 @@ class MooncakeStoreScheduler:
|
||||
)
|
||||
)
|
||||
|
||||
self._reference_save_blocks(meta)
|
||||
return meta
|
||||
|
||||
def request_finished(
|
||||
self,
|
||||
request: Request,
|
||||
block_ids: tuple[list[int], ...],
|
||||
) -> tuple[bool, dict[str, Any] | None]:
|
||||
"""Determine whether to delay freeing blocks for async save."""
|
||||
if self.kv_role == "kv_consumer":
|
||||
return False, None
|
||||
tracker = self._request_trackers.get(request.request_id)
|
||||
# Missing tracker can happen when the request is aborted before the
|
||||
# connector observes the normal finished lifecycle or is preempted
|
||||
# before finishing.
|
||||
if tracker is None or (
|
||||
tracker.num_saved_tokens <= 0 and not tracker.has_pending_offload
|
||||
):
|
||||
return False, None
|
||||
total_blocks = sum(len(g) for g in block_ids)
|
||||
delay_free_blocks = total_blocks > 0
|
||||
if delay_free_blocks:
|
||||
logger.debug(
|
||||
"Delaying free of %d blocks for request %s",
|
||||
total_blocks,
|
||||
request.request_id,
|
||||
def _reference_save_blocks(self, meta: MooncakeStoreConnectorMetadata) -> None:
|
||||
"""Take a GPU block reference for every store job this step emits.
|
||||
|
||||
The worker DMAs out of these blocks after the step that scheduled them,
|
||||
so a reference keeps them out of the free queue even once the request
|
||||
itself is freed, until every rank reports the job done.
|
||||
"""
|
||||
pool = self._gpu_block_pool
|
||||
for req_meta in meta.requests:
|
||||
if not req_meta.can_save:
|
||||
continue
|
||||
assert pool is not None, (
|
||||
"GPU block pool must be bound before any store job is emitted"
|
||||
)
|
||||
return delay_free_blocks, None
|
||||
req_meta.store_job_id = store_job_id = self._next_store_job_id
|
||||
self._next_store_job_id += 1
|
||||
block_ids: list[int] = []
|
||||
if req_meta.partial_tail_offloads:
|
||||
# A partial-tail CoW block is deliberately kept out of the
|
||||
# request's block table, so it is absent from `block_ids` even
|
||||
# though the worker DMAs out of it just as asynchronously.
|
||||
# It leads the list, as in `pop_blocks_for_free`.
|
||||
block_ids += [bid for _, bid, _ in req_meta.partial_tail_offloads]
|
||||
# Every allocated block is referenced, not just the ones covering
|
||||
# this job's token range: a rank resumes from its own last
|
||||
# successful offset, which lags the scheduler's whenever a save was
|
||||
# skipped or failed, so it may read anywhere below the range.
|
||||
block_ids += [bid for group in req_meta.block_ids for bid in group]
|
||||
if not block_ids:
|
||||
continue
|
||||
self._pinned_saves[store_job_id] = (block_ids, self._num_workers)
|
||||
pool.touch([pool.blocks[bid] for bid in block_ids])
|
||||
|
||||
def update_connector_output(self, connector_output: KVConnectorOutput) -> None:
|
||||
"""Drop the block references of store jobs every rank has finished."""
|
||||
meta = connector_output.kv_connector_worker_meta
|
||||
if not isinstance(meta, MooncakeStoreWorkerMetadata):
|
||||
return
|
||||
pool = self._gpu_block_pool
|
||||
assert pool is not None
|
||||
for store_job_id, count in meta.completed_saves.items():
|
||||
pinned = self._pinned_saves.get(store_job_id)
|
||||
if pinned is None:
|
||||
# The job referenced no blocks, so nothing was recorded for it.
|
||||
continue
|
||||
block_ids, remaining = pinned
|
||||
remaining -= count
|
||||
if remaining > 0:
|
||||
self._pinned_saves[store_job_id] = (block_ids, remaining)
|
||||
continue
|
||||
assert remaining == 0, (
|
||||
f"store job {store_job_id} reported by too many ranks"
|
||||
)
|
||||
del self._pinned_saves[store_job_id]
|
||||
# Tail-first, as elsewhere, so the shared prefix is evicted last.
|
||||
pool.free_blocks(pool.blocks[bid] for bid in reversed(block_ids))
|
||||
|
||||
def has_pending_push_work(self) -> bool:
|
||||
"""Keep the engine stepping while any store job still holds block refs.
|
||||
|
||||
Completions only reach the scheduler as worker metadata on a step, so an
|
||||
engine that quiesced with jobs in flight would leave those references
|
||||
held indefinitely. Nothing else keeps it alive now that a finishing
|
||||
request no longer defers its own free.
|
||||
"""
|
||||
return bool(self._pinned_saves)
|
||||
|
||||
def reset_store(self) -> bool:
|
||||
"""Trigger a global ``remove_all(force=True)`` on the Mooncake master.
|
||||
|
||||
@@ -17,7 +17,6 @@ import queue
|
||||
import socket
|
||||
import threading
|
||||
import time
|
||||
from collections import defaultdict
|
||||
from collections.abc import Callable, Sequence
|
||||
from concurrent.futures import Future, ThreadPoolExecutor
|
||||
from dataclasses import dataclass
|
||||
@@ -46,6 +45,7 @@ from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.data import (
|
||||
ChunkedTokenDatabase,
|
||||
KeyMetadata,
|
||||
MooncakeStoreConnectorMetadata,
|
||||
MooncakeStoreWorkerMetadata,
|
||||
PoolKey,
|
||||
ReqMeta,
|
||||
)
|
||||
@@ -506,7 +506,16 @@ class KVCacheStoreSendingThread(KVTransferThread):
|
||||
self.group_put_steps = group_put_steps
|
||||
self.coord = coord
|
||||
self.kv_role = kv_role
|
||||
self.stored_requests: defaultdict[str, int] = defaultdict(int)
|
||||
# req_id -> ids of its store jobs that are still queued or running.
|
||||
# Keying by store_job_id, which never repeats for the engine's lifetime,
|
||||
# rather than counting jobs per request id makes the ledger immune to id
|
||||
# reuse across preemption: a job left over from a retired generation is
|
||||
# missing from the set its resumed generation builds, so it can no longer
|
||||
# retire that generation, rewind its resume offset, or mark it skipped.
|
||||
self.stored_requests: dict[str, set[int]] = {}
|
||||
# store_job_id -> times this rank finished with it, drained every step
|
||||
# so the scheduler can release the blocks it referenced for those jobs.
|
||||
self._completed_saves: dict[int, int] = {}
|
||||
self.enable_kv_event = enable_kv_event
|
||||
# Caller always passes a non-None ReplicateConfig — see
|
||||
# MooncakeStoreWorker.__init__ where store_replicate_config is built.
|
||||
@@ -522,14 +531,20 @@ class KVCacheStoreSendingThread(KVTransferThread):
|
||||
# batch resumes here, so pressure-skipped or failed ranges are retried.
|
||||
self._saved_offset: dict[str, int] = {}
|
||||
|
||||
def add_stored_request(self, req_id: str):
|
||||
def add_request(self, request: ReqMeta) -> None:
|
||||
# Register before enqueueing so a job is never picked up unledgered.
|
||||
assert request.store_job_id is not None
|
||||
with self.done_task_lock:
|
||||
self.stored_requests[req_id] += 1
|
||||
self.stored_requests.setdefault(request.req_id, set()).add(
|
||||
request.store_job_id
|
||||
)
|
||||
super().add_request(request)
|
||||
|
||||
def dec_stored_request(self, req_id: str):
|
||||
def is_live_store_job(self, req_meta: ReqMeta) -> bool:
|
||||
with self.done_task_lock:
|
||||
if req_id in self.stored_requests:
|
||||
self.stored_requests[req_id] -= 1
|
||||
return req_meta.store_job_id in self.stored_requests.get(
|
||||
req_meta.req_id, ()
|
||||
)
|
||||
|
||||
def delete_finished_stored_request(self, req_id: str):
|
||||
with self.done_task_lock:
|
||||
@@ -538,21 +553,51 @@ class KVCacheStoreSendingThread(KVTransferThread):
|
||||
self._skip_store_requests.discard(req_id)
|
||||
self._saved_offset.pop(req_id, None)
|
||||
|
||||
def _record_saved(self, req_id: str, token_len: int) -> None:
|
||||
# Guard on liveness so a concurrent finish/preempt pop isn't recreated.
|
||||
def finish_store_job(self, req_meta: ReqMeta) -> None:
|
||||
"""Retire a job from the ledger and report its blocks as no longer read.
|
||||
|
||||
Every path out of a job must reach this, skips and failures included: a
|
||||
job that never reports leaves its blocks referenced for the rest of the
|
||||
run. The discard is a no-op for a job whose generation already retired.
|
||||
"""
|
||||
store_job_id = req_meta.store_job_id
|
||||
assert store_job_id is not None, (
|
||||
"a queued store job always carries a store_job_id"
|
||||
)
|
||||
with self.done_task_lock:
|
||||
if req_id in self.stored_requests:
|
||||
self._saved_offset[req_id] = token_len
|
||||
live = self.stored_requests.get(req_meta.req_id)
|
||||
if live is not None:
|
||||
live.discard(store_job_id)
|
||||
self._completed_saves[store_job_id] = (
|
||||
self._completed_saves.get(store_job_id, 0) + 1
|
||||
)
|
||||
|
||||
def take_completed_saves(self) -> dict[int, int]:
|
||||
with self.done_task_lock:
|
||||
completed = self._completed_saves
|
||||
self._completed_saves = {}
|
||||
return completed
|
||||
|
||||
def _record_saved(self, req_meta: ReqMeta, token_len: int) -> None:
|
||||
# Guard on job liveness so neither a concurrent finish/preempt pop nor a
|
||||
# stale job's offset is written back over the live generation's.
|
||||
with self.done_task_lock:
|
||||
if req_meta.store_job_id in self.stored_requests.get(req_meta.req_id, ()):
|
||||
self._saved_offset[req_meta.req_id] = token_len
|
||||
|
||||
def _should_skip_request(self, req_id: str) -> bool:
|
||||
with self.done_task_lock:
|
||||
return self._store_pressure_active and req_id in self._skip_store_requests
|
||||
|
||||
def _mark_request_skipped_for_pressure(self, req_id: str) -> bool:
|
||||
def _mark_request_skipped_for_pressure(self, req_meta: ReqMeta) -> bool:
|
||||
req_id = req_meta.req_id
|
||||
with self.done_task_lock:
|
||||
already_skipped = req_id in self._skip_store_requests
|
||||
self._store_pressure_active = True
|
||||
self._skip_store_requests.add(req_id)
|
||||
# The pressure itself is global, but only a live job may sentence its
|
||||
# own request to being skipped.
|
||||
if req_meta.store_job_id in self.stored_requests.get(req_id, ()):
|
||||
self._skip_store_requests.add(req_id)
|
||||
return already_skipped
|
||||
|
||||
def _clear_store_pressure(self) -> bool:
|
||||
@@ -727,7 +772,7 @@ class KVCacheStoreSendingThread(KVTransferThread):
|
||||
failed_codes,
|
||||
)
|
||||
if MOONCAKE_NO_AVAILABLE_HANDLE in failed_codes:
|
||||
self._mark_request_skipped_for_pressure(req_meta.req_id)
|
||||
self._mark_request_skipped_for_pressure(req_meta)
|
||||
return False
|
||||
|
||||
if self._clear_store_pressure():
|
||||
@@ -738,22 +783,20 @@ class KVCacheStoreSendingThread(KVTransferThread):
|
||||
return True
|
||||
|
||||
def _handle_request(self, req_meta: ReqMeta):
|
||||
# Cache hits are always a multiple of ``lcm_block_size`` tokens, which
|
||||
# is also ``store_mask``'s precondition.
|
||||
lcm_block_size = self.coord.lcm_block_size
|
||||
token_len = req_meta.token_len_chunk // lcm_block_size * lcm_block_size
|
||||
block_ids_per_group = req_meta.block_ids
|
||||
req_id = req_meta.req_id
|
||||
current_event = req_meta.current_event
|
||||
|
||||
if req_id not in self.stored_requests:
|
||||
self.request_queue.task_done()
|
||||
return
|
||||
|
||||
# Decrement the in-flight counter and signal task_done() in `finally`
|
||||
# so the scheduler can release the GPU blocks it pinned for this
|
||||
# request (via `delay_free_blocks`) even when the store path raises.
|
||||
# The single `finally` is the only way out, so the scheduler releases
|
||||
# this job's GPU block references however the job ends.
|
||||
try:
|
||||
# Cache hits are always a multiple of ``lcm_block_size`` tokens,
|
||||
# which is also ``store_mask``'s precondition.
|
||||
lcm_block_size = self.coord.lcm_block_size
|
||||
token_len = req_meta.token_len_chunk // lcm_block_size * lcm_block_size
|
||||
block_ids_per_group = req_meta.block_ids
|
||||
req_id = req_meta.req_id
|
||||
current_event = req_meta.current_event
|
||||
|
||||
if not self.is_live_store_job(req_meta):
|
||||
return
|
||||
|
||||
if self._should_skip_request(req_id):
|
||||
logger.debug(
|
||||
"Skipping Mooncake store for request %s while CPU/disk "
|
||||
@@ -808,7 +851,7 @@ class KVCacheStoreSendingThread(KVTransferThread):
|
||||
group_indices.append(g_idx)
|
||||
|
||||
if not keys:
|
||||
self._record_saved(req_id, token_len)
|
||||
self._record_saved(req_meta, token_len)
|
||||
return
|
||||
|
||||
# Check which blocks already exist (dedup)
|
||||
@@ -834,7 +877,7 @@ class KVCacheStoreSendingThread(KVTransferThread):
|
||||
]
|
||||
|
||||
if not missing_indices:
|
||||
self._record_saved(req_id, token_len)
|
||||
self._record_saved(req_meta, token_len)
|
||||
return
|
||||
|
||||
if len(missing_indices) != len(keys):
|
||||
@@ -954,7 +997,7 @@ class KVCacheStoreSendingThread(KVTransferThread):
|
||||
)
|
||||
if (
|
||||
MOONCAKE_NO_AVAILABLE_HANDLE in failed_codes
|
||||
and not self._mark_request_skipped_for_pressure(req_id)
|
||||
and not self._mark_request_skipped_for_pressure(req_meta)
|
||||
):
|
||||
logger.warning(
|
||||
"Detected Mooncake CPU/disk offloading pressure "
|
||||
@@ -964,7 +1007,7 @@ class KVCacheStoreSendingThread(KVTransferThread):
|
||||
req_id,
|
||||
)
|
||||
else:
|
||||
self._record_saved(req_id, token_len)
|
||||
self._record_saved(req_meta, token_len)
|
||||
if self._clear_store_pressure():
|
||||
logger.info(
|
||||
"Mooncake CPU/disk offloading pressure cleared "
|
||||
@@ -984,7 +1027,7 @@ class KVCacheStoreSendingThread(KVTransferThread):
|
||||
if self.enable_kv_event and stored_events:
|
||||
self.update_kv_event(stored_events)
|
||||
finally:
|
||||
self.dec_stored_request(req_id)
|
||||
self.finish_store_job(req_meta)
|
||||
self.request_queue.task_done()
|
||||
|
||||
|
||||
@@ -1670,16 +1713,15 @@ class MooncakeStoreWorker:
|
||||
continue
|
||||
request.current_event = current_event
|
||||
assert self.kv_send_thread is not None
|
||||
self.kv_send_thread.add_stored_request(request.req_id)
|
||||
self.kv_send_thread.add_request(request)
|
||||
|
||||
# Check completion of previously queued transfers
|
||||
done_sending = (
|
||||
self._get_and_clear_finished_sending(finished_req_ids, meta)
|
||||
if self.kv_role in ["kv_producer", "kv_both"]
|
||||
else set()
|
||||
)
|
||||
if self.kv_role in ["kv_producer", "kv_both"]:
|
||||
self._close_ended_store_requests(finished_req_ids, meta)
|
||||
|
||||
# Blocks read by a store job are released by the scheduler when the job
|
||||
# reports back (see build_connector_worker_meta), so no request ever waits
|
||||
# on a `finished_sending` signal to get its blocks back.
|
||||
done_sending: set[str] = set()
|
||||
done_recving: set[str] = set()
|
||||
if self.load_async:
|
||||
for recv_thread in self.kv_recv_threads:
|
||||
@@ -1727,35 +1769,37 @@ class MooncakeStoreWorker:
|
||||
self.kv_connector_stats = MooncakeStoreConnectorStats()
|
||||
return kv_connector_stats
|
||||
|
||||
def _get_and_clear_finished_sending(
|
||||
def _close_ended_store_requests(
|
||||
self,
|
||||
finished_req_ids: set[str],
|
||||
meta: MooncakeStoreConnectorMetadata,
|
||||
) -> set[str]:
|
||||
) -> None:
|
||||
"""Retire the ledger entries of requests that finished or were preempted.
|
||||
|
||||
An entry may only go once its jobs have drained, because they still read
|
||||
the resume offset it owns; a request that comes back after preemption
|
||||
then saves from the start rather than from where the last attempt got to.
|
||||
"""
|
||||
assert self.kv_send_thread is not None
|
||||
finished_sending: set[str] = set()
|
||||
|
||||
for req_id in meta.preempted_req_ids:
|
||||
self.kv_send_thread.delete_finished_stored_request(req_id)
|
||||
|
||||
for req_id in self.kv_send_thread.stored_requests.copy():
|
||||
if (
|
||||
self.kv_send_thread.stored_requests[req_id] == 0
|
||||
and req_id in self.finished_store_req
|
||||
):
|
||||
self.finished_store_req.remove(req_id)
|
||||
finished_sending.add(req_id)
|
||||
self.kv_send_thread.delete_finished_stored_request(req_id)
|
||||
|
||||
for req_id in finished_req_ids:
|
||||
req_remain_jobs = self.kv_send_thread.stored_requests.get(req_id)
|
||||
if req_remain_jobs == 0:
|
||||
finished_sending.add(req_id)
|
||||
self.kv_send_thread.delete_finished_stored_request(req_id)
|
||||
elif req_remain_jobs is not None:
|
||||
for req_id in finished_req_ids | self.finished_store_req:
|
||||
if self.kv_send_thread.stored_requests.get(req_id):
|
||||
# Queued jobs still need the resume offset; retire on a later step.
|
||||
self.finished_store_req.add(req_id)
|
||||
else:
|
||||
self.finished_store_req.discard(req_id)
|
||||
self.kv_send_thread.delete_finished_stored_request(req_id)
|
||||
|
||||
return finished_sending
|
||||
def build_connector_worker_meta(self) -> MooncakeStoreWorkerMetadata | None:
|
||||
if self.kv_send_thread is None:
|
||||
return None
|
||||
completed_saves = self.kv_send_thread.take_completed_saves()
|
||||
if not completed_saves:
|
||||
return None
|
||||
return MooncakeStoreWorkerMetadata(completed_saves=completed_saves)
|
||||
|
||||
def lookup(self, num_tokens: int, block_hashes: Sequence[BlockHash]) -> int:
|
||||
"""Check how many prefix tokens exist in the store.
|
||||
|
||||
Reference in New Issue
Block a user