[Bugfix][Mooncake] Offload producer partial tails on request finish (#52832)
Signed-off-by: Dao Le <Dao007forever@gmail.com> Co-authored-by: Roger Wang <hey@rogerw.io> Co-authored-by: OpenAI Codex <noreply@openai.com>
This commit is contained in:
co-authored by
Roger Wang
OpenAI Codex
parent
55178f2d09
commit
7cb9a88f50
@@ -561,6 +561,19 @@ class KVConnectorBase_V1(ABC):
|
||||
"""
|
||||
return False, None
|
||||
|
||||
def register_finished_partial_tail(
|
||||
self,
|
||||
request: "Request",
|
||||
block_ids: tuple[list[int], ...],
|
||||
partial_tail_offloads: list[tuple[int, int, int]],
|
||||
) -> bool:
|
||||
"""Register finish-time partial-tail sources before block cleanup.
|
||||
|
||||
Returns True when the connector accepts responsibility for the sources
|
||||
and the request's blocks must remain alive until ``get_finished()``.
|
||||
"""
|
||||
return False
|
||||
|
||||
def take_events(self) -> Iterable["KVCacheEvent"]:
|
||||
"""
|
||||
Take the KV cache events from the connector.
|
||||
|
||||
@@ -265,6 +265,17 @@ class MooncakeStoreConnector(KVConnectorBase_V1, SupportsHMA):
|
||||
# so a finishing request never has to defer freeing them.
|
||||
return False, None
|
||||
|
||||
def register_finished_partial_tail(
|
||||
self,
|
||||
request: Request,
|
||||
block_ids: tuple[list[int], ...],
|
||||
partial_tail_offloads: list[tuple[int, int, int]],
|
||||
) -> bool:
|
||||
assert self.connector_scheduler is not None
|
||||
return self.connector_scheduler.register_finished_partial_tail(
|
||||
request, block_ids, partial_tail_offloads
|
||||
)
|
||||
|
||||
def reset_cache(self) -> bool | None:
|
||||
"""Reset the external Mooncake store on prefix-cache reset.
|
||||
|
||||
|
||||
@@ -102,6 +102,7 @@ class MooncakeStoreScheduler:
|
||||
self._request_trackers: dict[str, RequestTracker] = {} # scheduled new requests
|
||||
self._unfinished_requests: dict[str, tuple[Request, tuple[list[int], ...]]] = {}
|
||||
self._unfinished_request_ids: set[str] = set()
|
||||
self._finished_partial_tail_metas: dict[str, ReqMeta] = {}
|
||||
|
||||
def bind_gpu_block_pool(self, gpu_block_pool: BlockPool) -> None:
|
||||
self._gpu_block_pool = gpu_block_pool
|
||||
@@ -402,6 +403,14 @@ class MooncakeStoreScheduler:
|
||||
)
|
||||
|
||||
self._apply_current_save_block_ids(meta, scheduler_output)
|
||||
|
||||
# Finish-time handoffs arrive after the producing step's metadata was
|
||||
# built. Their exact blocks were pinned when they were registered, so
|
||||
# they remain valid after request cleanup and need no current snapshot.
|
||||
for req_meta in self._finished_partial_tail_metas.values():
|
||||
meta.add_request(req_meta)
|
||||
self._finished_partial_tail_metas.clear()
|
||||
|
||||
self._reference_save_blocks(meta)
|
||||
return meta
|
||||
|
||||
@@ -440,6 +449,9 @@ class MooncakeStoreScheduler:
|
||||
assert pool is not None, (
|
||||
"GPU block pool must be bound before any store job is emitted"
|
||||
)
|
||||
if req_meta.store_job_id is not None:
|
||||
assert req_meta.store_job_id in self._pinned_saves
|
||||
continue
|
||||
req_meta.store_job_id = store_job_id = self._next_store_job_id
|
||||
self._next_store_job_id += 1
|
||||
block_ids: list[int] = []
|
||||
@@ -470,6 +482,60 @@ class MooncakeStoreScheduler:
|
||||
self._pinned_saves[store_job_id] = (block_ids, self._num_workers)
|
||||
pool.touch([pool.blocks[block_id] for block_id in block_ids])
|
||||
|
||||
def register_finished_partial_tail(
|
||||
self,
|
||||
request: Request,
|
||||
block_ids: tuple[list[int], ...],
|
||||
partial_tail_offloads: list[tuple[int, int, int]],
|
||||
) -> bool:
|
||||
"""Queue and pin a finish-time tail for the next connector step."""
|
||||
if self.kv_role == "kv_consumer" or not partial_tail_offloads:
|
||||
return False
|
||||
tracker = self._request_trackers.get(request.request_id)
|
||||
if tracker is None or not any(block_ids):
|
||||
return False
|
||||
boundaries = {boundary for _, _, boundary in partial_tail_offloads}
|
||||
if len(boundaries) != 1:
|
||||
raise ValueError(
|
||||
"Partial-tail offloads for one request must share a boundary"
|
||||
)
|
||||
boundary_tokens = next(iter(boundaries))
|
||||
if boundary_tokens > tracker.prefill_end_tokens:
|
||||
return False
|
||||
|
||||
pinned_block_ids: list[int] = []
|
||||
for group_id, block_id, _ in partial_tail_offloads:
|
||||
if group_id not in self._boundary_state_group_ids:
|
||||
return False
|
||||
if block_id == NULL_BLOCK_ID:
|
||||
return False
|
||||
pinned_block_ids.append(block_id)
|
||||
pinned_block_ids = list(dict.fromkeys(pinned_block_ids))
|
||||
|
||||
pool = self._gpu_block_pool
|
||||
assert pool is not None, (
|
||||
"GPU block pool must be bound before a finish-time handoff"
|
||||
)
|
||||
assert request.request_id not in self._finished_partial_tail_metas
|
||||
store_job_id = self._next_store_job_id
|
||||
self._next_store_job_id += 1
|
||||
self._pinned_saves[store_job_id] = (pinned_block_ids, self._num_workers)
|
||||
pool.touch([pool.blocks[block_id] for block_id in pinned_block_ids])
|
||||
|
||||
self._finished_partial_tail_metas[request.request_id] = ReqMeta(
|
||||
req_id=request.request_id,
|
||||
token_len_chunk=0,
|
||||
block_ids=tuple(group.copy() for group in block_ids),
|
||||
block_hashes=list(request.block_hashes),
|
||||
can_save=True,
|
||||
num_prompt_tokens=tracker.prefill_end_tokens,
|
||||
store_job_id=store_job_id,
|
||||
boundary_state_offloads=partial_tail_offloads,
|
||||
)
|
||||
tracker.has_pending_offload = True
|
||||
# The store job owns exact block refs, so request cleanup need not wait.
|
||||
return False
|
||||
|
||||
def _handle_boundary_state_offloads(
|
||||
self,
|
||||
offloads: dict[str, list[tuple[int, int, int]]],
|
||||
|
||||
@@ -523,6 +523,18 @@ class MultiConnector(KVConnectorBase_V1, SupportsHMA):
|
||||
lambda c: c.request_finished(request, blocks),
|
||||
)
|
||||
|
||||
def register_finished_partial_tail(
|
||||
self,
|
||||
request: "Request",
|
||||
block_ids: tuple[list[int], ...],
|
||||
partial_tail_offloads: list[tuple[int, int, int]],
|
||||
) -> bool:
|
||||
accepted = [
|
||||
c.register_finished_partial_tail(request, block_ids, partial_tail_offloads)
|
||||
for c in self._connectors
|
||||
]
|
||||
return any(accepted)
|
||||
|
||||
def request_finished_all_groups(
|
||||
self,
|
||||
request: "Request",
|
||||
|
||||
Reference in New Issue
Block a user