[Bugfix][Mooncake] Offload producer partial tails on request finish (#52832)

Signed-off-by: Dao Le <Dao007forever@gmail.com>
Co-authored-by: Roger Wang <hey@rogerw.io>
Co-authored-by: OpenAI Codex <noreply@openai.com>
This commit is contained in:
Dao007forever
2026-09-01 13:23:35 -07:00
committed by GitHub
co-authored by Roger Wang OpenAI Codex
parent 55178f2d09
commit 7cb9a88f50
10 changed files with 375 additions and 10 deletions
@@ -561,6 +561,19 @@ class KVConnectorBase_V1(ABC):
"""
return False, None
def register_finished_partial_tail(
self,
request: "Request",
block_ids: tuple[list[int], ...],
partial_tail_offloads: list[tuple[int, int, int]],
) -> bool:
"""Register finish-time partial-tail sources before block cleanup.
Returns True when the connector accepts responsibility for the sources
and the request's blocks must remain alive until ``get_finished()``.
"""
return False
def take_events(self) -> Iterable["KVCacheEvent"]:
"""
Take the KV cache events from the connector.
@@ -265,6 +265,17 @@ class MooncakeStoreConnector(KVConnectorBase_V1, SupportsHMA):
# so a finishing request never has to defer freeing them.
return False, None
def register_finished_partial_tail(
self,
request: Request,
block_ids: tuple[list[int], ...],
partial_tail_offloads: list[tuple[int, int, int]],
) -> bool:
assert self.connector_scheduler is not None
return self.connector_scheduler.register_finished_partial_tail(
request, block_ids, partial_tail_offloads
)
def reset_cache(self) -> bool | None:
"""Reset the external Mooncake store on prefix-cache reset.
@@ -102,6 +102,7 @@ class MooncakeStoreScheduler:
self._request_trackers: dict[str, RequestTracker] = {} # scheduled new requests
self._unfinished_requests: dict[str, tuple[Request, tuple[list[int], ...]]] = {}
self._unfinished_request_ids: set[str] = set()
self._finished_partial_tail_metas: dict[str, ReqMeta] = {}
def bind_gpu_block_pool(self, gpu_block_pool: BlockPool) -> None:
self._gpu_block_pool = gpu_block_pool
@@ -402,6 +403,14 @@ class MooncakeStoreScheduler:
)
self._apply_current_save_block_ids(meta, scheduler_output)
# Finish-time handoffs arrive after the producing step's metadata was
# built. Their exact blocks were pinned when they were registered, so
# they remain valid after request cleanup and need no current snapshot.
for req_meta in self._finished_partial_tail_metas.values():
meta.add_request(req_meta)
self._finished_partial_tail_metas.clear()
self._reference_save_blocks(meta)
return meta
@@ -440,6 +449,9 @@ class MooncakeStoreScheduler:
assert pool is not None, (
"GPU block pool must be bound before any store job is emitted"
)
if req_meta.store_job_id is not None:
assert req_meta.store_job_id in self._pinned_saves
continue
req_meta.store_job_id = store_job_id = self._next_store_job_id
self._next_store_job_id += 1
block_ids: list[int] = []
@@ -470,6 +482,60 @@ class MooncakeStoreScheduler:
self._pinned_saves[store_job_id] = (block_ids, self._num_workers)
pool.touch([pool.blocks[block_id] for block_id in block_ids])
def register_finished_partial_tail(
self,
request: Request,
block_ids: tuple[list[int], ...],
partial_tail_offloads: list[tuple[int, int, int]],
) -> bool:
"""Queue and pin a finish-time tail for the next connector step."""
if self.kv_role == "kv_consumer" or not partial_tail_offloads:
return False
tracker = self._request_trackers.get(request.request_id)
if tracker is None or not any(block_ids):
return False
boundaries = {boundary for _, _, boundary in partial_tail_offloads}
if len(boundaries) != 1:
raise ValueError(
"Partial-tail offloads for one request must share a boundary"
)
boundary_tokens = next(iter(boundaries))
if boundary_tokens > tracker.prefill_end_tokens:
return False
pinned_block_ids: list[int] = []
for group_id, block_id, _ in partial_tail_offloads:
if group_id not in self._boundary_state_group_ids:
return False
if block_id == NULL_BLOCK_ID:
return False
pinned_block_ids.append(block_id)
pinned_block_ids = list(dict.fromkeys(pinned_block_ids))
pool = self._gpu_block_pool
assert pool is not None, (
"GPU block pool must be bound before a finish-time handoff"
)
assert request.request_id not in self._finished_partial_tail_metas
store_job_id = self._next_store_job_id
self._next_store_job_id += 1
self._pinned_saves[store_job_id] = (pinned_block_ids, self._num_workers)
pool.touch([pool.blocks[block_id] for block_id in pinned_block_ids])
self._finished_partial_tail_metas[request.request_id] = ReqMeta(
req_id=request.request_id,
token_len_chunk=0,
block_ids=tuple(group.copy() for group in block_ids),
block_hashes=list(request.block_hashes),
can_save=True,
num_prompt_tokens=tracker.prefill_end_tokens,
store_job_id=store_job_id,
boundary_state_offloads=partial_tail_offloads,
)
tracker.has_pending_offload = True
# The store job owns exact block refs, so request cleanup need not wait.
return False
def _handle_boundary_state_offloads(
self,
offloads: dict[str, list[tuple[int, int, int]]],
@@ -523,6 +523,18 @@ class MultiConnector(KVConnectorBase_V1, SupportsHMA):
lambda c: c.request_finished(request, blocks),
)
def register_finished_partial_tail(
self,
request: "Request",
block_ids: tuple[list[int], ...],
partial_tail_offloads: list[tuple[int, int, int]],
) -> bool:
accepted = [
c.register_finished_partial_tail(request, block_ids, partial_tail_offloads)
for c in self._connectors
]
return any(accepted)
def request_finished_all_groups(
self,
request: "Request",
+23
View File
@@ -874,6 +874,29 @@ class KVCacheManager:
)
return offloads
def finalize_partial_tail_offloads(
self, request: Request
) -> list[tuple[int, int, int]]:
"""Consume safe producer partial tails when a request finishes.
A mamba align table block is a valid boundary source only if no later
token was forwarded. The connector pins and queues the exact table
block before request cleanup, then releases the pin when every worker
reports the store job complete.
"""
offloads: list[tuple[int, int, int]] = []
for mgr in self.coordinator.single_type_managers:
finalized = mgr.finalize_partial_tail_offload(
request.request_id,
request.num_computed_tokens,
request.num_in_flight_tokens,
)
if finalized is None:
continue
group_id, block, boundary_tokens = finalized
offloads.append((group_id, block.block_id, boundary_tokens))
return offloads
def new_step_starts(self) -> None:
"""Notify the coordinator that a new step is starting."""
self.coordinator.new_step_starts()
+22 -3
View File
@@ -2775,6 +2775,13 @@ class Scheduler(SchedulerInterface):
if self.connector is None:
return False, None
finished_partial_tails: list[tuple[int, int, int]] = []
kv_transfer_config = self.vllm_config.kv_transfer_config
if kv_transfer_config is not None and kv_transfer_config.is_kv_producer:
finished_partial_tails = (
self.kv_cache_manager.finalize_partial_tail_offloads(request)
)
# Free any out-of-window prefix blocks before we hand the block table to
# the connector, on the processed-token basis (see `allocate_slots`).
self.kv_cache_manager.remove_skipped_blocks(
@@ -2789,6 +2796,13 @@ class Scheduler(SchedulerInterface):
request_id=request.request_id,
num_computed_tokens=request.num_computed_tokens,
)
partial_tail_delay = False
if finished_partial_tails:
partial_tail_delay = self.connector.register_finished_partial_tail(
request,
block_ids,
finished_partial_tails,
)
if not isinstance(self.connector, SupportsHMA):
# NOTE(Kuntai): We should deprecate this code path after we enforce
@@ -2796,9 +2810,14 @@ class Scheduler(SchedulerInterface):
# Hybrid memory allocator should be already turned off for this
# code path, but let's double-check here.
assert len(self.kv_cache_config.kv_cache_groups) == 1
return self.connector.request_finished(request, block_ids[0])
return self.connector.request_finished_all_groups(request, block_ids)
delay_free, kv_xfer_params = self.connector.request_finished(
request, block_ids[0]
)
else:
delay_free, kv_xfer_params = self.connector.request_finished_all_groups(
request, block_ids
)
return delay_free or partial_tail_delay, kv_xfer_params
def _request_remaining_blocks(self, request: Request) -> int:
"""Blocks `request` still needs to allocate to hold its full sequence."""
+37 -7
View File
@@ -407,6 +407,15 @@ class SingleTypeKVCacheManager(ABC):
self._pending_boundary_state_offloads = []
return pending
def finalize_partial_tail_offload(
self,
request_id: str,
num_computed_tokens: int,
num_in_flight_tokens: int,
) -> tuple[int, KVCacheBlock, int] | None:
"""Finalize a producer partial tail when its request finishes."""
return None
def _apply_cow(
self,
request_id: str,
@@ -1383,10 +1392,10 @@ class MambaManager(SingleTypeKVCacheManager):
# current allocation.
self._num_checkpoint_blocks: dict[str, int] = {}
# Requests that registered their own last-prompt-boundary partial
# tail (producers). On the next step's CoW the boundary state moves
# into a private cow_block; we record that block for connector
# offload (see _pending_boundary_state_offloads).
self._producer_partial_tail_reqs: dict[str, int] = {}
# tail (producers). A later CoW hands its private copy to the
# connector; a request that finishes first hands off this table
# source directly.
self._producer_partial_tail_reqs: dict[str, tuple[KVCacheBlock, int]] = {}
@classmethod
def find_longest_cache_hit(
@@ -1749,10 +1758,12 @@ class MambaManager(SingleTypeKVCacheManager):
self.block_pool.move_block_hashes(source_block, cow_block)
self._pending_cow_copies.append((source_block, cow_block))
source_block.ref_cnt += 1
boundary_tokens = self._producer_partial_tail_reqs.pop(
producer_tail = self._producer_partial_tail_reqs.pop(
request_id, None
)
if boundary_tokens is not None:
if producer_tail is not None:
marker_block, boundary_tokens = producer_tail
assert marker_block is source_block
# This CoW preserved a producer's own boundary
# state in cow_block; hand it to the connector for
# partial-tail offload once the copy has run.
@@ -1788,6 +1799,22 @@ class MambaManager(SingleTypeKVCacheManager):
req_blocks.append(block)
req_blocks[block_idx] = self._null_block
def finalize_partial_tail_offload(
self,
request_id: str,
num_computed_tokens: int,
num_in_flight_tokens: int,
) -> tuple[int, KVCacheBlock, int] | None:
if self.mamba_cache_mode != "align":
return None
producer_tail = self._producer_partial_tail_reqs.pop(request_id, None)
if producer_tail is None:
return None
source_block, boundary_tokens = producer_tail
if num_in_flight_tokens != 0 or num_computed_tokens != boundary_tokens:
return None
return self.kv_cache_group_id, source_block, boundary_tokens
def pop_blocks_for_free(self, request_id: str) -> list[KVCacheBlock]:
if self.mamba_cache_mode == "align":
self._allocated_block_reqs.discard(request_id)
@@ -1894,7 +1921,10 @@ class MambaManager(SingleTypeKVCacheManager):
# in ``source_block`` but the next step's forward overwrites it. The
# upcoming CoW copies it into a durable cow_block; record the req so
# allocate_new_blocks hands that block to the connector for offload.
self._producer_partial_tail_reqs[request.request_id] = num_tokens
self._producer_partial_tail_reqs[request.request_id] = (
source_block,
num_tokens,
)
return partial_hash