[Bugfix][HiSparse] Preserve host prefix publication after request completion (#59007)
Signed-off-by: Yueh-Ting Chen <yueh.ting.chen@gmail.com> Co-authored-by: Matthew Bonanni <mbonanni@redhat.com>
This commit is contained in:
co-authored by
Matthew Bonanni
parent
279012f94d
commit
ff1b87cca2
@@ -44,6 +44,8 @@ class _PendingPublication:
|
||||
num_pages: int
|
||||
retention_interval: int | None
|
||||
replay_boundaries: Sequence[int]
|
||||
detached_blocks: list[KVCacheBlock] | None = None
|
||||
num_cached_blocks: int = 0
|
||||
|
||||
|
||||
@dataclass
|
||||
@@ -724,6 +726,10 @@ class HiSparseCoordinator:
|
||||
if pending.restore:
|
||||
self._record_copy(request_id, page_idx, pending.host_block)
|
||||
completed_request_ids.add(request_id)
|
||||
elif pending.request_state.publication is not None:
|
||||
if not pending.restore:
|
||||
pending.request_state.valid_pages.add(page_idx)
|
||||
self._publish_detached_blocks(pending.request_state)
|
||||
assert self.host_manager is not None
|
||||
self.host_manager.block_pool.free_blocks([pending.host_block])
|
||||
for request_id in completed_request_ids:
|
||||
@@ -732,12 +738,55 @@ class HiSparseCoordinator:
|
||||
state.ready_prefix_pages += 1
|
||||
self._publish_host_blocks_if_ready(request_id)
|
||||
|
||||
def _publish_detached_blocks(self, state: _HiSparseRequestState) -> None:
|
||||
publication = state.publication
|
||||
if publication is None or state.pending_pages:
|
||||
return
|
||||
blocks = publication.detached_blocks
|
||||
if blocks is None:
|
||||
return
|
||||
assert self.host_manager is not None
|
||||
while state.ready_prefix_pages in state.valid_pages:
|
||||
state.ready_prefix_pages += 1
|
||||
# The host group uses full-attention retention. Only sealed pages with
|
||||
# completed copies can outlive the request, including on cancellation.
|
||||
self.host_manager.block_pool.cache_full_blocks(
|
||||
request=publication.request,
|
||||
blocks=blocks,
|
||||
num_cached_blocks=publication.num_cached_blocks,
|
||||
num_full_blocks=min(state.ready_prefix_pages, publication.num_pages),
|
||||
block_size=self.host_manager.block_size,
|
||||
kv_cache_group_id=self.host_manager.kv_cache_group_id,
|
||||
)
|
||||
self.host_manager.block_pool.free_blocks(reversed(blocks))
|
||||
state.publication = None
|
||||
|
||||
def free(self, request_id: str) -> None:
|
||||
"""Detach the request; its clean pages stay readable copies in the pool."""
|
||||
self._pending_imports.pop(request_id, None)
|
||||
state = self.request_states.pop(request_id, None)
|
||||
if state is None:
|
||||
return
|
||||
publication = state.publication
|
||||
if (
|
||||
publication is not None
|
||||
and publication.request.is_finished()
|
||||
and state.pending_pages
|
||||
):
|
||||
assert self.host_manager is not None
|
||||
publication.detached_blocks = self.host_manager.req_to_blocks[request_id][
|
||||
: publication.num_pages
|
||||
]
|
||||
publication.num_cached_blocks = self.host_manager.num_cached_block.get(
|
||||
request_id, 0
|
||||
)
|
||||
# Completed pages need leases too: another allocation may evict
|
||||
# them before the remaining copies finish and the prefix publishes.
|
||||
self.host_manager.block_pool.touch(publication.detached_blocks)
|
||||
else:
|
||||
# Preemption can cancel a scheduled forward after its spills were
|
||||
# planned, so its copies do not prove that the KV was computed.
|
||||
state.publication = None
|
||||
for manager in self.resident_managers:
|
||||
blocks = manager.req_to_blocks.get(request_id)
|
||||
if not blocks:
|
||||
|
||||
Reference in New Issue
Block a user