[Bugfix][HiSparse] Preserve host prefix publication after request completion (#59007)

Signed-off-by: Yueh-Ting Chen <yueh.ting.chen@gmail.com>
Co-authored-by: Matthew Bonanni <mbonanni@redhat.com>
This commit is contained in:
Yueh-Ting (eop) Chen
2026-09-30 16:14:51 +00:00
committed by GitHub
co-authored by Matthew Bonanni
parent 279012f94d
commit ff1b87cca2
3 changed files with 223 additions and 0 deletions
+49
View File
@@ -44,6 +44,8 @@ class _PendingPublication:
num_pages: int
retention_interval: int | None
replay_boundaries: Sequence[int]
detached_blocks: list[KVCacheBlock] | None = None
num_cached_blocks: int = 0
@dataclass
@@ -724,6 +726,10 @@ class HiSparseCoordinator:
if pending.restore:
self._record_copy(request_id, page_idx, pending.host_block)
completed_request_ids.add(request_id)
elif pending.request_state.publication is not None:
if not pending.restore:
pending.request_state.valid_pages.add(page_idx)
self._publish_detached_blocks(pending.request_state)
assert self.host_manager is not None
self.host_manager.block_pool.free_blocks([pending.host_block])
for request_id in completed_request_ids:
@@ -732,12 +738,55 @@ class HiSparseCoordinator:
state.ready_prefix_pages += 1
self._publish_host_blocks_if_ready(request_id)
def _publish_detached_blocks(self, state: _HiSparseRequestState) -> None:
publication = state.publication
if publication is None or state.pending_pages:
return
blocks = publication.detached_blocks
if blocks is None:
return
assert self.host_manager is not None
while state.ready_prefix_pages in state.valid_pages:
state.ready_prefix_pages += 1
# The host group uses full-attention retention. Only sealed pages with
# completed copies can outlive the request, including on cancellation.
self.host_manager.block_pool.cache_full_blocks(
request=publication.request,
blocks=blocks,
num_cached_blocks=publication.num_cached_blocks,
num_full_blocks=min(state.ready_prefix_pages, publication.num_pages),
block_size=self.host_manager.block_size,
kv_cache_group_id=self.host_manager.kv_cache_group_id,
)
self.host_manager.block_pool.free_blocks(reversed(blocks))
state.publication = None
def free(self, request_id: str) -> None:
"""Detach the request; its clean pages stay readable copies in the pool."""
self._pending_imports.pop(request_id, None)
state = self.request_states.pop(request_id, None)
if state is None:
return
publication = state.publication
if (
publication is not None
and publication.request.is_finished()
and state.pending_pages
):
assert self.host_manager is not None
publication.detached_blocks = self.host_manager.req_to_blocks[request_id][
: publication.num_pages
]
publication.num_cached_blocks = self.host_manager.num_cached_block.get(
request_id, 0
)
# Completed pages need leases too: another allocation may evict
# them before the remaining copies finish and the prefix publishes.
self.host_manager.block_pool.touch(publication.detached_blocks)
else:
# Preemption can cancel a scheduled forward after its spills were
# planned, so its copies do not prove that the KV was computed.
state.publication = None
for manager in self.resident_managers:
blocks = manager.req_to_blocks.get(request_id)
if not blocks: