[Perf][HiSparse] Cache per-request residency instead of rescanning every page each step (#60083)

Signed-off-by: Matthew Bonanni <mbonanni@redhat.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Matthew Bonanni
2026-10-06 13:04:55 -04:00
committed by GitHub
co-authored by Claude Opus 5.5
parent fba31f31a9
commit e82b80099a
+23 -8
View File
@@ -168,6 +168,10 @@ class HiSparseCoordinator:
# request -> prefix length an external load is still filling in.
self._pending_imports: dict[str, int] = {}
self.block_table_updates: set[str] = set()
# Requests whose resident pages were all non-null when last scanned.
# Pages only become null at admission or import (before the first
# scan) and when the pool reuses an unpinned page (_lose_page).
self._resident_requests: set[str] = set()
self.spills_to_send: list[SparseKVPageTransfer] = []
self.pending_spills: dict[int, _PendingSpill] = {}
self.request_states: dict[str, _HiSparseRequestState] = {}
@@ -188,6 +192,7 @@ class HiSparseCoordinator:
def commit_computed_blocks(self, request_id: str, num_host_pages: int) -> None:
"""Account for a prefix hit on host pages the request now references."""
self._resident_requests.discard(request_id)
if not self.resident_managers or self.host_manager is None:
return
host_blocks = self.host_manager.req_to_blocks.get(request_id, ())
@@ -382,6 +387,7 @@ class HiSparseCoordinator:
self._lose_page(request_id, page_idx)
def _lose_page(self, request_id: str, page_idx: int) -> None:
self._resident_requests.discard(request_id)
state = self.request_states.get(request_id)
if state is None:
return
@@ -530,6 +536,7 @@ class HiSparseCoordinator:
def record_pending_host_import(self, request_id: str, num_tokens: int) -> None:
"""Note a prefix an external load is populating in host pages."""
self._resident_requests.discard(request_id)
self._pending_imports[request_id] = num_tokens
def finish_host_import(self, request_id: str, *, failed: bool) -> None:
@@ -674,15 +681,22 @@ class HiSparseCoordinator:
"""Return whether every scheduled request can read only resident KV."""
if not self.resident_managers:
return False
block_size = self.resident_managers[0].block_size
first = self.resident_managers[0]
for request_id, num_computed_tokens, num_scheduled_tokens in requests:
num_pages = cdiv(num_computed_tokens + num_scheduled_tokens, block_size)
for page_idx in range(num_pages):
if any(
manager.get_resident_page(request_id, page_idx) is None
for manager in self.resident_managers
):
return False
num_pages = cdiv(
num_computed_tokens + num_scheduled_tokens, first.block_size
)
if len(first.req_to_blocks.get(request_id, ())) < num_pages:
return False
if request_id in self._resident_requests:
continue
if any(
block.is_null
for manager in self.resident_managers
for block in manager.req_to_blocks.get(request_id, ())
):
return False
self._resident_requests.add(request_id)
return True
def take_block_table_updates(self) -> dict[str, tuple[list[int], ...]]:
@@ -806,6 +820,7 @@ class HiSparseCoordinator:
def free(self, request_id: str) -> None:
"""Detach the request; its clean pages stay readable copies in the pool."""
self._resident_requests.discard(request_id)
self._pending_imports.pop(request_id, None)
state = self.request_states.pop(request_id, None)
if state is None: