[Perf][HiSparse] Cache per-request residency instead of rescanning every page each step (#60083)
Signed-off-by: Matthew Bonanni <mbonanni@redhat.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
fba31f31a9
commit
e82b80099a
@@ -168,6 +168,10 @@ class HiSparseCoordinator:
|
||||
# request -> prefix length an external load is still filling in.
|
||||
self._pending_imports: dict[str, int] = {}
|
||||
self.block_table_updates: set[str] = set()
|
||||
# Requests whose resident pages were all non-null when last scanned.
|
||||
# Pages only become null at admission or import (before the first
|
||||
# scan) and when the pool reuses an unpinned page (_lose_page).
|
||||
self._resident_requests: set[str] = set()
|
||||
self.spills_to_send: list[SparseKVPageTransfer] = []
|
||||
self.pending_spills: dict[int, _PendingSpill] = {}
|
||||
self.request_states: dict[str, _HiSparseRequestState] = {}
|
||||
@@ -188,6 +192,7 @@ class HiSparseCoordinator:
|
||||
|
||||
def commit_computed_blocks(self, request_id: str, num_host_pages: int) -> None:
|
||||
"""Account for a prefix hit on host pages the request now references."""
|
||||
self._resident_requests.discard(request_id)
|
||||
if not self.resident_managers or self.host_manager is None:
|
||||
return
|
||||
host_blocks = self.host_manager.req_to_blocks.get(request_id, ())
|
||||
@@ -382,6 +387,7 @@ class HiSparseCoordinator:
|
||||
self._lose_page(request_id, page_idx)
|
||||
|
||||
def _lose_page(self, request_id: str, page_idx: int) -> None:
|
||||
self._resident_requests.discard(request_id)
|
||||
state = self.request_states.get(request_id)
|
||||
if state is None:
|
||||
return
|
||||
@@ -530,6 +536,7 @@ class HiSparseCoordinator:
|
||||
|
||||
def record_pending_host_import(self, request_id: str, num_tokens: int) -> None:
|
||||
"""Note a prefix an external load is populating in host pages."""
|
||||
self._resident_requests.discard(request_id)
|
||||
self._pending_imports[request_id] = num_tokens
|
||||
|
||||
def finish_host_import(self, request_id: str, *, failed: bool) -> None:
|
||||
@@ -674,15 +681,22 @@ class HiSparseCoordinator:
|
||||
"""Return whether every scheduled request can read only resident KV."""
|
||||
if not self.resident_managers:
|
||||
return False
|
||||
block_size = self.resident_managers[0].block_size
|
||||
first = self.resident_managers[0]
|
||||
for request_id, num_computed_tokens, num_scheduled_tokens in requests:
|
||||
num_pages = cdiv(num_computed_tokens + num_scheduled_tokens, block_size)
|
||||
for page_idx in range(num_pages):
|
||||
if any(
|
||||
manager.get_resident_page(request_id, page_idx) is None
|
||||
for manager in self.resident_managers
|
||||
):
|
||||
return False
|
||||
num_pages = cdiv(
|
||||
num_computed_tokens + num_scheduled_tokens, first.block_size
|
||||
)
|
||||
if len(first.req_to_blocks.get(request_id, ())) < num_pages:
|
||||
return False
|
||||
if request_id in self._resident_requests:
|
||||
continue
|
||||
if any(
|
||||
block.is_null
|
||||
for manager in self.resident_managers
|
||||
for block in manager.req_to_blocks.get(request_id, ())
|
||||
):
|
||||
return False
|
||||
self._resident_requests.add(request_id)
|
||||
return True
|
||||
|
||||
def take_block_table_updates(self) -> dict[str, tuple[list[int], ...]]:
|
||||
@@ -806,6 +820,7 @@ class HiSparseCoordinator:
|
||||
|
||||
def free(self, request_id: str) -> None:
|
||||
"""Detach the request; its clean pages stay readable copies in the pool."""
|
||||
self._resident_requests.discard(request_id)
|
||||
self._pending_imports.pop(request_id, None)
|
||||
state = self.request_states.pop(request_id, None)
|
||||
if state is None:
|
||||
|
||||
Reference in New Issue
Block a user