[HiSparse] Own GPU residency instead of editing core block-table rows (#60647)

Signed-off-by: Matthew Bonanni <mbonanni@redhat.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Matthew Bonanni
2026-10-09 21:25:44 -07:00
committed by GitHub
co-authored by Claude Opus 5.5
parent 9bd074a996
commit d5f356dae0
15 changed files with 429 additions and 128 deletions
-3
View File
@@ -304,9 +304,6 @@ class SchedulerOutput:
# CoW copies to apply after zeroing new blocks and before forward.
kv_cache_block_copies: list[KVCacheBlockCopy] | None = None
# Complete block-table rows that replace incrementally appended block IDs.
block_table_updates: dict[str, tuple[list[int], ...]] | None = None
# Scheduler-local; always None by the time this reaches a worker.
kv_connector_block_state: KVConnectorBlockState | None = None
+38 -4
View File
@@ -2503,6 +2503,10 @@ class HiSparseResidentManager(_HiSparseAuxiliaryManager):
max_admission_blocks_per_request: int
def __init__(self, kv_cache_spec: HiSparseResidentSpec, **kwargs) -> None:
super().__init__(kv_cache_spec, **kwargs)
self.residency_changes: dict[str, dict[int, KVCacheBlock]] = {}
def get_num_blocks_to_allocate(
self,
request_id: str,
@@ -2546,7 +2550,7 @@ class HiSparseResidentManager(_HiSparseAuxiliaryManager):
tail_page = (num_tokens - 1) // self.block_size
if len(blocks) <= tail_page:
blocks.extend([self._null_block] * (tail_page + 1 - len(blocks)))
if blocks[tail_page].is_null:
if self.get_resident_page(request_id, tail_page) is None:
blocks[tail_page] = self.block_pool.get_new_blocks(1)[0]
def add_local_computed_blocks(
@@ -2580,25 +2584,55 @@ class HiSparseResidentManager(_HiSparseAuxiliaryManager):
def pop_blocks_for_free(self, request_id: str) -> list[KVCacheBlock]:
assert self.coordinator is not None
self.coordinator.free(request_id)
return super().pop_blocks_for_free(request_id)
blocks = self.get_residency_row(request_id)
self.residency_changes.pop(request_id, None)
super().pop_blocks_for_free(request_id)
return blocks
def adopt_resident_page(
self, request_id: str, block_idx: int, block: KVCacheBlock
) -> bool:
"""Point a null prefix page at a pinned GPU copy of its contents."""
blocks = self.req_to_blocks.get(request_id)
if blocks is None or block_idx >= len(blocks) or not blocks[block_idx].is_null:
if blocks is None or block_idx >= len(blocks):
return False
blocks[block_idx] = block
if self.get_resident_page(request_id, block_idx) is not None:
return False
self.residency_changes.setdefault(request_id, {})[block_idx] = block
return True
def drop_resident_page(
self, request_id: str, block_idx: int
) -> KVCacheBlock | None:
"""Stop reading a page from the GPU; return the block it was read from."""
block = self.get_resident_page(request_id, block_idx)
if block is not None:
changes = self.residency_changes.setdefault(request_id, {})
changes[block_idx] = self._null_block
return block
def get_resident_page(self, request_id: str, block_idx: int) -> KVCacheBlock | None:
blocks = self.req_to_blocks.get(request_id)
if blocks is None or block_idx >= len(blocks):
return None
block = blocks[block_idx]
changes = self.residency_changes.get(request_id)
if changes:
block = changes.get(block_idx, block)
return None if block.is_null else block
def get_residency_row(self, request_id: str, start: int = 0) -> list[KVCacheBlock]:
"""The request's row from ``start`` with residency changes applied. Unlike
``get_resident_page``, host-only pages stay as the null block."""
blocks = self.req_to_blocks.get(request_id, [])[start:]
changes = self.residency_changes.get(request_id)
if not changes:
return blocks
return [
changes.get(block_idx, block)
for block_idx, block in enumerate(blocks, start)
]
def get_manager_for_kv_cache_spec(
kv_cache_spec: KVCacheSpec,