[HiSparse] Own GPU residency instead of editing core block-table rows (#60647)
Signed-off-by: Matthew Bonanni <mbonanni@redhat.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
9bd074a996
commit
d5f356dae0
@@ -304,9 +304,6 @@ class SchedulerOutput:
|
||||
# CoW copies to apply after zeroing new blocks and before forward.
|
||||
kv_cache_block_copies: list[KVCacheBlockCopy] | None = None
|
||||
|
||||
# Complete block-table rows that replace incrementally appended block IDs.
|
||||
block_table_updates: dict[str, tuple[list[int], ...]] | None = None
|
||||
|
||||
# Scheduler-local; always None by the time this reaches a worker.
|
||||
kv_connector_block_state: KVConnectorBlockState | None = None
|
||||
|
||||
|
||||
@@ -2503,6 +2503,10 @@ class HiSparseResidentManager(_HiSparseAuxiliaryManager):
|
||||
|
||||
max_admission_blocks_per_request: int
|
||||
|
||||
def __init__(self, kv_cache_spec: HiSparseResidentSpec, **kwargs) -> None:
|
||||
super().__init__(kv_cache_spec, **kwargs)
|
||||
self.residency_changes: dict[str, dict[int, KVCacheBlock]] = {}
|
||||
|
||||
def get_num_blocks_to_allocate(
|
||||
self,
|
||||
request_id: str,
|
||||
@@ -2546,7 +2550,7 @@ class HiSparseResidentManager(_HiSparseAuxiliaryManager):
|
||||
tail_page = (num_tokens - 1) // self.block_size
|
||||
if len(blocks) <= tail_page:
|
||||
blocks.extend([self._null_block] * (tail_page + 1 - len(blocks)))
|
||||
if blocks[tail_page].is_null:
|
||||
if self.get_resident_page(request_id, tail_page) is None:
|
||||
blocks[tail_page] = self.block_pool.get_new_blocks(1)[0]
|
||||
|
||||
def add_local_computed_blocks(
|
||||
@@ -2580,25 +2584,55 @@ class HiSparseResidentManager(_HiSparseAuxiliaryManager):
|
||||
def pop_blocks_for_free(self, request_id: str) -> list[KVCacheBlock]:
|
||||
assert self.coordinator is not None
|
||||
self.coordinator.free(request_id)
|
||||
return super().pop_blocks_for_free(request_id)
|
||||
blocks = self.get_residency_row(request_id)
|
||||
self.residency_changes.pop(request_id, None)
|
||||
super().pop_blocks_for_free(request_id)
|
||||
return blocks
|
||||
|
||||
def adopt_resident_page(
|
||||
self, request_id: str, block_idx: int, block: KVCacheBlock
|
||||
) -> bool:
|
||||
"""Point a null prefix page at a pinned GPU copy of its contents."""
|
||||
blocks = self.req_to_blocks.get(request_id)
|
||||
if blocks is None or block_idx >= len(blocks) or not blocks[block_idx].is_null:
|
||||
if blocks is None or block_idx >= len(blocks):
|
||||
return False
|
||||
blocks[block_idx] = block
|
||||
if self.get_resident_page(request_id, block_idx) is not None:
|
||||
return False
|
||||
self.residency_changes.setdefault(request_id, {})[block_idx] = block
|
||||
return True
|
||||
|
||||
def drop_resident_page(
|
||||
self, request_id: str, block_idx: int
|
||||
) -> KVCacheBlock | None:
|
||||
"""Stop reading a page from the GPU; return the block it was read from."""
|
||||
block = self.get_resident_page(request_id, block_idx)
|
||||
if block is not None:
|
||||
changes = self.residency_changes.setdefault(request_id, {})
|
||||
changes[block_idx] = self._null_block
|
||||
return block
|
||||
|
||||
def get_resident_page(self, request_id: str, block_idx: int) -> KVCacheBlock | None:
|
||||
blocks = self.req_to_blocks.get(request_id)
|
||||
if blocks is None or block_idx >= len(blocks):
|
||||
return None
|
||||
block = blocks[block_idx]
|
||||
changes = self.residency_changes.get(request_id)
|
||||
if changes:
|
||||
block = changes.get(block_idx, block)
|
||||
return None if block.is_null else block
|
||||
|
||||
def get_residency_row(self, request_id: str, start: int = 0) -> list[KVCacheBlock]:
|
||||
"""The request's row from ``start`` with residency changes applied. Unlike
|
||||
``get_resident_page``, host-only pages stay as the null block."""
|
||||
blocks = self.req_to_blocks.get(request_id, [])[start:]
|
||||
changes = self.residency_changes.get(request_id)
|
||||
if not changes:
|
||||
return blocks
|
||||
return [
|
||||
changes.get(block_idx, block)
|
||||
for block_idx, block in enumerate(blocks, start)
|
||||
]
|
||||
|
||||
|
||||
def get_manager_for_kv_cache_spec(
|
||||
kv_cache_spec: KVCacheSpec,
|
||||
|
||||
Reference in New Issue
Block a user