[HiSparse] Own GPU residency instead of editing core block-table rows (#60647)
Signed-off-by: Matthew Bonanni <mbonanni@redhat.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
9bd074a996
commit
d5f356dae0
@@ -22,6 +22,7 @@ from vllm.v1.hisparse.types import (
|
||||
ACTIVE_TAIL_PAGES,
|
||||
SparseKVOffloadCommand,
|
||||
SparseKVPageTransfer,
|
||||
SparseKVResidencyUpdate,
|
||||
SparseKVRowMirror,
|
||||
)
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
@@ -65,6 +66,10 @@ class _HiSparseRequestState:
|
||||
unpinned_pages: set[int] = field(default_factory=set)
|
||||
# Prefix pages whose GPU copies are adopted after the admitting allocation.
|
||||
pages_to_adopt: int = 0
|
||||
# The worker holds this request's residency for every page below
|
||||
# synced_pages, except the stale pages changed since it was sent.
|
||||
synced_pages: int = 0
|
||||
stale_pages: set[int] = field(default_factory=set)
|
||||
|
||||
|
||||
@dataclass
|
||||
@@ -86,9 +91,10 @@ class HiSparseCoordinator:
|
||||
GPU-resident pages are a write-back cache of the host tier. Once a page's
|
||||
host copy is durable and its owner can read from host (it has, or has
|
||||
asked for, a hot region), the page's allocation reference is released with
|
||||
``BlockPool.unpin_blocks``: the block stays in the block table and keeps
|
||||
being read, but the pool counts it as free and may hand it out, at which
|
||||
point the owner's page is nulled and its block table republished.
|
||||
``BlockPool.unpin_blocks``: the block keeps being read, but the pool counts
|
||||
it as free and may hand it out, at which point the owner stops reading it.
|
||||
Residency changes reach the worker through ``take_residency_updates``, not
|
||||
the block table, which stays append-only.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
@@ -167,7 +173,6 @@ class HiSparseCoordinator:
|
||||
self._retained_copies: dict[int, tuple[KVCacheBlock, KVCacheBlock]] = {}
|
||||
# request -> prefix length an external load is still filling in.
|
||||
self._pending_imports: dict[str, int] = {}
|
||||
self.block_table_updates: set[str] = set()
|
||||
# Requests whose resident pages were all non-null when last scanned.
|
||||
# Pages only become null at admission or import (before the first
|
||||
# scan) and when the pool reuses an unpinned page (_lose_page).
|
||||
@@ -269,7 +274,8 @@ class HiSparseCoordinator:
|
||||
if manager.adopt_resident_page(request_id, host_idx, block):
|
||||
manager.block_pool.touch([block])
|
||||
state.pinned_clean.add(host_idx)
|
||||
self.block_table_updates.add(request_id)
|
||||
if host_idx < state.synced_pages:
|
||||
state.stale_pages.add(host_idx)
|
||||
|
||||
def _record_copies(self, request_id: str, num_computed_tokens: int) -> None:
|
||||
"""Index the GPU copies of just-published host blocks for later hits."""
|
||||
@@ -342,14 +348,11 @@ class HiSparseCoordinator:
|
||||
) -> list[KVCacheBlock] | None:
|
||||
blocks: list[KVCacheBlock] = []
|
||||
for manager in self.resident_managers:
|
||||
req_blocks = manager.req_to_blocks.get(request_id)
|
||||
if (
|
||||
req_blocks is None
|
||||
or page_idx >= len(req_blocks) - ACTIVE_TAIL_PAGES
|
||||
or req_blocks[page_idx].is_null
|
||||
):
|
||||
num_pages = len(manager.req_to_blocks.get(request_id, ()))
|
||||
block = manager.get_resident_page(request_id, page_idx)
|
||||
if page_idx >= num_pages - ACTIVE_TAIL_PAGES or block is None:
|
||||
return None
|
||||
blocks.append(req_blocks[page_idx])
|
||||
blocks.append(block)
|
||||
return blocks
|
||||
|
||||
def _unpin_page(
|
||||
@@ -392,16 +395,15 @@ class HiSparseCoordinator:
|
||||
if state is None:
|
||||
return
|
||||
for manager in self.resident_managers:
|
||||
blocks = manager.req_to_blocks.get(request_id)
|
||||
if blocks is None or page_idx >= len(blocks) or blocks[page_idx].is_null:
|
||||
block = manager.drop_resident_page(request_id, page_idx)
|
||||
if block is None:
|
||||
continue
|
||||
block = blocks[page_idx]
|
||||
blocks[page_idx] = manager._null_block
|
||||
owners = self._owners.get(block.block_id)
|
||||
if owners is not None:
|
||||
owners.discard((request_id, page_idx))
|
||||
state.unpinned_pages.discard(page_idx)
|
||||
self.block_table_updates.add(request_id)
|
||||
if page_idx < state.synced_pages:
|
||||
state.stale_pages.add(page_idx)
|
||||
for hot_manager in self.hot_managers:
|
||||
hot_manager.require_hot(request_id)
|
||||
|
||||
@@ -627,11 +629,10 @@ class HiSparseCoordinator:
|
||||
mirrors: list[SparseKVRowMirror] = []
|
||||
for request_id, num_computed_tokens, num_scheduled_tokens in requests:
|
||||
host_blocks = self.host_manager.req_to_blocks.get(request_id)
|
||||
resident_blocks = [
|
||||
manager.req_to_blocks.get(request_id)
|
||||
if host_blocks is None or any(
|
||||
request_id not in manager.req_to_blocks
|
||||
for manager in self.resident_managers
|
||||
]
|
||||
if host_blocks is None or any(blocks is None for blocks in resident_blocks):
|
||||
):
|
||||
continue
|
||||
token_position = num_computed_tokens
|
||||
end_position = token_position + num_scheduled_tokens
|
||||
@@ -644,14 +645,12 @@ class HiSparseCoordinator:
|
||||
# window only ever moves forward, so dropping them here would
|
||||
# leave their host rows permanently stale.
|
||||
source_starts = []
|
||||
for blocks in resident_blocks:
|
||||
assert blocks is not None
|
||||
if page_idx >= len(blocks) or blocks[page_idx].is_null:
|
||||
for manager in self.resident_managers:
|
||||
block = manager.get_resident_page(request_id, page_idx)
|
||||
if block is None:
|
||||
break
|
||||
source_starts.append(
|
||||
blocks[page_idx].block_id * block_size + row_offset
|
||||
)
|
||||
if len(source_starts) != len(resident_blocks):
|
||||
source_starts.append(block.block_id * block_size + row_offset)
|
||||
if len(source_starts) != len(self.resident_managers):
|
||||
token_position += num_rows
|
||||
continue
|
||||
host_block_idx = page_idx
|
||||
@@ -693,21 +692,53 @@ class HiSparseCoordinator:
|
||||
if any(
|
||||
block.is_null
|
||||
for manager in self.resident_managers
|
||||
for block in manager.req_to_blocks.get(request_id, ())
|
||||
for block in manager.get_residency_row(request_id)
|
||||
):
|
||||
return False
|
||||
self._resident_requests.add(request_id)
|
||||
return True
|
||||
|
||||
def take_block_table_updates(self) -> dict[str, tuple[list[int], ...]]:
|
||||
updates = {
|
||||
request_id: tuple(
|
||||
[block.block_id for block in manager.req_to_blocks.get(request_id, [])]
|
||||
for manager in self.managers
|
||||
def take_residency_updates(
|
||||
self, request_ids: Iterable[str]
|
||||
) -> dict[str, SparseKVResidencyUpdate]:
|
||||
"""Resident block ids the worker lacks for the scheduled requests:
|
||||
stale pages plus pages appended since the last update.
|
||||
|
||||
Requests that are not scheduled keep their changes until they are:
|
||||
only a scheduled request's residency is read.
|
||||
"""
|
||||
updates: dict[str, SparseKVResidencyUpdate] = {}
|
||||
if not self.resident_managers:
|
||||
return updates
|
||||
first = self.resident_managers[0]
|
||||
for request_id in request_ids:
|
||||
state = self._get_request_state(request_id)
|
||||
num_pages = len(first.req_to_blocks.get(request_id, ()))
|
||||
start = state.synced_pages
|
||||
if not state.stale_pages and start >= num_pages:
|
||||
continue
|
||||
stale_pages = sorted(state.stale_pages)
|
||||
block_ids: list[list[int]] = []
|
||||
for manager in self.resident_managers:
|
||||
null_block_id = manager.block_pool.null_block.block_id
|
||||
group_block_ids = [
|
||||
null_block_id if block is None else block.block_id
|
||||
for block in (
|
||||
manager.get_resident_page(request_id, page_idx)
|
||||
for page_idx in stale_pages
|
||||
)
|
||||
]
|
||||
group_block_ids.extend(
|
||||
block.block_id
|
||||
for block in manager.get_residency_row(request_id, start)
|
||||
)
|
||||
block_ids.append(group_block_ids)
|
||||
updates[request_id] = SparseKVResidencyUpdate(
|
||||
pages=[*stale_pages, *range(start, num_pages)],
|
||||
block_ids=tuple(block_ids),
|
||||
)
|
||||
for request_id in self.block_table_updates
|
||||
}
|
||||
self.block_table_updates.clear()
|
||||
state.stale_pages.clear()
|
||||
state.synced_pages = num_pages
|
||||
return updates
|
||||
|
||||
def build_offload_command(self) -> SparseKVOffloadCommand | None:
|
||||
@@ -846,9 +877,7 @@ class HiSparseCoordinator:
|
||||
# planned, so its copies do not prove that the KV was computed.
|
||||
state.publication = None
|
||||
for manager in self.resident_managers:
|
||||
blocks = manager.req_to_blocks.get(request_id)
|
||||
if not blocks:
|
||||
continue
|
||||
blocks = manager.get_residency_row(request_id)
|
||||
# Tail first, so a prefix loses its tail pages before its head.
|
||||
for page_idx in reversed(range(len(blocks))):
|
||||
block = blocks[page_idx]
|
||||
@@ -865,7 +894,7 @@ class HiSparseCoordinator:
|
||||
manager.block_pool.unpin_blocks([block], self._on_block_reused)
|
||||
else:
|
||||
continue
|
||||
blocks[page_idx] = manager._null_block
|
||||
manager.drop_resident_page(request_id, page_idx)
|
||||
|
||||
|
||||
def get_hisparse_coordinator(
|
||||
|
||||
Reference in New Issue
Block a user