[HiSparse] Own GPU residency instead of editing core block-table rows (#60647)
Signed-off-by: Matthew Bonanni <mbonanni@redhat.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
9bd074a996
commit
d5f356dae0
@@ -214,10 +214,26 @@ def bind_hisparse_kv_caches(
|
||||
for tensor_config in kv_cache_config.kv_cache_tensors
|
||||
for layer_index, name in enumerate(tensor_config.layers)
|
||||
}
|
||||
resident_source_index = 0
|
||||
for group_id, group in enumerate(kv_cache_config.kv_cache_groups):
|
||||
if not isinstance(group.kv_cache_spec, HiSparseResidentSpec):
|
||||
continue
|
||||
resident_groups = [
|
||||
(group_id, group)
|
||||
for group_id, group in enumerate(kv_cache_config.kv_cache_groups)
|
||||
if isinstance(group.kv_cache_spec, HiSparseResidentSpec)
|
||||
]
|
||||
residency: torch.Tensor | None = None
|
||||
if resident_groups:
|
||||
table_shapes = {
|
||||
block_tables.input_block_tables[group_id].shape
|
||||
for group_id, _ in resident_groups
|
||||
}
|
||||
assert len(table_shapes) == 1
|
||||
max_num_reqs, max_num_pages = table_shapes.pop()
|
||||
residency = torch.zeros(
|
||||
(max_num_reqs, len(resident_groups), max_num_pages),
|
||||
dtype=torch.int32,
|
||||
device=block_tables.input_block_tables[resident_groups[0][0]].device,
|
||||
)
|
||||
for resident_source_index, (group_id, group) in enumerate(resident_groups):
|
||||
assert residency is not None
|
||||
for cache_name in group.layer_names:
|
||||
assert cache_name.endswith(HISPARSE_RESIDENT_SUFFIX)
|
||||
layer_name = cache_name[: -len(HISPARSE_RESIDENT_SUFFIX)]
|
||||
@@ -230,13 +246,13 @@ def bind_hisparse_kv_caches(
|
||||
block_stride=tensor_config.block_stride,
|
||||
num_blocks=kv_cache_config.num_blocks,
|
||||
block_size=group.kv_cache_spec.block_size,
|
||||
block_table=block_tables.input_block_tables[group_id],
|
||||
block_table=residency[:, resident_source_index],
|
||||
slot_mapping=block_tables.slot_mappings[group_id],
|
||||
)
|
||||
cache_handle.residency = residency
|
||||
assert cache_handle.view is not None
|
||||
kv_caches[cache_name] = cache_handle.view.cache
|
||||
cache_handle.runtime.resident_source_index = resident_source_index
|
||||
resident_source_index += 1
|
||||
|
||||
hot_backing: torch.Tensor | None = None
|
||||
cache_handles: list[HiSparseCacheHandle] = []
|
||||
|
||||
@@ -22,6 +22,7 @@ from vllm.v1.hisparse.types import (
|
||||
ACTIVE_TAIL_PAGES,
|
||||
SparseKVOffloadCommand,
|
||||
SparseKVPageTransfer,
|
||||
SparseKVResidencyUpdate,
|
||||
SparseKVRowMirror,
|
||||
)
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
@@ -65,6 +66,10 @@ class _HiSparseRequestState:
|
||||
unpinned_pages: set[int] = field(default_factory=set)
|
||||
# Prefix pages whose GPU copies are adopted after the admitting allocation.
|
||||
pages_to_adopt: int = 0
|
||||
# The worker holds this request's residency for every page below
|
||||
# synced_pages, except the stale pages changed since it was sent.
|
||||
synced_pages: int = 0
|
||||
stale_pages: set[int] = field(default_factory=set)
|
||||
|
||||
|
||||
@dataclass
|
||||
@@ -86,9 +91,10 @@ class HiSparseCoordinator:
|
||||
GPU-resident pages are a write-back cache of the host tier. Once a page's
|
||||
host copy is durable and its owner can read from host (it has, or has
|
||||
asked for, a hot region), the page's allocation reference is released with
|
||||
``BlockPool.unpin_blocks``: the block stays in the block table and keeps
|
||||
being read, but the pool counts it as free and may hand it out, at which
|
||||
point the owner's page is nulled and its block table republished.
|
||||
``BlockPool.unpin_blocks``: the block keeps being read, but the pool counts
|
||||
it as free and may hand it out, at which point the owner stops reading it.
|
||||
Residency changes reach the worker through ``take_residency_updates``, not
|
||||
the block table, which stays append-only.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
@@ -167,7 +173,6 @@ class HiSparseCoordinator:
|
||||
self._retained_copies: dict[int, tuple[KVCacheBlock, KVCacheBlock]] = {}
|
||||
# request -> prefix length an external load is still filling in.
|
||||
self._pending_imports: dict[str, int] = {}
|
||||
self.block_table_updates: set[str] = set()
|
||||
# Requests whose resident pages were all non-null when last scanned.
|
||||
# Pages only become null at admission or import (before the first
|
||||
# scan) and when the pool reuses an unpinned page (_lose_page).
|
||||
@@ -269,7 +274,8 @@ class HiSparseCoordinator:
|
||||
if manager.adopt_resident_page(request_id, host_idx, block):
|
||||
manager.block_pool.touch([block])
|
||||
state.pinned_clean.add(host_idx)
|
||||
self.block_table_updates.add(request_id)
|
||||
if host_idx < state.synced_pages:
|
||||
state.stale_pages.add(host_idx)
|
||||
|
||||
def _record_copies(self, request_id: str, num_computed_tokens: int) -> None:
|
||||
"""Index the GPU copies of just-published host blocks for later hits."""
|
||||
@@ -342,14 +348,11 @@ class HiSparseCoordinator:
|
||||
) -> list[KVCacheBlock] | None:
|
||||
blocks: list[KVCacheBlock] = []
|
||||
for manager in self.resident_managers:
|
||||
req_blocks = manager.req_to_blocks.get(request_id)
|
||||
if (
|
||||
req_blocks is None
|
||||
or page_idx >= len(req_blocks) - ACTIVE_TAIL_PAGES
|
||||
or req_blocks[page_idx].is_null
|
||||
):
|
||||
num_pages = len(manager.req_to_blocks.get(request_id, ()))
|
||||
block = manager.get_resident_page(request_id, page_idx)
|
||||
if page_idx >= num_pages - ACTIVE_TAIL_PAGES or block is None:
|
||||
return None
|
||||
blocks.append(req_blocks[page_idx])
|
||||
blocks.append(block)
|
||||
return blocks
|
||||
|
||||
def _unpin_page(
|
||||
@@ -392,16 +395,15 @@ class HiSparseCoordinator:
|
||||
if state is None:
|
||||
return
|
||||
for manager in self.resident_managers:
|
||||
blocks = manager.req_to_blocks.get(request_id)
|
||||
if blocks is None or page_idx >= len(blocks) or blocks[page_idx].is_null:
|
||||
block = manager.drop_resident_page(request_id, page_idx)
|
||||
if block is None:
|
||||
continue
|
||||
block = blocks[page_idx]
|
||||
blocks[page_idx] = manager._null_block
|
||||
owners = self._owners.get(block.block_id)
|
||||
if owners is not None:
|
||||
owners.discard((request_id, page_idx))
|
||||
state.unpinned_pages.discard(page_idx)
|
||||
self.block_table_updates.add(request_id)
|
||||
if page_idx < state.synced_pages:
|
||||
state.stale_pages.add(page_idx)
|
||||
for hot_manager in self.hot_managers:
|
||||
hot_manager.require_hot(request_id)
|
||||
|
||||
@@ -627,11 +629,10 @@ class HiSparseCoordinator:
|
||||
mirrors: list[SparseKVRowMirror] = []
|
||||
for request_id, num_computed_tokens, num_scheduled_tokens in requests:
|
||||
host_blocks = self.host_manager.req_to_blocks.get(request_id)
|
||||
resident_blocks = [
|
||||
manager.req_to_blocks.get(request_id)
|
||||
if host_blocks is None or any(
|
||||
request_id not in manager.req_to_blocks
|
||||
for manager in self.resident_managers
|
||||
]
|
||||
if host_blocks is None or any(blocks is None for blocks in resident_blocks):
|
||||
):
|
||||
continue
|
||||
token_position = num_computed_tokens
|
||||
end_position = token_position + num_scheduled_tokens
|
||||
@@ -644,14 +645,12 @@ class HiSparseCoordinator:
|
||||
# window only ever moves forward, so dropping them here would
|
||||
# leave their host rows permanently stale.
|
||||
source_starts = []
|
||||
for blocks in resident_blocks:
|
||||
assert blocks is not None
|
||||
if page_idx >= len(blocks) or blocks[page_idx].is_null:
|
||||
for manager in self.resident_managers:
|
||||
block = manager.get_resident_page(request_id, page_idx)
|
||||
if block is None:
|
||||
break
|
||||
source_starts.append(
|
||||
blocks[page_idx].block_id * block_size + row_offset
|
||||
)
|
||||
if len(source_starts) != len(resident_blocks):
|
||||
source_starts.append(block.block_id * block_size + row_offset)
|
||||
if len(source_starts) != len(self.resident_managers):
|
||||
token_position += num_rows
|
||||
continue
|
||||
host_block_idx = page_idx
|
||||
@@ -693,21 +692,53 @@ class HiSparseCoordinator:
|
||||
if any(
|
||||
block.is_null
|
||||
for manager in self.resident_managers
|
||||
for block in manager.req_to_blocks.get(request_id, ())
|
||||
for block in manager.get_residency_row(request_id)
|
||||
):
|
||||
return False
|
||||
self._resident_requests.add(request_id)
|
||||
return True
|
||||
|
||||
def take_block_table_updates(self) -> dict[str, tuple[list[int], ...]]:
|
||||
updates = {
|
||||
request_id: tuple(
|
||||
[block.block_id for block in manager.req_to_blocks.get(request_id, [])]
|
||||
for manager in self.managers
|
||||
def take_residency_updates(
|
||||
self, request_ids: Iterable[str]
|
||||
) -> dict[str, SparseKVResidencyUpdate]:
|
||||
"""Resident block ids the worker lacks for the scheduled requests:
|
||||
stale pages plus pages appended since the last update.
|
||||
|
||||
Requests that are not scheduled keep their changes until they are:
|
||||
only a scheduled request's residency is read.
|
||||
"""
|
||||
updates: dict[str, SparseKVResidencyUpdate] = {}
|
||||
if not self.resident_managers:
|
||||
return updates
|
||||
first = self.resident_managers[0]
|
||||
for request_id in request_ids:
|
||||
state = self._get_request_state(request_id)
|
||||
num_pages = len(first.req_to_blocks.get(request_id, ()))
|
||||
start = state.synced_pages
|
||||
if not state.stale_pages and start >= num_pages:
|
||||
continue
|
||||
stale_pages = sorted(state.stale_pages)
|
||||
block_ids: list[list[int]] = []
|
||||
for manager in self.resident_managers:
|
||||
null_block_id = manager.block_pool.null_block.block_id
|
||||
group_block_ids = [
|
||||
null_block_id if block is None else block.block_id
|
||||
for block in (
|
||||
manager.get_resident_page(request_id, page_idx)
|
||||
for page_idx in stale_pages
|
||||
)
|
||||
]
|
||||
group_block_ids.extend(
|
||||
block.block_id
|
||||
for block in manager.get_residency_row(request_id, start)
|
||||
)
|
||||
block_ids.append(group_block_ids)
|
||||
updates[request_id] = SparseKVResidencyUpdate(
|
||||
pages=[*stale_pages, *range(start, num_pages)],
|
||||
block_ids=tuple(block_ids),
|
||||
)
|
||||
for request_id in self.block_table_updates
|
||||
}
|
||||
self.block_table_updates.clear()
|
||||
state.stale_pages.clear()
|
||||
state.synced_pages = num_pages
|
||||
return updates
|
||||
|
||||
def build_offload_command(self) -> SparseKVOffloadCommand | None:
|
||||
@@ -846,9 +877,7 @@ class HiSparseCoordinator:
|
||||
# planned, so its copies do not prove that the KV was computed.
|
||||
state.publication = None
|
||||
for manager in self.resident_managers:
|
||||
blocks = manager.req_to_blocks.get(request_id)
|
||||
if not blocks:
|
||||
continue
|
||||
blocks = manager.get_residency_row(request_id)
|
||||
# Tail first, so a prefix loses its tail pages before its head.
|
||||
for page_idx in reversed(range(len(blocks))):
|
||||
block = blocks[page_idx]
|
||||
@@ -865,7 +894,7 @@ class HiSparseCoordinator:
|
||||
manager.block_pool.unpin_blocks([block], self._on_block_reused)
|
||||
else:
|
||||
continue
|
||||
blocks[page_idx] = manager._null_block
|
||||
manager.drop_resident_page(request_id, page_idx)
|
||||
|
||||
|
||||
def get_hisparse_coordinator(
|
||||
|
||||
@@ -7,10 +7,11 @@ from __future__ import annotations
|
||||
import math
|
||||
import mmap
|
||||
import time
|
||||
from collections.abc import Callable
|
||||
from collections.abc import Callable, Mapping, Sequence
|
||||
from dataclasses import dataclass
|
||||
from typing import TYPE_CHECKING, Any, TypeAlias
|
||||
|
||||
import numpy as np
|
||||
import psutil
|
||||
import torch
|
||||
|
||||
@@ -21,8 +22,9 @@ from vllm.logger import init_logger
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.math_utils import cdiv, round_up
|
||||
from vllm.utils.mem_utils import get_max_shared_memory_bytes
|
||||
from vllm.utils.torch_utils import current_stream
|
||||
from vllm.utils.torch_utils import async_tensor_h2d, current_stream
|
||||
from vllm.v1.attention.backend import max_decode_query_len
|
||||
from vllm.v1.hisparse.types import SparseKVResidencyUpdate
|
||||
from vllm.v1.kv_offload.cpu.shared_offload_region import SharedOffloadRegion
|
||||
from vllm.v1.simple_kv_offload.cuda_mem_ops import pin_tensor
|
||||
|
||||
@@ -499,16 +501,19 @@ class HiSparsePrefillStagingPlan:
|
||||
|
||||
def ensure_gpu_sources(
|
||||
self,
|
||||
resident_block_table: torch.Tensor,
|
||||
resident_state_rows: torch.Tensor,
|
||||
state_indices: torch.Tensor,
|
||||
resident_block_size: int,
|
||||
) -> None:
|
||||
"""Resolve which staged rows can be served from the resident cache.
|
||||
|
||||
Computed once per plan (the resident block table is shared by every
|
||||
layer in the group); non-null resident pages become miss_mask=0 rows
|
||||
gathered device-to-device by ``gather_prefill_cache``.
|
||||
``resident_state_rows`` is a resident group's persistent table by
|
||||
request state row and ``state_indices`` holds each staged request's
|
||||
state row. Computed once per plan and resident group (every layer in
|
||||
the group shares the table); non-null resident pages become
|
||||
miss_mask=0 rows gathered device-to-device by ``gather_prefill_cache``.
|
||||
"""
|
||||
source_key = (resident_block_table.data_ptr(), resident_block_size)
|
||||
source_key = (resident_state_rows.data_ptr(), resident_block_size)
|
||||
if self.gpu_source_key == source_key:
|
||||
return
|
||||
block_size = self.block_size
|
||||
@@ -519,8 +524,11 @@ class HiSparsePrefillStagingPlan:
|
||||
host_ids = self.row_ids[0].view(num_unique, block_size)[:, 0] // block_size
|
||||
new_bt = self.block_table.to(torch.int64)
|
||||
num_rows, num_cols = new_bt.shape
|
||||
if num_rows == 0 or resident_block_table.shape[0] < num_rows:
|
||||
if num_rows == 0 or state_indices.shape[0] < num_rows:
|
||||
return
|
||||
resident_block_table = resident_state_rows.index_select(
|
||||
0, state_indices[:num_rows].clamp(min=0)
|
||||
)
|
||||
# One representative (row, col) per unique host block: any request
|
||||
# referencing the block holds an equivalent (refcounted) resident view.
|
||||
flat_pos = torch.arange(num_rows * num_cols, device=device)
|
||||
@@ -1107,11 +1115,50 @@ class HiSparseRuntime:
|
||||
return physical_topk_indices
|
||||
|
||||
|
||||
def update_hisparse_residency(
|
||||
residency: torch.Tensor,
|
||||
updates: Mapping[str, SparseKVResidencyUpdate],
|
||||
request_ids: Sequence[str],
|
||||
request_state_indices: torch.Tensor,
|
||||
) -> None:
|
||||
"""Apply scheduled requests' residency changes.
|
||||
|
||||
``residency`` is ``[state rows, resident groups, pages]``: the GPU block of
|
||||
each page, or block 0 when the page is read from the host.
|
||||
"""
|
||||
if not updates:
|
||||
return
|
||||
batch_rows = {request_id: row for row, request_id in enumerate(request_ids)}
|
||||
num_groups = residency.shape[1]
|
||||
rows: list[int] = []
|
||||
pages: list[int] = []
|
||||
block_ids: list[list[int]] = [[] for _ in range(num_groups)]
|
||||
for request_id, update in updates.items():
|
||||
rows.append(batch_rows[request_id])
|
||||
pages.extend(update.pages)
|
||||
for group_block_ids, update_block_ids in zip(
|
||||
block_ids, update.block_ids, strict=True
|
||||
):
|
||||
group_block_ids.extend(update_block_ids)
|
||||
host_values = np.empty((len(pages), 2 + num_groups), dtype=np.int32)
|
||||
host_values[:, 0] = np.repeat(
|
||||
rows, [len(update.pages) for update in updates.values()]
|
||||
)
|
||||
host_values[:, 1] = pages
|
||||
host_values[:, 2:] = np.asarray(block_ids, dtype=np.int32).T
|
||||
values = async_tensor_h2d(
|
||||
host_values, device=request_state_indices.device, dtype=torch.int32
|
||||
)
|
||||
state_rows = request_state_indices[values[:, 0]]
|
||||
residency[state_rows, :, values[:, 1]] = values[:, 2:]
|
||||
|
||||
|
||||
class HiSparseCacheHandle:
|
||||
"""Attention-facing handle for resident KV and sparse offload state."""
|
||||
|
||||
def __init__(self, runtime: HiSparseRuntime) -> None:
|
||||
self.view: PagedCacheView | None = None
|
||||
self.residency: torch.Tensor | None = None
|
||||
self.block_table: torch.Tensor | None = None
|
||||
self.source_block_table: torch.Tensor | None = None
|
||||
self.slot_mapping: torch.Tensor | None = None
|
||||
@@ -1128,6 +1175,7 @@ class HiSparseCacheHandle:
|
||||
# Speculator layers write their rows after the target forward.
|
||||
self.draft_layer = False
|
||||
self.index_group_caches: list[HiSparseCacheHandle] = [self]
|
||||
self._batch_block_table: torch.Tensor | None = None
|
||||
|
||||
def prepare_group_for_batch(self, attn_metadata: Any | None) -> None:
|
||||
assert self.runtime.is_group_leader
|
||||
@@ -1136,6 +1184,7 @@ class HiSparseCacheHandle:
|
||||
|
||||
def _prepare_for_batch(self, attn_metadata: Any | None) -> None:
|
||||
self.dummy_batch = attn_metadata is None
|
||||
self._batch_block_table = None
|
||||
self.runtime.begin_forward()
|
||||
self.num_actual_tokens = (
|
||||
attn_metadata.num_actual_tokens if attn_metadata is not None else 0
|
||||
@@ -1151,6 +1200,17 @@ class HiSparseCacheHandle:
|
||||
not self.decode_batch or self.runtime.eager_host_mirror
|
||||
)
|
||||
|
||||
def batch_block_table(self) -> torch.Tensor:
|
||||
"""Resident rows by this step's batch row, for the prefill paths."""
|
||||
if self._batch_block_table is None:
|
||||
assert self.block_table is not None
|
||||
indices = self.runtime.request_state_indices
|
||||
assert indices is not None
|
||||
self._batch_block_table = self.block_table.index_select(
|
||||
0, indices.clamp(min=0)
|
||||
)
|
||||
return self._batch_block_table
|
||||
|
||||
def write_target(
|
||||
self, num_input_rows: int, num_slot_rows: int
|
||||
) -> tuple[torch.Tensor, torch.Tensor, int]:
|
||||
|
||||
@@ -28,6 +28,15 @@ class SparseKVRowMirror:
|
||||
num_rows: int
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class SparseKVResidencyUpdate:
|
||||
"""GPU block ids of some of a request's resident pages, per resident group.
|
||||
A null block id means the page is read from the host."""
|
||||
|
||||
pages: list[int]
|
||||
block_ids: tuple[list[int], ...]
|
||||
|
||||
|
||||
@dataclass
|
||||
class SparseKVOffloadCommand:
|
||||
"""Opaque scheduler-to-worker command for one model step."""
|
||||
|
||||
Reference in New Issue
Block a user