[3/N] HiSparse: host-resident sparse-MLA decode hot-buffering (#53781)
Signed-off-by: Matthew Bonanni <mbonanni@redhat.com> Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com> Signed-off-by: Lucas Wilkinson <lwilkinson@neuralmagic.com> Signed-off-by: Matthew Bonanni <matthew@neuralmagic.com> Signed-off-by: Nicolò Lucchesi <nicolo.lucchesi@mistral.ai> Co-authored-by: OpenAI Codex <codex@openai.com> Co-authored-by: Lucas Wilkinson <lwilkins@redhat.com> Co-authored-by: Lucas Wilkinson <lwilkinson@neuralmagic.com> Co-authored-by: Nicolò Lucchesi <nicolo.lucchesi@mistral.ai> Co-authored-by: OpenAI Codex <noreply@openai.com> Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com> Co-authored-by: Lucas Wilkinson <LucasWilkinson@users.noreply.github.com>
This commit is contained in:
co-authored by
OpenAI Codex
Lucas Wilkinson
Lucas Wilkinson
Nicolò Lucchesi
OpenAI Codex
Claude Opus 5
Lucas Wilkinson
parent
120ec4ebd2
commit
d43bb2f37f
@@ -0,0 +1,2 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
@@ -0,0 +1,277 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Worker-side wiring of HiSparse caches to attention layers."""
|
||||
|
||||
from collections.abc import Mapping
|
||||
from copy import copy
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.v1.hisparse.layout import (
|
||||
HISPARSE_HOT_SUFFIX,
|
||||
HISPARSE_RESIDENT_SUFFIX,
|
||||
)
|
||||
from vllm.v1.hisparse.runtime import (
|
||||
HiSparseCacheHandle,
|
||||
HiSparseHostPool,
|
||||
initialize_hisparse_runtime_buffers,
|
||||
release_pinned_state,
|
||||
)
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
HiSparseHotSpec,
|
||||
HiSparseResidentSpec,
|
||||
KVCacheConfig,
|
||||
KVCacheLayout,
|
||||
KVCacheSpec,
|
||||
MLAAttentionSpec,
|
||||
UniformTypeKVCacheSpecs,
|
||||
create_kv_cache_views,
|
||||
)
|
||||
from vllm.v1.worker.utils import allocate_kv_cache, select_common_block_size
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.model_executor.layers.attention_layer_base import AttentionLayerBase
|
||||
from vllm.v1.worker.gpu.block_table import BlockTables
|
||||
|
||||
|
||||
def resolve_hisparse_block_size(
|
||||
vllm_config: VllmConfig,
|
||||
kv_cache_spec: dict[str, KVCacheSpec],
|
||||
attn_layers: Mapping[str, "AttentionLayerBase"],
|
||||
) -> None:
|
||||
"""Resolve a common kernel block size in-place for HiSparse MLA specs."""
|
||||
if vllm_config.attention_config.hisparse_config is None:
|
||||
return
|
||||
mla_specs = {
|
||||
name: spec
|
||||
for name, spec in kv_cache_spec.items()
|
||||
if isinstance(spec, MLAAttentionSpec)
|
||||
}
|
||||
if not mla_specs:
|
||||
return
|
||||
block_sizes = {spec.block_size for spec in mla_specs.values()}
|
||||
if len(block_sizes) != 1:
|
||||
raise ValueError("HiSparse requires one scheduler block size.")
|
||||
backends = [attn_layers[name].get_attn_backend() for name in mla_specs]
|
||||
try:
|
||||
block_size = select_common_block_size(block_sizes.pop(), backends)
|
||||
except ValueError as error:
|
||||
raise ValueError(
|
||||
"HiSparse requires a GPU block size supported by every sparse "
|
||||
f"attention and indexer backend: {error}"
|
||||
) from error
|
||||
kv_cache_spec.update(
|
||||
(name, spec.copy_with_new_block_size(block_size))
|
||||
for name, spec in mla_specs.items()
|
||||
)
|
||||
|
||||
|
||||
def allocate_hisparse_kv_caches(
|
||||
kv_cache_config: KVCacheConfig,
|
||||
device: torch.device,
|
||||
layout: KVCacheLayout,
|
||||
kernel_block_sizes: list[int],
|
||||
host_pool: HiSparseHostPool,
|
||||
) -> dict[str, torch.Tensor]:
|
||||
"""Allocate the host pool separately from the shared device backing."""
|
||||
device_config = copy(kv_cache_config)
|
||||
device_config.kv_cache_tensors = [
|
||||
tensor
|
||||
for tensor in kv_cache_config.kv_cache_tensors
|
||||
if not tensor.host_resident
|
||||
]
|
||||
kv_caches = allocate_kv_cache(device_config, device, layout, kernel_block_sizes)
|
||||
host_tensors = [
|
||||
tensor for tensor in kv_cache_config.kv_cache_tensors if tensor.host_resident
|
||||
]
|
||||
(host_size,) = {tensor.size for tensor in host_tensors}
|
||||
backing = host_pool.allocate(host_size)
|
||||
(host_group_id,) = kv_cache_config.host_group_ids
|
||||
host_spec = kv_cache_config.kv_cache_groups[host_group_id].kv_cache_spec
|
||||
for tensor in host_tensors:
|
||||
spec = (
|
||||
host_spec.kv_cache_specs[tensor.layers[0]]
|
||||
if isinstance(host_spec, UniformTypeKVCacheSpecs)
|
||||
else host_spec
|
||||
)
|
||||
kernel_block_size = kernel_block_sizes[host_group_id]
|
||||
if isinstance(spec, MLAAttentionSpec) and spec.storage_block_size is not None:
|
||||
kernel_block_size = spec.storage_block_size
|
||||
views = create_kv_cache_views(
|
||||
backing,
|
||||
spec,
|
||||
kv_cache_config.num_blocks_of(tensor),
|
||||
layout,
|
||||
tensor,
|
||||
kernel_block_size=kernel_block_size,
|
||||
)
|
||||
kv_caches.update(zip(tensor.layers, views))
|
||||
return kv_caches
|
||||
|
||||
|
||||
def init_hisparse_kv_cache(
|
||||
kv_cache_config: KVCacheConfig,
|
||||
device: torch.device,
|
||||
kernel_block_sizes: list[int],
|
||||
vllm_config: VllmConfig,
|
||||
forward_context: dict[str, Any],
|
||||
block_tables: "BlockTables",
|
||||
) -> dict[str, torch.Tensor]:
|
||||
"""Allocate and bind HiSparse caches within the caller's allocation context."""
|
||||
host_pool = HiSparseHostPool()
|
||||
kv_caches = allocate_hisparse_kv_caches(
|
||||
kv_cache_config,
|
||||
device,
|
||||
vllm_config.cache_config.get_resolved_kv_cache_layout(),
|
||||
kernel_block_sizes,
|
||||
host_pool,
|
||||
)
|
||||
cache_handles = bind_hisparse_kv_caches(
|
||||
forward_context=forward_context,
|
||||
kv_cache_config=kv_cache_config,
|
||||
kv_caches=kv_caches,
|
||||
block_tables=block_tables,
|
||||
host_pool=host_pool,
|
||||
)
|
||||
initialize_hisparse_runtime_buffers(
|
||||
cache_handles,
|
||||
max_num_reqs=vllm_config.scheduler_config.max_num_seqs,
|
||||
max_num_batched_tokens=vllm_config.scheduler_config.max_num_batched_tokens,
|
||||
)
|
||||
return kv_caches
|
||||
|
||||
|
||||
def _get_hisparse_cache(
|
||||
forward_context: dict[str, Any], layer_name: str
|
||||
) -> HiSparseCacheHandle:
|
||||
attention_layer = forward_context[layer_name]
|
||||
hisparse_cache = attention_layer.hisparse_cache
|
||||
assert hisparse_cache is not None
|
||||
return hisparse_cache
|
||||
|
||||
|
||||
def release_hisparse_profiling_cache(forward_context: dict[str, Any]) -> None:
|
||||
cache_handles = [
|
||||
cache
|
||||
for layer in forward_context.values()
|
||||
if (cache := getattr(layer, "hisparse_cache", None)) is not None
|
||||
]
|
||||
runtimes = {
|
||||
id(cache.runtime): cache.runtime
|
||||
for cache in cache_handles
|
||||
if hasattr(cache.runtime, "_host_cache")
|
||||
}
|
||||
if not runtimes:
|
||||
return
|
||||
|
||||
registered_pools = list(
|
||||
{
|
||||
runtime.registered_host_pool.data_ptr(): runtime.registered_host_pool
|
||||
for runtime in runtimes.values()
|
||||
}.values()
|
||||
)
|
||||
release_pinned_state(list(runtimes.values()), registered_pools)
|
||||
for cache in cache_handles:
|
||||
cache.mirror_staging_cache = None
|
||||
cache.mirror_staging_slots = None
|
||||
|
||||
|
||||
def bind_hisparse_kv_caches(
|
||||
*,
|
||||
forward_context: dict[str, Any],
|
||||
kv_cache_config: KVCacheConfig,
|
||||
kv_caches: dict[str, torch.Tensor],
|
||||
block_tables: "BlockTables",
|
||||
host_pool: HiSparseHostPool,
|
||||
) -> list[HiSparseCacheHandle]:
|
||||
"""Bind existing cache storage and block tables; return the bound handles."""
|
||||
assert host_pool.registered is not None
|
||||
tensor_configs = {
|
||||
name: tensor_config
|
||||
for tensor_config in kv_cache_config.kv_cache_tensors
|
||||
for name in tensor_config.layers
|
||||
}
|
||||
resident_source_index = 0
|
||||
for group_id, group in enumerate(kv_cache_config.kv_cache_groups):
|
||||
if not isinstance(group.kv_cache_spec, HiSparseResidentSpec):
|
||||
continue
|
||||
for cache_name in group.layer_names:
|
||||
assert cache_name.endswith(HISPARSE_RESIDENT_SUFFIX)
|
||||
layer_name = cache_name[: -len(HISPARSE_RESIDENT_SUFFIX)]
|
||||
tensor_config = tensor_configs[cache_name]
|
||||
assert not tensor_config.host_resident
|
||||
cache_handle = _get_hisparse_cache(forward_context, layer_name)
|
||||
cache_handle.bind_cache(
|
||||
kv_caches[cache_name],
|
||||
byte_offset=tensor_config.offset,
|
||||
block_stride=tensor_config.block_stride,
|
||||
num_blocks=kv_cache_config.num_blocks,
|
||||
block_size=group.kv_cache_spec.block_size,
|
||||
block_table=block_tables.input_block_tables[group_id],
|
||||
slot_mapping=block_tables.slot_mappings[group_id],
|
||||
)
|
||||
assert cache_handle.view is not None
|
||||
kv_caches[cache_name] = cache_handle.view.cache
|
||||
cache_handle.runtime.resident_source_index = resident_source_index
|
||||
resident_source_index += 1
|
||||
|
||||
hot_backing: torch.Tensor | None = None
|
||||
cache_handles: list[HiSparseCacheHandle] = []
|
||||
for group_id, group in enumerate(kv_cache_config.kv_cache_groups):
|
||||
if not isinstance(group.kv_cache_spec, HiSparseHotSpec):
|
||||
continue
|
||||
for cache_name in group.layer_names:
|
||||
assert cache_name.endswith(HISPARSE_HOT_SUFFIX)
|
||||
raw_tensor = kv_caches[cache_name]
|
||||
if hot_backing is None:
|
||||
hot_backing = raw_tensor
|
||||
elif hot_backing.untyped_storage().data_ptr() != (
|
||||
raw_tensor.untyped_storage().data_ptr()
|
||||
):
|
||||
raise RuntimeError("HiSparse hot tensors must share one GPU backing.")
|
||||
layer_name = cache_name[: -len(HISPARSE_HOT_SUFFIX)]
|
||||
cache_handle = _get_hisparse_cache(forward_context, layer_name)
|
||||
tensor_config = tensor_configs[cache_name]
|
||||
assert not tensor_config.host_resident
|
||||
cache_handle.runtime.bind_hot_cache(
|
||||
raw_tensor,
|
||||
byte_offset=tensor_config.offset,
|
||||
block_stride=tensor_config.block_stride,
|
||||
num_blocks=kv_cache_config.num_blocks,
|
||||
block_size=group.kv_cache_spec.block_size,
|
||||
block_table=block_tables.input_block_tables[group_id],
|
||||
)
|
||||
resident = cache_handle.view
|
||||
hot = cache_handle.runtime.hot
|
||||
assert resident is not None
|
||||
if (
|
||||
resident.cache.untyped_storage().data_ptr()
|
||||
!= hot.cache.untyped_storage().data_ptr()
|
||||
or resident.cache.data_ptr() != hot.cache.data_ptr()
|
||||
or resident.cache.stride() != hot.cache.stride()
|
||||
):
|
||||
raise RuntimeError("HiSparse resident and hot layouts must match.")
|
||||
source_cache = kv_caches[layer_name]
|
||||
assert source_cache.untyped_storage().data_ptr() == (
|
||||
host_pool.registered.untyped_storage().data_ptr()
|
||||
)
|
||||
cache_handle.runtime.bind_source_cache(
|
||||
source_cache,
|
||||
registered_host_pool=host_pool.registered,
|
||||
)
|
||||
cache_handles.append(cache_handle)
|
||||
# The hot slab is raw storage owned by the runtime, not a layer
|
||||
# cache: nothing downstream binds or registers it.
|
||||
del kv_caches[cache_name]
|
||||
|
||||
if hot_backing is None or not cache_handles:
|
||||
raise RuntimeError("HiSparse found no hot-cache handles.")
|
||||
(source_group_id,) = kv_cache_config.host_group_ids
|
||||
for cache_handle in cache_handles:
|
||||
cache_handle.source_block_table = block_tables.input_block_tables[
|
||||
source_group_id
|
||||
]
|
||||
cache_handle.mirror_slot_mapping = block_tables.slot_mappings[source_group_id]
|
||||
return cache_handles
|
||||
@@ -0,0 +1,27 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from vllm.distributed.kv_events import KVCacheEvent
|
||||
from vllm.v1.core.block_pool import BlockPool
|
||||
|
||||
|
||||
class SharedEventQueueBlockPool(BlockPool):
|
||||
"""A pool that publishes into another pool's live KV event queue.
|
||||
|
||||
The owner rebinds its queue on every drain, so this reads it through the
|
||||
owner rather than holding a reference.
|
||||
"""
|
||||
|
||||
def __init__(self, *args, event_owner: BlockPool, **kwargs) -> None:
|
||||
self._event_owner = event_owner
|
||||
super().__init__(*args, **kwargs)
|
||||
|
||||
@property
|
||||
def kv_event_queue(self) -> list[KVCacheEvent]:
|
||||
return self._event_owner.kv_event_queue
|
||||
|
||||
@kv_event_queue.setter
|
||||
def kv_event_queue(self, events: list[KVCacheEvent]) -> None:
|
||||
# ``BlockPool.__init__`` seeds an empty queue and ``take_events``
|
||||
# swaps in a fresh one; both belong to the owner, which drains it.
|
||||
assert not events
|
||||
@@ -0,0 +1,752 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from collections.abc import Iterable, Mapping, Sequence
|
||||
from dataclasses import dataclass, field
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from vllm.utils.math_utils import cdiv
|
||||
from vllm.v1.core.block_pool import BlockPool
|
||||
from vllm.v1.core.kv_cache_utils import (
|
||||
BlockHashWithGroupId,
|
||||
KVCacheBlock,
|
||||
KVCacheBlockCopy,
|
||||
)
|
||||
from vllm.v1.core.single_type_kv_cache_manager import (
|
||||
HiSparseHotManager,
|
||||
HiSparseResidentManager,
|
||||
HiSparseSourceManager,
|
||||
SingleTypeKVCacheManager,
|
||||
)
|
||||
from vllm.v1.hisparse.types import (
|
||||
SparseKVOffloadCommand,
|
||||
SparseKVPageTransfer,
|
||||
SparseKVRowMirror,
|
||||
)
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
HiSparseResidentSpec,
|
||||
KVCacheConfig,
|
||||
)
|
||||
from vllm.v1.request import Request
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.v1.core.kv_cache_manager import KVCacheManager
|
||||
|
||||
# Sealed pages this many positions behind the block-table tail stay pinned so
|
||||
# a page written by an in-flight step is never handed out under it.
|
||||
_ACTIVE_TAIL_PAGES = 2
|
||||
|
||||
|
||||
@dataclass
|
||||
class _PendingPublication:
|
||||
request: Request
|
||||
num_computed_tokens: int
|
||||
num_pages: int
|
||||
retention_interval: int | None
|
||||
replay_boundaries: Sequence[int]
|
||||
|
||||
|
||||
@dataclass
|
||||
class _HiSparseRequestState:
|
||||
"""Residency bookkeeping for one request.
|
||||
|
||||
A resident page moves through three states: dirty (only the GPU copy
|
||||
exists), clean (``valid_pages``: the host copy is complete) and unpinned
|
||||
(``unpinned_pages``: clean, and its allocation reference was released so
|
||||
the pool may reuse it). ``pinned_clean`` tracks clean pages still holding
|
||||
their reference, which only happens before the request can read from host.
|
||||
"""
|
||||
|
||||
valid_pages: set[int] = field(default_factory=set)
|
||||
ready_prefix_pages: int = 0
|
||||
pending_pages: dict[int, int] = field(default_factory=dict)
|
||||
publication: _PendingPublication | None = None
|
||||
copies_recorded_blocks: int = 0
|
||||
pinned_clean: set[int] = field(default_factory=set)
|
||||
unpinned_pages: set[int] = field(default_factory=set)
|
||||
|
||||
|
||||
@dataclass
|
||||
class _PendingSpill:
|
||||
transfer_id: int
|
||||
page: tuple[str, int]
|
||||
request_state: _HiSparseRequestState
|
||||
host_block: KVCacheBlock
|
||||
resident_blocks: tuple[KVCacheBlock, ...]
|
||||
expected_worker_completions: int = 0
|
||||
worker_completions: int = 0
|
||||
enqueue_applied: bool = False
|
||||
|
||||
|
||||
class HiSparseCoordinator:
|
||||
"""Own HiSparse host allocation, host publication, spills, and GPU residency.
|
||||
|
||||
GPU-resident pages are a write-back cache of the host tier. Once a page's
|
||||
host copy is durable and its owner can read from host (it has, or has
|
||||
asked for, a hot region), the page's allocation reference is released with
|
||||
``BlockPool.unpin_blocks``: the block stays in the block table and keeps
|
||||
being read, but the pool counts it as free and may hand it out, at which
|
||||
point the owner's page is nulled and its block table republished.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
kv_cache_config: KVCacheConfig,
|
||||
managers: tuple[SingleTypeKVCacheManager, ...],
|
||||
max_model_len: int,
|
||||
) -> None:
|
||||
self.managers = managers
|
||||
self.max_model_len = max_model_len
|
||||
groups = kv_cache_config.kv_cache_groups
|
||||
|
||||
resident_managers: list[HiSparseResidentManager] = []
|
||||
hot_managers: list[HiSparseHotManager] = []
|
||||
resident_group_ids: list[int] = []
|
||||
for group_id, (group, manager) in enumerate(zip(groups, managers)):
|
||||
if isinstance(group.kv_cache_spec, HiSparseResidentSpec):
|
||||
if not isinstance(manager, HiSparseResidentManager):
|
||||
raise TypeError(
|
||||
"HiSparse resident specs require resident cache managers."
|
||||
)
|
||||
resident_managers.append(manager)
|
||||
resident_group_ids.append(group_id)
|
||||
assert not group.host_resident
|
||||
if isinstance(manager, HiSparseHotManager):
|
||||
hot_managers.append(manager)
|
||||
assert not group.host_resident
|
||||
if isinstance(manager, (HiSparseHotManager, HiSparseResidentManager)):
|
||||
manager.coordinator = self
|
||||
self.resident_managers = tuple(resident_managers)
|
||||
self.hot_managers = tuple(hot_managers)
|
||||
self.host_manager: HiSparseSourceManager | None = None
|
||||
self.host_group_id: int | None = None
|
||||
self.max_spill_pages = 0
|
||||
for group_id, manager in enumerate(managers):
|
||||
if isinstance(manager, HiSparseSourceManager):
|
||||
if self.host_manager is not None:
|
||||
raise ValueError("Only one HiSparse host group is supported.")
|
||||
num_host_blocks = kv_cache_config.hisparse_host_num_blocks
|
||||
if num_host_blocks is None:
|
||||
raise ValueError("HiSparse host group needs host capacity.")
|
||||
self.host_group_id = group_id
|
||||
self.host_manager = manager
|
||||
manager.coordinator = self
|
||||
manager.bind_host_pool(num_host_blocks)
|
||||
self.has_host_cache = self.host_manager is not None
|
||||
self.gpu_pool: BlockPool | None = None
|
||||
self.transition_watermark = 0
|
||||
if self.resident_managers:
|
||||
assert self.host_manager is not None
|
||||
assert self.host_group_id is not None
|
||||
if self.host_group_id > min(resident_group_ids):
|
||||
raise ValueError("HiSparse host group must precede resident groups.")
|
||||
resident_block_sizes = {
|
||||
manager.block_size for manager in self.resident_managers
|
||||
}
|
||||
if len(resident_block_sizes) != 1:
|
||||
raise ValueError(
|
||||
"HiSparse resident cache groups must use one block size."
|
||||
)
|
||||
resident_block_size = resident_block_sizes.pop()
|
||||
if self.host_manager.block_size != resident_block_size:
|
||||
raise ValueError("HiSparse host and resident block sizes must match.")
|
||||
self.max_spill_pages = max_model_len // resident_block_size
|
||||
self.gpu_pool = self.resident_managers[0].block_pool
|
||||
# Requests start reading from host once the shared pool runs this
|
||||
# low, so admissions find evictable pages rather than pinned ones.
|
||||
hot_cost = sum(manager.blocks_per_request for manager in hot_managers)
|
||||
self.transition_watermark = max(hot_cost, kv_cache_config.num_blocks // 10)
|
||||
|
||||
# Host copies handed to the worker, by destination block id, until it
|
||||
# reports having run them.
|
||||
self._retained_copies: dict[int, tuple[KVCacheBlock, KVCacheBlock]] = {}
|
||||
# request -> prefix length an external load is still filling in.
|
||||
self._pending_imports: dict[str, int] = {}
|
||||
self.block_table_updates: set[str] = set()
|
||||
self.spills_to_send: list[SparseKVPageTransfer] = []
|
||||
self.pending_spills: dict[int, _PendingSpill] = {}
|
||||
self.request_states: dict[str, _HiSparseRequestState] = {}
|
||||
self.next_spill_id = 0
|
||||
# Published host block hash -> GPU copies of that page, readable until
|
||||
# the pool reuses them. Lets a host prefix hit come back GPU-resident.
|
||||
self.copies: dict[BlockHashWithGroupId, tuple[KVCacheBlock, ...]] = {}
|
||||
self._copy_by_block: dict[int, BlockHashWithGroupId] = {}
|
||||
# Unpinned GPU block id -> (request, page) pairs still reading it.
|
||||
self._owners: dict[int, set[tuple[str, int]]] = {}
|
||||
|
||||
def _get_request_state(self, request_id: str) -> _HiSparseRequestState:
|
||||
state = self.request_states.get(request_id)
|
||||
if state is None:
|
||||
state = _HiSparseRequestState()
|
||||
self.request_states[request_id] = state
|
||||
return state
|
||||
|
||||
def commit_computed_blocks(self, request_id: str, num_host_pages: int) -> None:
|
||||
"""Account for a prefix hit on host pages the request now references."""
|
||||
if not self.resident_managers or self.host_manager is None:
|
||||
return
|
||||
host_blocks = self.host_manager.req_to_blocks.get(request_id, ())
|
||||
num_host_pages = min(num_host_pages, len(host_blocks))
|
||||
if num_host_pages == 0:
|
||||
return
|
||||
state = self._get_request_state(request_id)
|
||||
state.valid_pages.update(range(num_host_pages))
|
||||
state.ready_prefix_pages = max(num_host_pages, state.ready_prefix_pages)
|
||||
self._adopt_copies(request_id, state, host_blocks[:num_host_pages])
|
||||
state.copies_recorded_blocks = max(state.copies_recorded_blocks, num_host_pages)
|
||||
|
||||
def take_host_block_copies(self) -> tuple[KVCacheBlockCopy, ...]:
|
||||
"""Drain host copy-on-write work, retaining both endpoints.
|
||||
|
||||
The worker runs the copies in the step that carries them, so the
|
||||
endpoints are released when that step's output comes back.
|
||||
"""
|
||||
if self.host_manager is None:
|
||||
return ()
|
||||
pairs = self.host_manager.take_host_cow_copies()
|
||||
for source_block, cow_block in pairs:
|
||||
self._retained_copies[cow_block.block_id] = (source_block, cow_block)
|
||||
return tuple(
|
||||
KVCacheBlockCopy(
|
||||
src_block_id=source_block.block_id,
|
||||
dst_block_id=cow_block.block_id,
|
||||
)
|
||||
for source_block, cow_block in pairs
|
||||
)
|
||||
|
||||
def release_completed_host_copies(self, dst_block_ids: Iterable[int]) -> None:
|
||||
"""Release the endpoints of host copies the worker reports having run."""
|
||||
blocks: list[KVCacheBlock] = []
|
||||
for dst_block_id in dst_block_ids:
|
||||
pair = self._retained_copies.pop(dst_block_id, None)
|
||||
if pair is not None:
|
||||
blocks.extend(pair)
|
||||
if not blocks:
|
||||
return
|
||||
assert self.host_manager is not None
|
||||
self.host_manager.block_pool.free_blocks(reversed(blocks))
|
||||
|
||||
def get_host_block_pool(self) -> BlockPool | None:
|
||||
manager = self.host_manager
|
||||
return manager.block_pool if manager is not None else None
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# GPU copies of published host pages
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def _adopt_copies(
|
||||
self,
|
||||
request_id: str,
|
||||
state: _HiSparseRequestState,
|
||||
host_blocks: Sequence[KVCacheBlock],
|
||||
) -> None:
|
||||
"""Point null prefix pages at readable GPU copies of their contents.
|
||||
|
||||
Runs after the hit length is fully reconciled, so copy hits and misses
|
||||
can be scattered per page without affecting the prefix hit.
|
||||
"""
|
||||
if not self.copies:
|
||||
return
|
||||
for host_idx, host_block in enumerate(host_blocks):
|
||||
if host_block.is_null or host_block.block_hash is None:
|
||||
continue
|
||||
blocks = self.copies.get(host_block.block_hash)
|
||||
if blocks is None:
|
||||
continue
|
||||
for manager, block in zip(self.resident_managers, blocks):
|
||||
if manager.adopt_resident_page(request_id, host_idx, block):
|
||||
manager.block_pool.touch([block])
|
||||
state.pinned_clean.add(host_idx)
|
||||
|
||||
def _record_copies(self, request_id: str, num_computed_tokens: int) -> None:
|
||||
"""Index the GPU copies of just-published host blocks for later hits."""
|
||||
if not self.resident_managers:
|
||||
return
|
||||
assert self.host_manager is not None
|
||||
host_blocks = self.host_manager.req_to_blocks.get(request_id)
|
||||
if not host_blocks:
|
||||
return
|
||||
state = self._get_request_state(request_id)
|
||||
num_host_blocks = min(
|
||||
num_computed_tokens // self.host_manager.block_size, len(host_blocks)
|
||||
)
|
||||
for host_idx in range(state.copies_recorded_blocks, num_host_blocks):
|
||||
host_block = host_blocks[host_idx]
|
||||
if host_block.is_null or host_block.block_hash is None:
|
||||
continue
|
||||
blocks: list[KVCacheBlock] = []
|
||||
for manager in self.resident_managers:
|
||||
block = manager.get_resident_page(request_id, host_idx)
|
||||
if block is None:
|
||||
break
|
||||
blocks.append(block)
|
||||
if len(blocks) != len(self.resident_managers):
|
||||
continue
|
||||
self._drop_copy(host_block.block_hash)
|
||||
self.copies[host_block.block_hash] = tuple(blocks)
|
||||
for block in blocks:
|
||||
self._copy_by_block[block.block_id] = host_block.block_hash
|
||||
state.copies_recorded_blocks = max(
|
||||
state.copies_recorded_blocks, num_host_blocks
|
||||
)
|
||||
|
||||
def _drop_copy(self, host_hash: BlockHashWithGroupId) -> None:
|
||||
blocks = self.copies.pop(host_hash, None)
|
||||
if blocks is None:
|
||||
return
|
||||
for block in blocks:
|
||||
self._copy_by_block.pop(block.block_id, None)
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Residency: pinning, unpinning and losing pages
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def _can_read_from_host(self, request_id: str) -> bool:
|
||||
"""Whether resident pages may be dropped under the request."""
|
||||
return bool(self.hot_managers) and all(
|
||||
manager.has_hot(request_id) for manager in self.hot_managers
|
||||
)
|
||||
|
||||
def _resident_page_blocks(
|
||||
self, request_id: str, page_idx: int
|
||||
) -> list[KVCacheBlock] | None:
|
||||
blocks: list[KVCacheBlock] = []
|
||||
for manager in self.resident_managers:
|
||||
req_blocks = manager.req_to_blocks.get(request_id)
|
||||
if (
|
||||
req_blocks is None
|
||||
or page_idx >= len(req_blocks) - _ACTIVE_TAIL_PAGES
|
||||
or req_blocks[page_idx].is_null
|
||||
):
|
||||
return None
|
||||
blocks.append(req_blocks[page_idx])
|
||||
return blocks
|
||||
|
||||
def _unpin_page(
|
||||
self, request_id: str, state: _HiSparseRequestState, page_idx: int
|
||||
) -> bool:
|
||||
if page_idx in state.pending_pages or page_idx not in state.valid_pages:
|
||||
return False
|
||||
blocks = self._resident_page_blocks(request_id, page_idx)
|
||||
if blocks is None:
|
||||
return False
|
||||
for manager, block in zip(self.resident_managers, blocks):
|
||||
self._owners.setdefault(block.block_id, set()).add((request_id, page_idx))
|
||||
manager.block_pool.unpin_blocks([block], self._on_block_reused)
|
||||
state.pinned_clean.discard(page_idx)
|
||||
state.unpinned_pages.add(page_idx)
|
||||
return True
|
||||
|
||||
def _unpin_clean_pages(self, request_id: str, state: _HiSparseRequestState) -> None:
|
||||
for page_idx in sorted(state.pinned_clean):
|
||||
self._unpin_page(request_id, state, page_idx)
|
||||
|
||||
def _page_became_clean(
|
||||
self, request_id: str, state: _HiSparseRequestState, page_idx: int
|
||||
) -> None:
|
||||
state.pinned_clean.add(page_idx)
|
||||
if self._can_read_from_host(request_id):
|
||||
self._unpin_page(request_id, state, page_idx)
|
||||
|
||||
def _on_block_reused(self, block: KVCacheBlock) -> None:
|
||||
host_hash = self._copy_by_block.get(block.block_id)
|
||||
if host_hash is not None:
|
||||
self._drop_copy(host_hash)
|
||||
for request_id, page_idx in self._owners.pop(block.block_id, ()):
|
||||
self._lose_page(request_id, page_idx)
|
||||
|
||||
def _lose_page(self, request_id: str, page_idx: int) -> None:
|
||||
state = self.request_states.get(request_id)
|
||||
if state is None:
|
||||
return
|
||||
for manager in self.resident_managers:
|
||||
blocks = manager.req_to_blocks.get(request_id)
|
||||
if blocks is None or page_idx >= len(blocks) or blocks[page_idx].is_null:
|
||||
continue
|
||||
block = blocks[page_idx]
|
||||
blocks[page_idx] = manager._null_block
|
||||
owners = self._owners.get(block.block_id)
|
||||
if owners is not None:
|
||||
owners.discard((request_id, page_idx))
|
||||
state.unpinned_pages.discard(page_idx)
|
||||
self.block_table_updates.add(request_id)
|
||||
for hot_manager in self.hot_managers:
|
||||
hot_manager.require_hot(request_id)
|
||||
|
||||
def update_residency(self, request_id: str) -> None:
|
||||
"""Per-step residency policy for a scheduled request.
|
||||
|
||||
A request that can read from host releases every clean sealed page to
|
||||
the pool. One that cannot keeps its pages pinned until the shared pool
|
||||
runs low, then asks for a hot region. Pages remain pinned until that region
|
||||
is allocated on a subsequent scheduling pass.
|
||||
"""
|
||||
if not self.resident_managers:
|
||||
return
|
||||
state = self._get_request_state(request_id)
|
||||
if not self._can_read_from_host(request_id):
|
||||
assert self.gpu_pool is not None
|
||||
if self.gpu_pool.get_num_free_blocks() >= self.transition_watermark:
|
||||
return
|
||||
for manager in self.hot_managers:
|
||||
manager.require_hot(request_id)
|
||||
return
|
||||
self._unpin_clean_pages(request_id, state)
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Host publication and spills
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def plan_prefix_materialization(
|
||||
self, request_id: str, num_computed_tokens: int
|
||||
) -> None:
|
||||
"""Eagerly write back every sealed page that has no host copy yet."""
|
||||
if not self.resident_managers:
|
||||
return
|
||||
assert self.host_manager is not None
|
||||
host_block_size = self.host_manager.block_size
|
||||
num_pages = num_computed_tokens // host_block_size
|
||||
state = self._get_request_state(request_id)
|
||||
budget = max(self.max_spill_pages - len(self.spills_to_send), 0)
|
||||
for page_idx in range(num_pages):
|
||||
if budget == 0:
|
||||
break
|
||||
if page_idx in state.pending_pages:
|
||||
continue
|
||||
if page_idx in state.valid_pages:
|
||||
continue
|
||||
if not all(
|
||||
manager.get_resident_page(request_id, page_idx) is not None
|
||||
for manager in self.resident_managers
|
||||
):
|
||||
continue
|
||||
if self._plan_spill(request_id, page_idx, after_forward=True):
|
||||
budget -= 1
|
||||
|
||||
def publish_when_ready(
|
||||
self,
|
||||
request: Request,
|
||||
num_computed_tokens: int,
|
||||
retention_interval: int | None,
|
||||
*,
|
||||
replay_boundaries: Sequence[int],
|
||||
) -> None:
|
||||
"""Publish host-source hashes only after their pages are durable."""
|
||||
manager = self.host_manager
|
||||
if manager is None:
|
||||
return
|
||||
num_pages = num_computed_tokens // manager.block_size
|
||||
request_id = request.request_id
|
||||
state = self._get_request_state(request_id)
|
||||
if state.ready_prefix_pages >= num_pages:
|
||||
manager.publish_blocks(
|
||||
request,
|
||||
num_computed_tokens,
|
||||
retention_interval=retention_interval,
|
||||
replay_boundaries=replay_boundaries,
|
||||
)
|
||||
self._record_copies(request_id, num_computed_tokens)
|
||||
state.publication = None
|
||||
return
|
||||
state.publication = _PendingPublication(
|
||||
request=request,
|
||||
num_computed_tokens=num_computed_tokens,
|
||||
num_pages=num_pages,
|
||||
retention_interval=retention_interval,
|
||||
replay_boundaries=replay_boundaries,
|
||||
)
|
||||
|
||||
def _publish_host_blocks_if_ready(self, request_id: str) -> None:
|
||||
state = self.request_states.get(request_id)
|
||||
if state is None:
|
||||
return
|
||||
publication = state.publication
|
||||
if publication is None or state.ready_prefix_pages < publication.num_pages:
|
||||
return
|
||||
assert self.host_manager is not None
|
||||
self.host_manager.publish_blocks(
|
||||
publication.request,
|
||||
publication.num_computed_tokens,
|
||||
retention_interval=publication.retention_interval,
|
||||
replay_boundaries=publication.replay_boundaries,
|
||||
)
|
||||
self._record_copies(request_id, publication.num_computed_tokens)
|
||||
state.publication = None
|
||||
|
||||
def record_pending_host_import(self, request_id: str, num_tokens: int) -> None:
|
||||
"""Note a prefix an external load is populating in host pages."""
|
||||
self._pending_imports[request_id] = num_tokens
|
||||
|
||||
def finish_host_import(self, request_id: str, *, failed: bool) -> None:
|
||||
num_tokens = self._pending_imports.pop(request_id, None)
|
||||
if num_tokens is not None and not failed:
|
||||
self.complete_host_import(request_id, num_tokens)
|
||||
|
||||
def complete_host_import(self, request_id: str, num_computed_tokens: int) -> None:
|
||||
"""Publish externally populated host pages after connector completion."""
|
||||
if not self.resident_managers:
|
||||
return
|
||||
block_size = self.resident_managers[0].block_size
|
||||
num_pages = num_computed_tokens // block_size
|
||||
state = self._get_request_state(request_id)
|
||||
state.valid_pages = set(range(num_pages))
|
||||
state.ready_prefix_pages = num_pages
|
||||
self._publish_host_blocks_if_ready(request_id)
|
||||
|
||||
def _plan_spill(
|
||||
self,
|
||||
request_id: str,
|
||||
page_idx: int,
|
||||
*,
|
||||
after_forward: bool,
|
||||
) -> bool:
|
||||
assert self.host_manager is not None
|
||||
state = self._get_request_state(request_id)
|
||||
host_blocks = self.host_manager.req_to_blocks.get(request_id)
|
||||
if host_blocks is None or page_idx >= len(host_blocks):
|
||||
return False
|
||||
host_block = host_blocks[page_idx]
|
||||
if host_block.is_null:
|
||||
return False
|
||||
|
||||
blocks: list[KVCacheBlock] = []
|
||||
for manager in self.resident_managers:
|
||||
block = manager.get_resident_page(request_id, page_idx)
|
||||
if block is None:
|
||||
return False
|
||||
blocks.append(block)
|
||||
|
||||
self.host_manager.block_pool.touch([host_block])
|
||||
for manager, block in zip(self.resident_managers, blocks):
|
||||
manager.block_pool.touch([block])
|
||||
spill_id = self.next_spill_id
|
||||
self.next_spill_id += 1
|
||||
plan = SparseKVPageTransfer(
|
||||
transfer_id=spill_id,
|
||||
destination_block_id=host_block.block_id,
|
||||
source_block_ids=tuple(block.block_id for block in blocks),
|
||||
after_forward=after_forward,
|
||||
)
|
||||
self.pending_spills[spill_id] = _PendingSpill(
|
||||
transfer_id=spill_id,
|
||||
page=(request_id, page_idx),
|
||||
request_state=state,
|
||||
host_block=host_block,
|
||||
resident_blocks=tuple(blocks),
|
||||
)
|
||||
state.pending_pages[page_idx] = spill_id
|
||||
self.spills_to_send.append(plan)
|
||||
return True
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Worker-facing views
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def build_row_mirrors(
|
||||
self,
|
||||
requests: Iterable[tuple[str, int, int]],
|
||||
) -> tuple[SparseKVRowMirror, ...]:
|
||||
"""Map scheduled rows to scheduler-owned resident and host blocks."""
|
||||
if not self.resident_managers or self.host_manager is None:
|
||||
return ()
|
||||
block_size = self.resident_managers[0].block_size
|
||||
mirrors: list[SparseKVRowMirror] = []
|
||||
for request_id, num_computed_tokens, num_scheduled_tokens in requests:
|
||||
host_blocks = self.host_manager.req_to_blocks.get(request_id)
|
||||
resident_blocks = [
|
||||
manager.req_to_blocks.get(request_id)
|
||||
for manager in self.resident_managers
|
||||
]
|
||||
if host_blocks is None or any(blocks is None for blocks in resident_blocks):
|
||||
continue
|
||||
token_position = num_computed_tokens
|
||||
end_position = token_position + num_scheduled_tokens
|
||||
while token_position < end_position:
|
||||
page_idx, row_offset = divmod(token_position, block_size)
|
||||
num_rows = min(block_size - row_offset, end_position - token_position)
|
||||
# A page without a GPU copy has no mirror source, and a page
|
||||
# without a host block has no destination. Skip just that page:
|
||||
# its neighbours in the window are still mirrorable, and a
|
||||
# window only ever moves forward, so dropping them here would
|
||||
# leave their host rows permanently stale.
|
||||
source_starts = []
|
||||
for blocks in resident_blocks:
|
||||
assert blocks is not None
|
||||
if page_idx >= len(blocks) or blocks[page_idx].is_null:
|
||||
break
|
||||
source_starts.append(
|
||||
blocks[page_idx].block_id * block_size + row_offset
|
||||
)
|
||||
if len(source_starts) != len(resident_blocks):
|
||||
token_position += num_rows
|
||||
continue
|
||||
host_block_idx = page_idx
|
||||
if host_block_idx >= len(host_blocks):
|
||||
token_position += num_rows
|
||||
continue
|
||||
host_block = host_blocks[host_block_idx]
|
||||
if host_block.is_null:
|
||||
token_position += num_rows
|
||||
continue
|
||||
mirrors.append(
|
||||
SparseKVRowMirror(
|
||||
source_starts=tuple(source_starts),
|
||||
destination_start=(
|
||||
host_block.block_id * block_size + row_offset
|
||||
),
|
||||
num_rows=num_rows,
|
||||
)
|
||||
)
|
||||
token_position += num_rows
|
||||
return tuple(mirrors)
|
||||
|
||||
def all_context_pages_resident(
|
||||
self,
|
||||
requests: Iterable[tuple[str, int, int]],
|
||||
) -> bool:
|
||||
"""Return whether every scheduled request can read only resident KV."""
|
||||
if not self.resident_managers:
|
||||
return False
|
||||
block_size = self.resident_managers[0].block_size
|
||||
for request_id, num_computed_tokens, num_scheduled_tokens in requests:
|
||||
num_pages = cdiv(num_computed_tokens + num_scheduled_tokens, block_size)
|
||||
for page_idx in range(num_pages):
|
||||
if any(
|
||||
manager.get_resident_page(request_id, page_idx) is None
|
||||
for manager in self.resident_managers
|
||||
):
|
||||
return False
|
||||
return True
|
||||
|
||||
def take_block_table_updates(self) -> dict[str, tuple[list[int], ...]]:
|
||||
updates = {
|
||||
request_id: tuple(
|
||||
[block.block_id for block in manager.req_to_blocks.get(request_id, [])]
|
||||
for manager in self.managers
|
||||
)
|
||||
for request_id in self.block_table_updates
|
||||
}
|
||||
self.block_table_updates.clear()
|
||||
return updates
|
||||
|
||||
def build_offload_command(self) -> SparseKVOffloadCommand | None:
|
||||
"""Package residency decisions for the worker connector."""
|
||||
if not self.resident_managers:
|
||||
return None
|
||||
command = SparseKVOffloadCommand(page_transfers=self.spills_to_send)
|
||||
self.spills_to_send = []
|
||||
return command
|
||||
|
||||
def has_pending_reclamation(self) -> bool:
|
||||
"""Whether an in-flight spill will free GPU blocks on completion."""
|
||||
return any(
|
||||
pending.page[0] in self.request_states
|
||||
and self._can_read_from_host(pending.page[0])
|
||||
for pending in self.pending_spills.values()
|
||||
)
|
||||
|
||||
def has_pending_work(self) -> bool:
|
||||
return bool(self.spills_to_send or self.pending_spills or self._retained_copies)
|
||||
|
||||
def update_spills(
|
||||
self,
|
||||
enqueued_counts: Mapping[int, int],
|
||||
completed_counts: Mapping[int, int],
|
||||
) -> None:
|
||||
for spill_id, count in enqueued_counts.items():
|
||||
pending = self.pending_spills.get(spill_id)
|
||||
if pending is not None and pending.expected_worker_completions == 0:
|
||||
pending.expected_worker_completions = count
|
||||
for spill_id, count in completed_counts.items():
|
||||
pending = self.pending_spills.get(spill_id)
|
||||
if pending is not None:
|
||||
pending.worker_completions += count
|
||||
|
||||
self._apply_enqueued_spills()
|
||||
self._complete_host_writes()
|
||||
|
||||
def _apply_enqueued_spills(self) -> None:
|
||||
"""Drop the DMA pins once the worker has enqueued a spill."""
|
||||
for pending in self.pending_spills.values():
|
||||
if (
|
||||
pending.enqueue_applied
|
||||
or not pending.expected_worker_completions
|
||||
or pending.worker_completions < pending.expected_worker_completions
|
||||
):
|
||||
continue
|
||||
for manager, block in zip(self.resident_managers, pending.resident_blocks):
|
||||
manager.block_pool.free_blocks([block])
|
||||
pending.enqueue_applied = True
|
||||
|
||||
def _complete_host_writes(self) -> None:
|
||||
completed = [
|
||||
pending
|
||||
for pending in self.pending_spills.values()
|
||||
if pending.enqueue_applied
|
||||
and pending.expected_worker_completions
|
||||
and pending.worker_completions >= pending.expected_worker_completions
|
||||
]
|
||||
completed_request_ids: set[str] = set()
|
||||
for pending in completed:
|
||||
self.pending_spills.pop(pending.transfer_id, None)
|
||||
request_id, page_idx = pending.page
|
||||
pending.request_state.pending_pages.pop(page_idx, None)
|
||||
if self.request_states.get(request_id) is pending.request_state:
|
||||
pending.request_state.valid_pages.add(page_idx)
|
||||
self._page_became_clean(request_id, pending.request_state, page_idx)
|
||||
completed_request_ids.add(request_id)
|
||||
assert self.host_manager is not None
|
||||
self.host_manager.block_pool.free_blocks([pending.host_block])
|
||||
for request_id in completed_request_ids:
|
||||
state = self.request_states[request_id]
|
||||
while state.ready_prefix_pages in state.valid_pages:
|
||||
state.ready_prefix_pages += 1
|
||||
self._publish_host_blocks_if_ready(request_id)
|
||||
|
||||
def free(self, request_id: str) -> None:
|
||||
"""Detach the request; its clean pages stay readable copies in the pool."""
|
||||
self._pending_imports.pop(request_id, None)
|
||||
state = self.request_states.pop(request_id, None)
|
||||
if state is None:
|
||||
return
|
||||
for manager in self.resident_managers:
|
||||
blocks = manager.req_to_blocks.get(request_id)
|
||||
if not blocks:
|
||||
continue
|
||||
for page_idx, block in enumerate(blocks):
|
||||
if block.is_null:
|
||||
continue
|
||||
if page_idx in state.unpinned_pages:
|
||||
owners = self._owners.get(block.block_id)
|
||||
if owners is not None:
|
||||
owners.discard((request_id, page_idx))
|
||||
elif (
|
||||
page_idx in state.valid_pages
|
||||
and page_idx not in state.pending_pages
|
||||
):
|
||||
manager.block_pool.unpin_blocks([block], self._on_block_reused)
|
||||
else:
|
||||
continue
|
||||
blocks[page_idx] = manager._null_block
|
||||
|
||||
|
||||
def get_hisparse_coordinator(
|
||||
kv_cache_manager: "KVCacheManager",
|
||||
) -> HiSparseCoordinator:
|
||||
"""Return the coordinator shared by a KV cache manager's HiSparse groups.
|
||||
|
||||
Built on first call and memoised on the HiSparse managers it binds itself
|
||||
to. Must run before the first request is admitted.
|
||||
"""
|
||||
managers = tuple(kv_cache_manager.coordinator.single_type_managers)
|
||||
for manager in managers:
|
||||
coordinator = getattr(manager, "coordinator", None)
|
||||
if coordinator is not None:
|
||||
assert isinstance(coordinator, HiSparseCoordinator)
|
||||
return coordinator
|
||||
coordinator = HiSparseCoordinator(
|
||||
kv_cache_manager.kv_cache_config, managers, kv_cache_manager.max_model_len
|
||||
)
|
||||
if coordinator.host_manager is None:
|
||||
raise ValueError("No HiSparse cache group is configured.")
|
||||
return coordinator
|
||||
@@ -0,0 +1,345 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from collections import defaultdict
|
||||
from dataclasses import dataclass
|
||||
from typing import cast
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.config.kv_transfer import hisparse_host_pool_gib
|
||||
from vllm.logger import init_logger
|
||||
from vllm.utils.math_utils import cdiv
|
||||
from vllm.v1.hisparse.runtime import ResolvedHiSparseConfig
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
HiSparseHotSpec,
|
||||
HiSparseResidentSpec,
|
||||
KVCacheConfig,
|
||||
KVCacheGroupRole,
|
||||
KVCacheGroupSpec,
|
||||
KVCacheSpec,
|
||||
KVCacheTensor,
|
||||
MLAAttentionSpec,
|
||||
SparseCacheRole,
|
||||
UniformTypeKVCacheSpecs,
|
||||
compute_layout_strides,
|
||||
)
|
||||
from vllm.v1.kv_cache_layout import KVCacheLayout
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
HISPARSE_HOT_SUFFIX = ".hisparse_hot"
|
||||
HISPARSE_RESIDENT_SUFFIX = ".hisparse_resident"
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class HiSparseLayout:
|
||||
source_group: KVCacheGroupSpec
|
||||
device_groups: list[KVCacheGroupSpec]
|
||||
host_num_blocks: int
|
||||
|
||||
|
||||
def get_hisparse_kv_cache_groups(
|
||||
vllm_config: VllmConfig, kv_cache_spec: dict[str, KVCacheSpec]
|
||||
) -> list[KVCacheGroupSpec] | None:
|
||||
attention_config = getattr(vllm_config, "attention_config", None)
|
||||
if attention_config is None or attention_config.hisparse_config is None:
|
||||
return None
|
||||
|
||||
mla_specs: dict[str, KVCacheSpec] = {
|
||||
name: spec
|
||||
for name, spec in kv_cache_spec.items()
|
||||
if isinstance(spec, MLAAttentionSpec)
|
||||
}
|
||||
other_specs = {
|
||||
name: spec
|
||||
for name, spec in kv_cache_spec.items()
|
||||
if not isinstance(spec, MLAAttentionSpec)
|
||||
}
|
||||
if not mla_specs or not other_specs:
|
||||
return None
|
||||
|
||||
from vllm.v1.core.kv_cache_utils import get_kv_cache_groups
|
||||
|
||||
mla_group_spec = UniformTypeKVCacheSpecs.from_specs(mla_specs)
|
||||
assert mla_group_spec is not None
|
||||
mla_group = KVCacheGroupSpec(list(mla_specs), mla_group_spec)
|
||||
return [mla_group, *get_kv_cache_groups(vllm_config, other_specs)]
|
||||
|
||||
|
||||
def get_hisparse_host_pool_bytes(vllm_config: VllmConfig) -> int:
|
||||
host_pool_gib = hisparse_host_pool_gib(vllm_config.kv_transfer_config)
|
||||
if host_pool_gib is None:
|
||||
raise ValueError("HiSparse requires HiSparseConnector with host_pool_gib")
|
||||
return int(host_pool_gib * 2**30)
|
||||
|
||||
|
||||
def _partition_hisparse_specs(
|
||||
groups: list[KVCacheGroupSpec],
|
||||
) -> tuple[dict[str, MLAAttentionSpec], dict[str, MLAAttentionSpec]]:
|
||||
group_spec = groups[0].kv_cache_spec
|
||||
if not isinstance(group_spec, UniformTypeKVCacheSpecs):
|
||||
raise ValueError("HiSparse requires uniform sparse-MLA cache specs.")
|
||||
|
||||
specs = group_spec.kv_cache_specs
|
||||
if any(
|
||||
isinstance(spec, MLAAttentionSpec) and spec.model_version == "deepseek_v4"
|
||||
for spec in specs.values()
|
||||
):
|
||||
raise ValueError("HiSparse does not support DeepSeek V4.")
|
||||
if not all(isinstance(spec, MLAAttentionSpec) for spec in specs.values()):
|
||||
raise ValueError("HiSparse requires its first cache group to contain MLA only.")
|
||||
|
||||
source_specs = {
|
||||
name: spec
|
||||
for name, spec in specs.items()
|
||||
if isinstance(spec, MLAAttentionSpec)
|
||||
and spec.cache_role is SparseCacheRole.SPARSE
|
||||
}
|
||||
indexer_specs = {
|
||||
name: spec
|
||||
for name, spec in specs.items()
|
||||
if isinstance(spec, MLAAttentionSpec)
|
||||
and spec.cache_role is SparseCacheRole.INDEXER
|
||||
}
|
||||
if not source_specs or not indexer_specs:
|
||||
raise ValueError("HiSparse requires sparse-MLA and indexer cache specs.")
|
||||
return source_specs, indexer_specs
|
||||
|
||||
|
||||
def get_hisparse_gpu_memory_usage(
|
||||
vllm_config: VllmConfig,
|
||||
kv_cache_groups: list[KVCacheGroupSpec],
|
||||
) -> int:
|
||||
_, indexer_specs = _partition_hisparse_specs(kv_cache_groups)
|
||||
return sum(
|
||||
spec.max_memory_usage_bytes(vllm_config) for spec in indexer_specs.values()
|
||||
) + sum(
|
||||
group.kv_cache_spec.max_memory_usage_bytes(vllm_config)
|
||||
for group in kv_cache_groups[1:]
|
||||
)
|
||||
|
||||
|
||||
def create_hisparse_layout(
|
||||
vllm_config: VllmConfig,
|
||||
groups: list[KVCacheGroupSpec],
|
||||
host_budget: int,
|
||||
) -> HiSparseLayout:
|
||||
source_specs, indexer_specs = _partition_hisparse_specs(groups)
|
||||
block_sizes = {
|
||||
spec.block_size for spec in (*source_specs.values(), *indexer_specs.values())
|
||||
}
|
||||
if len(block_sizes) != 1:
|
||||
raise ValueError("HiSparse requires one resolved GPU block size.")
|
||||
gpu_block_size = block_sizes.pop()
|
||||
|
||||
config = ResolvedHiSparseConfig.from_vllm_config(
|
||||
vllm_config,
|
||||
vllm_config.model_config.hf_config.index_topk,
|
||||
gpu_block_size,
|
||||
)
|
||||
assert config is not None
|
||||
indexer_group_spec = UniformTypeKVCacheSpecs.from_specs(
|
||||
cast(dict[str, KVCacheSpec], indexer_specs)
|
||||
)
|
||||
assert indexer_group_spec is not None
|
||||
indexer_group = KVCacheGroupSpec(
|
||||
list(indexer_specs),
|
||||
indexer_group_spec,
|
||||
enable_kv_transfer=True,
|
||||
role=KVCacheGroupRole.HISPARSE_INDEXER,
|
||||
)
|
||||
|
||||
indexer_page = sum(spec.page_size_bytes for spec in indexer_specs.values())
|
||||
hot_blocks_per_request = cdiv(config.device_buffer_size, gpu_block_size)
|
||||
hot_units: list[list[tuple[str, MLAAttentionSpec]]] = []
|
||||
for layer_name, layer_spec in source_specs.items():
|
||||
if layer_spec.is_index_group_leader or not hot_units:
|
||||
hot_units.append([])
|
||||
hot_units[-1].append((f"{layer_name}{HISPARSE_HOT_SUFFIX}", layer_spec))
|
||||
|
||||
resident_groups: list[KVCacheGroupSpec] = []
|
||||
hot_groups: list[KVCacheGroupSpec] = []
|
||||
|
||||
def append_hot_group(layers: list[tuple[str, MLAAttentionSpec]]) -> None:
|
||||
page_sizes = {spec.page_size_bytes for _, spec in layers}
|
||||
if len(page_sizes) != 1:
|
||||
raise ValueError(
|
||||
"HiSparse hot-cache groups require one page size, got "
|
||||
f"{sorted(page_sizes)}."
|
||||
)
|
||||
page_size = page_sizes.pop()
|
||||
names = [name for name, _ in layers]
|
||||
resident_groups.append(
|
||||
KVCacheGroupSpec(
|
||||
[
|
||||
name[: -len(HISPARSE_HOT_SUFFIX)] + HISPARSE_RESIDENT_SUFFIX
|
||||
for name in names
|
||||
],
|
||||
HiSparseResidentSpec(
|
||||
block_size=gpu_block_size,
|
||||
page_size=page_size,
|
||||
),
|
||||
enable_kv_transfer=False,
|
||||
)
|
||||
)
|
||||
hot_groups.append(
|
||||
KVCacheGroupSpec(
|
||||
names,
|
||||
HiSparseHotSpec(
|
||||
block_size=gpu_block_size,
|
||||
page_size=page_size,
|
||||
blocks_per_request=hot_blocks_per_request,
|
||||
),
|
||||
enable_kv_transfer=False,
|
||||
)
|
||||
)
|
||||
|
||||
current: list[tuple[str, MLAAttentionSpec]] = []
|
||||
current_page = 0
|
||||
for unit in hot_units:
|
||||
unit_page = sum(spec.page_size_bytes for _, spec in unit)
|
||||
if current and current_page + unit_page > indexer_page:
|
||||
append_hot_group(current)
|
||||
current = []
|
||||
current_page = 0
|
||||
current.extend(unit)
|
||||
current_page += unit_page
|
||||
if current:
|
||||
append_hot_group(current)
|
||||
|
||||
source_group_spec = UniformTypeKVCacheSpecs.from_specs(
|
||||
cast(dict[str, KVCacheSpec], source_specs)
|
||||
)
|
||||
assert source_group_spec is not None
|
||||
source_group = KVCacheGroupSpec(
|
||||
list(source_specs),
|
||||
source_group_spec,
|
||||
role=KVCacheGroupRole.HISPARSE_SOURCE,
|
||||
host_resident=True,
|
||||
enable_kv_transfer=True,
|
||||
)
|
||||
regular_groups = groups[1:]
|
||||
gpu_groups = [indexer_group, *resident_groups, *hot_groups, *regular_groups]
|
||||
|
||||
host_num_blocks = host_budget // sum(
|
||||
spec.page_size_bytes for spec in source_specs.values()
|
||||
)
|
||||
if host_num_blocks <= 0:
|
||||
raise ValueError("HiSparse has no allocatable host blocks.")
|
||||
|
||||
return HiSparseLayout(
|
||||
source_group=source_group,
|
||||
device_groups=gpu_groups,
|
||||
host_num_blocks=host_num_blocks,
|
||||
)
|
||||
|
||||
|
||||
def _build_hisparse_kv_cache_tensors(
|
||||
kv_cache_groups: list[KVCacheGroupSpec],
|
||||
num_blocks: int,
|
||||
size: int,
|
||||
layout: KVCacheLayout,
|
||||
bytes_per_block: int,
|
||||
*,
|
||||
host_resident: bool = False,
|
||||
) -> list[KVCacheTensor]:
|
||||
interleaved_block_stride = bytes_per_block if layout.is_block_outermost else None
|
||||
tensors: list[KVCacheTensor] = []
|
||||
for group in kv_cache_groups:
|
||||
group_spec = group.kv_cache_spec
|
||||
layers_by_spec: defaultdict[KVCacheSpec, list[str]] = defaultdict(list)
|
||||
if isinstance(group_spec, UniformTypeKVCacheSpecs):
|
||||
for layer_name, spec in group_spec.kv_cache_specs.items():
|
||||
layers_by_spec[spec].append(layer_name)
|
||||
elif group.layer_names:
|
||||
layers_by_spec[group_spec].extend(group.layer_names)
|
||||
|
||||
byte_offset = 0
|
||||
for spec, layer_names in layers_by_spec.items():
|
||||
if isinstance(spec, (HiSparseHotSpec, HiSparseResidentSpec)):
|
||||
if not layout.is_block_outermost:
|
||||
raise ValueError("HiSparse requires a block-outermost KV layout.")
|
||||
layer_stride = spec.page_size_bytes
|
||||
block_stride = bytes_per_block
|
||||
else:
|
||||
layer_stride, block_stride, _, _, _ = compute_layout_strides(
|
||||
spec,
|
||||
num_blocks,
|
||||
len(layer_names),
|
||||
layout,
|
||||
fixed_strides=(None, interleaved_block_stride, None, None, None),
|
||||
)
|
||||
offset = (
|
||||
byte_offset
|
||||
* max(layer_stride, spec.page_size_bytes)
|
||||
// spec.page_size_bytes
|
||||
)
|
||||
tensors.append(
|
||||
KVCacheTensor(
|
||||
size=size,
|
||||
layers=layer_names,
|
||||
layer_stride=layer_stride,
|
||||
block_stride=block_stride,
|
||||
offset=offset,
|
||||
host_resident=host_resident,
|
||||
)
|
||||
)
|
||||
byte_offset += len(layer_names) * spec.page_size_bytes
|
||||
return tensors
|
||||
|
||||
|
||||
def get_hisparse_kv_cache_config(
|
||||
vllm_config: VllmConfig,
|
||||
kv_cache_groups: list[KVCacheGroupSpec],
|
||||
available_memory: int,
|
||||
host_budget: int,
|
||||
) -> KVCacheConfig:
|
||||
from vllm.v1.core.kv_cache_utils import (
|
||||
_get_kv_cache_bytes_per_block,
|
||||
may_override_num_blocks,
|
||||
validate_kv_cache_layout,
|
||||
)
|
||||
|
||||
hisparse_layout = create_hisparse_layout(vllm_config, kv_cache_groups, host_budget)
|
||||
device_groups = hisparse_layout.device_groups
|
||||
layout = vllm_config.cache_config.get_resolved_kv_cache_layout()
|
||||
validate_kv_cache_layout(layout, device_groups)
|
||||
bytes_per_block = _get_kv_cache_bytes_per_block(device_groups)
|
||||
num_blocks = may_override_num_blocks(
|
||||
vllm_config, available_memory // bytes_per_block
|
||||
)
|
||||
size = bytes_per_block * num_blocks
|
||||
kv_cache_tensors = _build_hisparse_kv_cache_tensors(
|
||||
device_groups, num_blocks, size, layout, bytes_per_block
|
||||
)
|
||||
|
||||
host_groups = [hisparse_layout.source_group]
|
||||
host_layout = KVCacheLayout.LBNHC
|
||||
validate_kv_cache_layout(host_layout, host_groups)
|
||||
host_bytes_per_block = _get_kv_cache_bytes_per_block(host_groups)
|
||||
host_size = host_bytes_per_block * hisparse_layout.host_num_blocks
|
||||
kv_cache_tensors[:0] = _build_hisparse_kv_cache_tensors(
|
||||
host_groups,
|
||||
hisparse_layout.host_num_blocks,
|
||||
host_size,
|
||||
host_layout,
|
||||
host_bytes_per_block,
|
||||
host_resident=True,
|
||||
)
|
||||
logger.info_once(
|
||||
"HiSparse HMA: %.1f GiB host source (%d blocks), %.1f GiB shared "
|
||||
"GPU indexer/resident/hot pool (%d blocks).",
|
||||
host_size / 2**30,
|
||||
hisparse_layout.host_num_blocks,
|
||||
size / 2**30,
|
||||
num_blocks,
|
||||
)
|
||||
return KVCacheConfig(
|
||||
num_blocks=num_blocks,
|
||||
kv_cache_tensors=kv_cache_tensors,
|
||||
kv_cache_groups=[*host_groups, *device_groups],
|
||||
hisparse_host_num_blocks=hisparse_layout.host_num_blocks,
|
||||
prefix_cache_retention_interval=(
|
||||
vllm_config.cache_config.prefix_cache_retention_interval
|
||||
),
|
||||
)
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,30 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from dataclasses import dataclass
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class SparseKVPageTransfer:
|
||||
"""Copy one logical KV page between cache-manager and worker-owned tiers."""
|
||||
|
||||
transfer_id: int
|
||||
destination_block_id: int
|
||||
source_block_ids: tuple[int, ...]
|
||||
after_forward: bool
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class SparseKVRowMirror:
|
||||
"""Mirror one contiguous resident-row span into the host cache."""
|
||||
|
||||
source_starts: tuple[int, ...]
|
||||
destination_start: int
|
||||
num_rows: int
|
||||
|
||||
|
||||
@dataclass
|
||||
class SparseKVOffloadCommand:
|
||||
"""Opaque scheduler-to-worker command for one model step."""
|
||||
|
||||
page_transfers: list[SparseKVPageTransfer]
|
||||
Reference in New Issue
Block a user