Signed-off-by: Matthew Bonanni <mbonanni@redhat.com> Signed-off-by: Chao Lei <leichao139636@163.com> Signed-off-by: Lucas Wilkinson <lwilkinson@neuralmagic.com> Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com> Co-authored-by: Chao Lei <leichao139636@163.com> Co-authored-by: Claude Fable 5 <noreply@anthropic.com> Co-authored-by: Matthew Bonanni <mbonanni@redhat.com> Co-authored-by: OpenAI Codex <codex@openai.com> Co-authored-by: OpenAI Codex <noreply@openai.com>
This commit is contained in:
co-authored by
Chao Lei
Claude Fable 5
Matthew Bonanni
OpenAI Codex
OpenAI Codex
parent
ebe1dec2da
commit
e19a3e172e
@@ -147,6 +147,12 @@ def _select_written_row_mirrors(
|
||||
)
|
||||
|
||||
|
||||
def _is_hisparse_host_writer(
|
||||
shared_host_region: SharedOffloadRegion | None,
|
||||
) -> bool:
|
||||
return shared_host_region is None or get_tensor_model_parallel_rank() == 0
|
||||
|
||||
|
||||
def _create_hisparse_host_events(
|
||||
shared_host_region: SharedOffloadRegion | None,
|
||||
is_host_writer: bool,
|
||||
@@ -200,15 +206,26 @@ class HiSparseConnectorWorker:
|
||||
raise RuntimeError("HiSparse connector found no hot-cache handles.")
|
||||
hot_backings: dict[int, torch.Tensor] = {}
|
||||
registered_host_pools: dict[int, torch.Tensor] = {}
|
||||
shared_host_regions: dict[int, SharedOffloadRegion] = {}
|
||||
for cache in cache_handles:
|
||||
hot_backing = cache.runtime.hot_backing
|
||||
hot_backings[hot_backing.untyped_storage().data_ptr()] = hot_backing
|
||||
registered_pool = cache.runtime.registered_host_pool
|
||||
registered_host_pools[registered_pool.data_ptr()] = registered_pool
|
||||
if (region := cache.runtime.shared_host_region) is not None:
|
||||
shared_host_regions[id(region)] = region
|
||||
if len(hot_backings) != 1:
|
||||
raise RuntimeError("HiSparse hot tensors must share one GPU backing.")
|
||||
hot_backing = next(iter(hot_backings.values()))
|
||||
pinned_host_pools = list(registered_host_pools.values())
|
||||
if len(shared_host_regions) > 1:
|
||||
raise RuntimeError("HiSparse caches must share one host region.")
|
||||
shared_host_region = next(iter(shared_host_regions.values()), None)
|
||||
pinned_host_pools = (
|
||||
[]
|
||||
if shared_host_region is not None
|
||||
else list(registered_host_pools.values())
|
||||
)
|
||||
is_host_writer = _is_hisparse_host_writer(shared_host_region)
|
||||
|
||||
resident = cache_handles[0].view
|
||||
assert resident is not None
|
||||
@@ -223,11 +240,14 @@ class HiSparseConnectorWorker:
|
||||
host_num_blocks,
|
||||
hot_backing.device,
|
||||
pinned_host_pools,
|
||||
shared_host_region=shared_host_region,
|
||||
is_host_writer=is_host_writer,
|
||||
)
|
||||
except Exception:
|
||||
release_pinned_state(
|
||||
[cache.runtime for cache in cache_handles],
|
||||
pinned_host_pools,
|
||||
shared_host_region,
|
||||
)
|
||||
raise
|
||||
|
||||
@@ -866,6 +886,8 @@ class HiSparseConnectorWorker:
|
||||
if self.dma_stream is not None:
|
||||
self.dma_stream.synchronize()
|
||||
release_pinned_state(
|
||||
[cache.runtime for cache in self.cache_handles], self.pinned_host_pools
|
||||
[cache.runtime for cache in self.cache_handles],
|
||||
self.pinned_host_pools,
|
||||
self.shared_host_region,
|
||||
)
|
||||
self._initialized = False
|
||||
|
||||
@@ -17,6 +17,7 @@ from vllm.v1.kv_cache_interface import (
|
||||
MLAAttentionSpec,
|
||||
SlidingWindowMLASpec,
|
||||
SlidingWindowSpec,
|
||||
UniformTypeKVCacheSpecs,
|
||||
iter_layer_specs,
|
||||
)
|
||||
from vllm.v1.kv_offload.config import (
|
||||
@@ -29,7 +30,7 @@ from vllm.v1.kv_offload.config import (
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.v1.kv_cache_interface import KVCacheConfig
|
||||
from vllm.v1.kv_cache_interface import KVCacheConfig, KVCacheGroupSpec
|
||||
|
||||
|
||||
def get_offloading_group_ids(kv_cache_config: "KVCacheConfig") -> tuple[int, ...]:
|
||||
@@ -42,6 +43,19 @@ def get_offloading_group_ids(kv_cache_config: "KVCacheConfig") -> tuple[int, ...
|
||||
)
|
||||
|
||||
|
||||
def _group_kv_bytes_per_block(group: "KVCacheGroupSpec") -> int:
|
||||
"""Return the physical bytes occupied by one block of a cache group.
|
||||
|
||||
Worker configs may retain ``UniformTypeKVCacheSpecs`` while scheduler
|
||||
configs flatten that wrapper to one representative per-layer spec. Keep
|
||||
the result invariant across those two representations.
|
||||
"""
|
||||
spec = group.kv_cache_spec
|
||||
if isinstance(spec, UniformTypeKVCacheSpecs):
|
||||
return spec.page_size_bytes
|
||||
return spec.page_size_bytes * len(group.layer_names)
|
||||
|
||||
|
||||
def build_offloading_config(
|
||||
vllm_config: "VllmConfig",
|
||||
kv_cache_config: "KVCacheConfig",
|
||||
@@ -133,7 +147,7 @@ def build_offloading_config(
|
||||
worker_kv_bytes_per_block = total_gpu_kv_bytes // kv_cache_config.num_blocks
|
||||
elif kv_cache_config.num_blocks > 0:
|
||||
worker_kv_bytes_per_block = sum(
|
||||
group.kv_cache_spec.page_size_bytes for _, group in selected_groups
|
||||
_group_kv_bytes_per_block(group) for _, group in selected_groups
|
||||
)
|
||||
|
||||
single_group_spec = (
|
||||
|
||||
Reference in New Issue
Block a user