[5/N] Share HiSparse host cache across TP ranks (reopens #52760) (#56629)

Signed-off-by: Matthew Bonanni <mbonanni@redhat.com>
Signed-off-by: Chao Lei <leichao139636@163.com>
Signed-off-by: Lucas Wilkinson <lwilkinson@neuralmagic.com>
Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com>
Co-authored-by: Chao Lei <leichao139636@163.com>
Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
Co-authored-by: Matthew Bonanni <mbonanni@redhat.com>
Co-authored-by: OpenAI Codex <codex@openai.com>
Co-authored-by: OpenAI Codex <noreply@openai.com>
This commit is contained in:
Lucas Wilkinson
2026-09-12 19:48:25 -04:00
committed by GitHub
co-authored by Chao Lei Claude Fable 5 Matthew Bonanni OpenAI Codex OpenAI Codex
parent ebe1dec2da
commit e19a3e172e
14 changed files with 878 additions and 85 deletions
@@ -147,6 +147,12 @@ def _select_written_row_mirrors(
)
def _is_hisparse_host_writer(
shared_host_region: SharedOffloadRegion | None,
) -> bool:
return shared_host_region is None or get_tensor_model_parallel_rank() == 0
def _create_hisparse_host_events(
shared_host_region: SharedOffloadRegion | None,
is_host_writer: bool,
@@ -200,15 +206,26 @@ class HiSparseConnectorWorker:
raise RuntimeError("HiSparse connector found no hot-cache handles.")
hot_backings: dict[int, torch.Tensor] = {}
registered_host_pools: dict[int, torch.Tensor] = {}
shared_host_regions: dict[int, SharedOffloadRegion] = {}
for cache in cache_handles:
hot_backing = cache.runtime.hot_backing
hot_backings[hot_backing.untyped_storage().data_ptr()] = hot_backing
registered_pool = cache.runtime.registered_host_pool
registered_host_pools[registered_pool.data_ptr()] = registered_pool
if (region := cache.runtime.shared_host_region) is not None:
shared_host_regions[id(region)] = region
if len(hot_backings) != 1:
raise RuntimeError("HiSparse hot tensors must share one GPU backing.")
hot_backing = next(iter(hot_backings.values()))
pinned_host_pools = list(registered_host_pools.values())
if len(shared_host_regions) > 1:
raise RuntimeError("HiSparse caches must share one host region.")
shared_host_region = next(iter(shared_host_regions.values()), None)
pinned_host_pools = (
[]
if shared_host_region is not None
else list(registered_host_pools.values())
)
is_host_writer = _is_hisparse_host_writer(shared_host_region)
resident = cache_handles[0].view
assert resident is not None
@@ -223,11 +240,14 @@ class HiSparseConnectorWorker:
host_num_blocks,
hot_backing.device,
pinned_host_pools,
shared_host_region=shared_host_region,
is_host_writer=is_host_writer,
)
except Exception:
release_pinned_state(
[cache.runtime for cache in cache_handles],
pinned_host_pools,
shared_host_region,
)
raise
@@ -866,6 +886,8 @@ class HiSparseConnectorWorker:
if self.dma_stream is not None:
self.dma_stream.synchronize()
release_pinned_state(
[cache.runtime for cache in self.cache_handles], self.pinned_host_pools
[cache.runtime for cache in self.cache_handles],
self.pinned_host_pools,
self.shared_host_region,
)
self._initialized = False
@@ -17,6 +17,7 @@ from vllm.v1.kv_cache_interface import (
MLAAttentionSpec,
SlidingWindowMLASpec,
SlidingWindowSpec,
UniformTypeKVCacheSpecs,
iter_layer_specs,
)
from vllm.v1.kv_offload.config import (
@@ -29,7 +30,7 @@ from vllm.v1.kv_offload.config import (
if TYPE_CHECKING:
from vllm.config import VllmConfig
from vllm.v1.kv_cache_interface import KVCacheConfig
from vllm.v1.kv_cache_interface import KVCacheConfig, KVCacheGroupSpec
def get_offloading_group_ids(kv_cache_config: "KVCacheConfig") -> tuple[int, ...]:
@@ -42,6 +43,19 @@ def get_offloading_group_ids(kv_cache_config: "KVCacheConfig") -> tuple[int, ...
)
def _group_kv_bytes_per_block(group: "KVCacheGroupSpec") -> int:
"""Return the physical bytes occupied by one block of a cache group.
Worker configs may retain ``UniformTypeKVCacheSpecs`` while scheduler
configs flatten that wrapper to one representative per-layer spec. Keep
the result invariant across those two representations.
"""
spec = group.kv_cache_spec
if isinstance(spec, UniformTypeKVCacheSpecs):
return spec.page_size_bytes
return spec.page_size_bytes * len(group.layer_names)
def build_offloading_config(
vllm_config: "VllmConfig",
kv_cache_config: "KVCacheConfig",
@@ -133,7 +147,7 @@ def build_offloading_config(
worker_kv_bytes_per_block = total_gpu_kv_bytes // kv_cache_config.num_blocks
elif kv_cache_config.num_blocks > 0:
worker_kv_bytes_per_block = sum(
group.kv_cache_spec.page_size_bytes for _, group in selected_groups
_group_kv_bytes_per_block(group) for _, group in selected_groups
)
single_group_spec = (