[5/N] Share HiSparse host cache across TP ranks (reopens #52760) (#56629)

Signed-off-by: Matthew Bonanni <mbonanni@redhat.com>
Signed-off-by: Chao Lei <leichao139636@163.com>
Signed-off-by: Lucas Wilkinson <lwilkinson@neuralmagic.com>
Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com>
Co-authored-by: Chao Lei <leichao139636@163.com>
Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
Co-authored-by: Matthew Bonanni <mbonanni@redhat.com>
Co-authored-by: OpenAI Codex <codex@openai.com>
Co-authored-by: OpenAI Codex <noreply@openai.com>
This commit is contained in:
Lucas Wilkinson
2026-09-12 19:48:25 -04:00
committed by GitHub
co-authored by Chao Lei Claude Fable 5 Matthew Bonanni OpenAI Codex OpenAI Codex
parent ebe1dec2da
commit e19a3e172e
14 changed files with 878 additions and 85 deletions
+44 -27
View File
@@ -120,27 +120,32 @@ def init_hisparse_kv_cache(
block_tables: "BlockTables",
) -> dict[str, torch.Tensor]:
"""Allocate and bind HiSparse caches within the caller's allocation context."""
host_pool = HiSparseHostPool()
kv_caches = allocate_hisparse_kv_caches(
kv_cache_config,
device,
vllm_config.cache_config.get_resolved_kv_cache_layout(),
kernel_block_sizes,
host_pool,
)
cache_handles = bind_hisparse_kv_caches(
forward_context=forward_context,
kv_cache_config=kv_cache_config,
kv_caches=kv_caches,
block_tables=block_tables,
host_pool=host_pool,
)
initialize_hisparse_runtime_buffers(
cache_handles,
max_num_reqs=vllm_config.scheduler_config.max_num_seqs,
max_num_batched_tokens=vllm_config.scheduler_config.max_num_batched_tokens,
)
return kv_caches
host_pool = HiSparseHostPool(vllm_config, kv_cache_config)
try:
kv_caches = allocate_hisparse_kv_caches(
kv_cache_config,
device,
vllm_config.cache_config.get_resolved_kv_cache_layout(),
kernel_block_sizes,
host_pool,
)
cache_handles = bind_hisparse_kv_caches(
forward_context=forward_context,
kv_cache_config=kv_cache_config,
kv_caches=kv_caches,
block_tables=block_tables,
host_pool=host_pool,
)
initialize_hisparse_runtime_buffers(
cache_handles,
max_num_reqs=vllm_config.scheduler_config.max_num_seqs,
max_num_batched_tokens=vllm_config.scheduler_config.max_num_batched_tokens,
)
return kv_caches
except Exception:
if host_pool.shared_region is not None:
host_pool.shared_region.cleanup()
raise
def _get_hisparse_cache(
@@ -166,13 +171,24 @@ def release_hisparse_profiling_cache(forward_context: dict[str, Any]) -> None:
if not runtimes:
return
registered_pools = list(
{
runtime.registered_host_pool.data_ptr(): runtime.registered_host_pool
for runtime in runtimes.values()
}.values()
shared_regions = {
id(runtime.shared_host_region): runtime.shared_host_region
for runtime in runtimes.values()
if runtime.shared_host_region is not None
}
assert len(shared_regions) <= 1
shared_region = next(iter(shared_regions.values()), None)
registered_pools = (
[]
if shared_region is not None
else list(
{
runtime.registered_host_pool.data_ptr(): runtime.registered_host_pool
for runtime in runtimes.values()
}.values()
)
)
release_pinned_state(list(runtimes.values()), registered_pools)
release_pinned_state(list(runtimes.values()), registered_pools, shared_region)
for cache in cache_handles:
cache.mirror_staging_cache = None
cache.mirror_staging_slots = None
@@ -257,6 +273,7 @@ def bind_hisparse_kv_caches(
assert source_cache.untyped_storage().data_ptr() == (
host_pool.registered.untyped_storage().data_ptr()
)
cache_handle.runtime.shared_host_region = host_pool.shared_region
cache_handle.runtime.bind_source_cache(
source_cache,
registered_host_pool=host_pool.registered,
+16 -3
View File
@@ -9,7 +9,11 @@ from vllm.config import VllmConfig
from vllm.config.kv_transfer import hisparse_host_pool_gib
from vllm.logger import init_logger
from vllm.utils.math_utils import cdiv
from vllm.v1.hisparse.runtime import ResolvedHiSparseConfig
from vllm.v1.hisparse.runtime import (
ResolvedHiSparseConfig,
get_hisparse_host_block_stride,
use_shared_hisparse_host_pool,
)
from vllm.v1.kv_cache_interface import (
HiSparseHotSpec,
HiSparseResidentSpec,
@@ -36,6 +40,8 @@ class HiSparseLayout:
source_group: KVCacheGroupSpec
device_groups: list[KVCacheGroupSpec]
host_num_blocks: int
host_block_stride: int
shared_host_pool: bool
def get_hisparse_kv_cache_groups(
@@ -221,9 +227,12 @@ def create_hisparse_layout(
regular_groups = groups[1:]
gpu_groups = [indexer_group, *resident_groups, *hot_groups, *regular_groups]
host_num_blocks = host_budget // sum(
spec.page_size_bytes for spec in source_specs.values()
shared_host_pool = use_shared_hisparse_host_pool(vllm_config)
host_block_stride = get_hisparse_host_block_stride(
sum(spec.page_size_bytes for spec in source_specs.values()),
use_shared_host_pool=shared_host_pool,
)
host_num_blocks = host_budget // host_block_stride
if host_num_blocks <= 0:
raise ValueError("HiSparse has no allocatable host blocks.")
@@ -231,6 +240,8 @@ def create_hisparse_layout(
source_group=source_group,
device_groups=gpu_groups,
host_num_blocks=host_num_blocks,
host_block_stride=host_block_stride,
shared_host_pool=shared_host_pool,
)
@@ -339,6 +350,8 @@ def get_hisparse_kv_cache_config(
kv_cache_tensors=kv_cache_tensors,
kv_cache_groups=[*host_groups, *device_groups],
hisparse_host_num_blocks=hisparse_layout.host_num_blocks,
hisparse_host_block_stride=hisparse_layout.host_block_stride,
hisparse_shared_host_pool=hisparse_layout.shared_host_pool,
prefix_cache_retention_interval=(
vllm_config.cache_config.prefix_cache_retention_interval
),
+189 -6
View File
@@ -4,6 +4,8 @@
from __future__ import annotations
import math
import mmap
import time
from collections.abc import Callable
from dataclasses import dataclass
@@ -13,17 +15,22 @@ import psutil
import torch
from vllm.config import CUDAGraphMode, VllmConfig
from vllm.distributed import get_tp_group
from vllm.forward_context import get_forward_context
from vllm.logger import init_logger
from vllm.platforms import current_platform
from vllm.utils.math_utils import round_up
from vllm.utils.torch_utils import current_stream
from vllm.v1.kv_offload.cpu.shared_offload_region import SharedOffloadRegion
from vllm.v1.simple_kv_offload.cuda_mem_ops import pin_tensor
logger = init_logger(__name__)
HOST_REGISTER_CHUNK_BYTES = 256 * 2**30
if TYPE_CHECKING:
from vllm.v1.attention.backends.mla.index_group import HiSparseMLAIndexGroup
from vllm.v1.kv_cache_interface import KVCacheConfig
# fp8_ds_mla KV row: 512 B quantized NoPE + 16 B scales + 128 B RoPE.
FP8_DS_MLA_ROW_BYTES = 656
@@ -185,22 +192,195 @@ def allocate_pinned_host_pool(size: int) -> tuple[torch.Tensor, torch.Tensor]:
class HiSparseHostPool:
"""Keep the single host backing and its exact CUDA registration range."""
def __init__(self) -> None:
def __init__(self, vllm_config: VllmConfig, kv_cache_config: KVCacheConfig) -> None:
self.vllm_config = vllm_config
self.kv_cache_config = kv_cache_config
self.backing: torch.Tensor | None = None
self.registered: torch.Tensor | None = None
self.shared_region: SharedOffloadRegion | None = None
def allocate(self, size: int) -> torch.Tensor:
assert self.backing is None, "HiSparse host pool is already allocated."
check_hisparse_host_memory(size)
self.backing, self.registered = allocate_pinned_host_pool(size)
config = self.kv_cache_config
if config.hisparse_shared_host_pool:
tensor_sizes = [
size
for _, size in sorted(
(tensor.offset + i * tensor.layer_stride, tensor.layer_stride)
for tensor in config.kv_cache_tensors
if tensor.host_resident
for i in range(len(tensor.layers))
)
]
assert config.hisparse_host_num_blocks is not None
assert config.hisparse_host_block_stride is not None
_, _, self.shared_region = allocate_hisparse_host_pools(
self.vllm_config,
tensor_sizes,
config.hisparse_host_num_blocks,
config.hisparse_host_block_stride,
use_shared_host_pool=True,
)
assert self.shared_region is not None
self.registered = self.shared_region.base_tensor
self.backing = self.registered[:size]
else:
check_hisparse_host_memory(size)
self.backing, self.registered = allocate_pinned_host_pool(size)
return self.backing
def use_shared_hisparse_host_pool(vllm_config: VllmConfig) -> bool:
"""Whether replicated MLA host KV can share one local mmap."""
parallel = vllm_config.parallel_config
return (
current_platform.is_cuda_alike()
and parallel.tensor_parallel_size > 1
and parallel.pipeline_parallel_size == 1
and parallel.prefill_context_parallel_size == 1
and parallel.decode_context_parallel_size == 1
and parallel.world_size == parallel.tensor_parallel_size
and parallel.distributed_executor_backend == "mp"
and parallel.nnodes_within_dp == 1
)
def get_hisparse_host_block_stride(
page_size_bytes: int, *, use_shared_host_pool: bool
) -> int:
if use_shared_host_pool:
return round_up(page_size_bytes, SharedOffloadRegion.BLOCK_SIZE_ALIGNMENT)
return page_size_bytes
def _hisparse_registration_ranges(
tensor_sizes: list[int],
num_blocks: int,
host_block_stride: int,
max_chunk_bytes: int = HOST_REGISTER_CHUNK_BYTES,
) -> tuple[tuple[int, int], ...]:
"""Split a shared pool without bisecting any tensor's host block.
CUDA batch copies reject descriptors spanning adjacent registrations.
"""
total_size = num_blocks * host_block_stride
if total_size % mmap.PAGESIZE:
raise ValueError("HiSparse host pool must be OS-page aligned.")
if max_chunk_bytes < mmap.PAGESIZE:
raise ValueError("HiSparse registration chunks must span at least one page.")
tensors: list[tuple[int, int, int]] = []
tensor_start = 0
for tensor_size in tensor_sizes:
if tensor_size % num_blocks:
raise ValueError(
f"HiSparse host tensor size {tensor_size} is not divisible by "
f"{num_blocks} blocks."
)
tensor_end = tensor_start + tensor_size
tensors.append((tensor_start, tensor_end, tensor_size // num_blocks))
tensor_start = tensor_end
if tensor_start > total_size:
raise ValueError("HiSparse host tensors exceed the shared host pool.")
def last_safe_boundary(lower: int, upper: int) -> int | None:
best = None
for start, end, block_bytes in tensors:
first_block = max(0, (lower - start) // block_bytes + 1)
last_block = min(num_blocks, (min(upper, end) - start) // block_bytes)
if first_block > last_block:
continue
divisor = math.gcd(block_bytes, mmap.PAGESIZE)
if start % divisor:
continue
period = mmap.PAGESIZE // divisor
if period == 1:
residue = 0
else:
# Find block boundaries that are also OS-page boundaries.
residue = (
(-start // divisor) * pow(block_bytes // divisor, -1, period)
) % period
block = last_block - (last_block - residue) % period
if block >= first_block:
candidate = start + block * block_bytes
best = candidate if best is None else max(best, candidate)
if upper >= tensor_start:
candidate = upper // mmap.PAGESIZE * mmap.PAGESIZE
if candidate >= tensor_start and candidate > lower:
best = candidate if best is None else max(best, candidate)
return best
ranges: list[tuple[int, int]] = []
start = 0
while total_size - start > max_chunk_bytes:
end = last_safe_boundary(start, start + max_chunk_bytes)
if end is None:
raise ValueError(
"A HiSparse host block is too large for the registration chunk limit."
)
ranges.append((start, end))
start = end
ranges.append((start, total_size))
return tuple(ranges)
def allocate_hisparse_host_pools(
vllm_config: VllmConfig,
tensor_sizes: list[int],
num_blocks: int,
host_block_stride: int,
*,
use_shared_host_pool: bool,
) -> tuple[list[torch.Tensor], list[torch.Tensor], SharedOffloadRegion | None]:
"""Allocate private host tensors or one mmap shared by local TP ranks."""
if not use_shared_host_pool:
check_hisparse_host_memory(sum(tensor_sizes))
private_pools = [allocate_pinned_host_pool(size) for size in tensor_sizes]
return (
[pool for pool, _ in private_pools],
[registered for _, registered in private_pools],
None,
)
region = SharedOffloadRegion(
engine_id=(
f"hisparse_{vllm_config.instance_id}_"
f"dp{vllm_config.parallel_config.data_parallel_index}"
),
num_chunks=1,
rank=0,
kv_bytes_per_chunk=num_blocks * host_block_stride,
cpu_page_size=sum(tensor_sizes),
barrier=get_tp_group().barrier,
creator_memory_check=check_hisparse_host_memory,
populate_only_on_creator=True,
)
try:
for start, end in _hisparse_registration_ranges(
tensor_sizes, num_blocks, host_block_stride
):
tensor = region.base_tensor[start:end]
pin_tensor(tensor)
region.pinned_addresses.append(tensor.data_ptr())
region.is_pinned = True
pools = [
region.create_next_canonical_view(size).view(-1) for size in tensor_sizes
]
except Exception:
region.cleanup()
raise
return pools, [], region
def release_pinned_state(
runtimes: list[HiSparseRuntime], pinned_host_pools: list[torch.Tensor]
runtimes: list[HiSparseRuntime],
pinned_host_pools: list[torch.Tensor],
shared_host_region: SharedOffloadRegion | None = None,
) -> None:
"""Synchronize and release registered host KV pools."""
if pinned_host_pools:
if pinned_host_pools or shared_host_region is not None:
try:
torch.accelerator.synchronize()
except RuntimeError as e:
@@ -208,7 +388,7 @@ def release_pinned_state(
"HiSparse: CUDA context unusable at teardown (%s); leaving "
"%d host-pool tensors pinned for kernel exit reclaim.",
e,
len(pinned_host_pools),
len(pinned_host_pools) + int(shared_host_region is not None),
)
return
@@ -240,6 +420,8 @@ def release_pinned_state(
del runtime._host_cache
del runtime.registered_host_pool
del runtime.hot_backing
if shared_host_region is not None:
shared_host_region.cleanup()
def hisparse_prefill_staging_remap(
@@ -536,6 +718,7 @@ class HiSparseRuntime:
self.eager_host_mirror = config.eager_host_mirror
self.resident_source_index = -1
self.request_state_indices: torch.Tensor | None = None
self.shared_host_region: SharedOffloadRegion | None = None
@property
def host_cache(self) -> torch.Tensor: