[5/N] Share HiSparse host cache across TP ranks (reopens #52760) (#56629)

Signed-off-by: Matthew Bonanni <mbonanni@redhat.com>
Signed-off-by: Chao Lei <leichao139636@163.com>
Signed-off-by: Lucas Wilkinson <lwilkinson@neuralmagic.com>
Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com>
Co-authored-by: Chao Lei <leichao139636@163.com>
Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
Co-authored-by: Matthew Bonanni <mbonanni@redhat.com>
Co-authored-by: OpenAI Codex <codex@openai.com>
Co-authored-by: OpenAI Codex <noreply@openai.com>
This commit is contained in:
Lucas Wilkinson
2026-09-12 19:48:25 -04:00
committed by GitHub
co-authored by Chao Lei Claude Fable 5 Matthew Bonanni OpenAI Codex OpenAI Codex
parent ebe1dec2da
commit e19a3e172e
14 changed files with 878 additions and 85 deletions
+189 -6
View File
@@ -4,6 +4,8 @@
from __future__ import annotations
import math
import mmap
import time
from collections.abc import Callable
from dataclasses import dataclass
@@ -13,17 +15,22 @@ import psutil
import torch
from vllm.config import CUDAGraphMode, VllmConfig
from vllm.distributed import get_tp_group
from vllm.forward_context import get_forward_context
from vllm.logger import init_logger
from vllm.platforms import current_platform
from vllm.utils.math_utils import round_up
from vllm.utils.torch_utils import current_stream
from vllm.v1.kv_offload.cpu.shared_offload_region import SharedOffloadRegion
from vllm.v1.simple_kv_offload.cuda_mem_ops import pin_tensor
logger = init_logger(__name__)
HOST_REGISTER_CHUNK_BYTES = 256 * 2**30
if TYPE_CHECKING:
from vllm.v1.attention.backends.mla.index_group import HiSparseMLAIndexGroup
from vllm.v1.kv_cache_interface import KVCacheConfig
# fp8_ds_mla KV row: 512 B quantized NoPE + 16 B scales + 128 B RoPE.
FP8_DS_MLA_ROW_BYTES = 656
@@ -185,22 +192,195 @@ def allocate_pinned_host_pool(size: int) -> tuple[torch.Tensor, torch.Tensor]:
class HiSparseHostPool:
"""Keep the single host backing and its exact CUDA registration range."""
def __init__(self) -> None:
def __init__(self, vllm_config: VllmConfig, kv_cache_config: KVCacheConfig) -> None:
self.vllm_config = vllm_config
self.kv_cache_config = kv_cache_config
self.backing: torch.Tensor | None = None
self.registered: torch.Tensor | None = None
self.shared_region: SharedOffloadRegion | None = None
def allocate(self, size: int) -> torch.Tensor:
assert self.backing is None, "HiSparse host pool is already allocated."
check_hisparse_host_memory(size)
self.backing, self.registered = allocate_pinned_host_pool(size)
config = self.kv_cache_config
if config.hisparse_shared_host_pool:
tensor_sizes = [
size
for _, size in sorted(
(tensor.offset + i * tensor.layer_stride, tensor.layer_stride)
for tensor in config.kv_cache_tensors
if tensor.host_resident
for i in range(len(tensor.layers))
)
]
assert config.hisparse_host_num_blocks is not None
assert config.hisparse_host_block_stride is not None
_, _, self.shared_region = allocate_hisparse_host_pools(
self.vllm_config,
tensor_sizes,
config.hisparse_host_num_blocks,
config.hisparse_host_block_stride,
use_shared_host_pool=True,
)
assert self.shared_region is not None
self.registered = self.shared_region.base_tensor
self.backing = self.registered[:size]
else:
check_hisparse_host_memory(size)
self.backing, self.registered = allocate_pinned_host_pool(size)
return self.backing
def use_shared_hisparse_host_pool(vllm_config: VllmConfig) -> bool:
"""Whether replicated MLA host KV can share one local mmap."""
parallel = vllm_config.parallel_config
return (
current_platform.is_cuda_alike()
and parallel.tensor_parallel_size > 1
and parallel.pipeline_parallel_size == 1
and parallel.prefill_context_parallel_size == 1
and parallel.decode_context_parallel_size == 1
and parallel.world_size == parallel.tensor_parallel_size
and parallel.distributed_executor_backend == "mp"
and parallel.nnodes_within_dp == 1
)
def get_hisparse_host_block_stride(
page_size_bytes: int, *, use_shared_host_pool: bool
) -> int:
if use_shared_host_pool:
return round_up(page_size_bytes, SharedOffloadRegion.BLOCK_SIZE_ALIGNMENT)
return page_size_bytes
def _hisparse_registration_ranges(
tensor_sizes: list[int],
num_blocks: int,
host_block_stride: int,
max_chunk_bytes: int = HOST_REGISTER_CHUNK_BYTES,
) -> tuple[tuple[int, int], ...]:
"""Split a shared pool without bisecting any tensor's host block.
CUDA batch copies reject descriptors spanning adjacent registrations.
"""
total_size = num_blocks * host_block_stride
if total_size % mmap.PAGESIZE:
raise ValueError("HiSparse host pool must be OS-page aligned.")
if max_chunk_bytes < mmap.PAGESIZE:
raise ValueError("HiSparse registration chunks must span at least one page.")
tensors: list[tuple[int, int, int]] = []
tensor_start = 0
for tensor_size in tensor_sizes:
if tensor_size % num_blocks:
raise ValueError(
f"HiSparse host tensor size {tensor_size} is not divisible by "
f"{num_blocks} blocks."
)
tensor_end = tensor_start + tensor_size
tensors.append((tensor_start, tensor_end, tensor_size // num_blocks))
tensor_start = tensor_end
if tensor_start > total_size:
raise ValueError("HiSparse host tensors exceed the shared host pool.")
def last_safe_boundary(lower: int, upper: int) -> int | None:
best = None
for start, end, block_bytes in tensors:
first_block = max(0, (lower - start) // block_bytes + 1)
last_block = min(num_blocks, (min(upper, end) - start) // block_bytes)
if first_block > last_block:
continue
divisor = math.gcd(block_bytes, mmap.PAGESIZE)
if start % divisor:
continue
period = mmap.PAGESIZE // divisor
if period == 1:
residue = 0
else:
# Find block boundaries that are also OS-page boundaries.
residue = (
(-start // divisor) * pow(block_bytes // divisor, -1, period)
) % period
block = last_block - (last_block - residue) % period
if block >= first_block:
candidate = start + block * block_bytes
best = candidate if best is None else max(best, candidate)
if upper >= tensor_start:
candidate = upper // mmap.PAGESIZE * mmap.PAGESIZE
if candidate >= tensor_start and candidate > lower:
best = candidate if best is None else max(best, candidate)
return best
ranges: list[tuple[int, int]] = []
start = 0
while total_size - start > max_chunk_bytes:
end = last_safe_boundary(start, start + max_chunk_bytes)
if end is None:
raise ValueError(
"A HiSparse host block is too large for the registration chunk limit."
)
ranges.append((start, end))
start = end
ranges.append((start, total_size))
return tuple(ranges)
def allocate_hisparse_host_pools(
vllm_config: VllmConfig,
tensor_sizes: list[int],
num_blocks: int,
host_block_stride: int,
*,
use_shared_host_pool: bool,
) -> tuple[list[torch.Tensor], list[torch.Tensor], SharedOffloadRegion | None]:
"""Allocate private host tensors or one mmap shared by local TP ranks."""
if not use_shared_host_pool:
check_hisparse_host_memory(sum(tensor_sizes))
private_pools = [allocate_pinned_host_pool(size) for size in tensor_sizes]
return (
[pool for pool, _ in private_pools],
[registered for _, registered in private_pools],
None,
)
region = SharedOffloadRegion(
engine_id=(
f"hisparse_{vllm_config.instance_id}_"
f"dp{vllm_config.parallel_config.data_parallel_index}"
),
num_chunks=1,
rank=0,
kv_bytes_per_chunk=num_blocks * host_block_stride,
cpu_page_size=sum(tensor_sizes),
barrier=get_tp_group().barrier,
creator_memory_check=check_hisparse_host_memory,
populate_only_on_creator=True,
)
try:
for start, end in _hisparse_registration_ranges(
tensor_sizes, num_blocks, host_block_stride
):
tensor = region.base_tensor[start:end]
pin_tensor(tensor)
region.pinned_addresses.append(tensor.data_ptr())
region.is_pinned = True
pools = [
region.create_next_canonical_view(size).view(-1) for size in tensor_sizes
]
except Exception:
region.cleanup()
raise
return pools, [], region
def release_pinned_state(
runtimes: list[HiSparseRuntime], pinned_host_pools: list[torch.Tensor]
runtimes: list[HiSparseRuntime],
pinned_host_pools: list[torch.Tensor],
shared_host_region: SharedOffloadRegion | None = None,
) -> None:
"""Synchronize and release registered host KV pools."""
if pinned_host_pools:
if pinned_host_pools or shared_host_region is not None:
try:
torch.accelerator.synchronize()
except RuntimeError as e:
@@ -208,7 +388,7 @@ def release_pinned_state(
"HiSparse: CUDA context unusable at teardown (%s); leaving "
"%d host-pool tensors pinned for kernel exit reclaim.",
e,
len(pinned_host_pools),
len(pinned_host_pools) + int(shared_host_region is not None),
)
return
@@ -240,6 +420,8 @@ def release_pinned_state(
del runtime._host_cache
del runtime.registered_host_pool
del runtime.hot_backing
if shared_host_region is not None:
shared_host_region.cleanup()
def hisparse_prefill_staging_remap(
@@ -536,6 +718,7 @@ class HiSparseRuntime:
self.eager_host_mirror = config.eager_host_mirror
self.resident_source_index = -1
self.request_state_indices: torch.Tensor | None = None
self.shared_host_region: SharedOffloadRegion | None = None
@property
def host_cache(self) -> torch.Tensor: