Signed-off-by: Matthew Bonanni <mbonanni@redhat.com> Signed-off-by: Chao Lei <leichao139636@163.com> Signed-off-by: Lucas Wilkinson <lwilkinson@neuralmagic.com> Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com> Co-authored-by: Chao Lei <leichao139636@163.com> Co-authored-by: Claude Fable 5 <noreply@anthropic.com> Co-authored-by: Matthew Bonanni <mbonanni@redhat.com> Co-authored-by: OpenAI Codex <codex@openai.com> Co-authored-by: OpenAI Codex <noreply@openai.com>
This commit is contained in:
co-authored by
Chao Lei
Claude Fable 5
Matthew Bonanni
OpenAI Codex
OpenAI Codex
parent
ebe1dec2da
commit
e19a3e172e
@@ -147,6 +147,12 @@ def _select_written_row_mirrors(
|
||||
)
|
||||
|
||||
|
||||
def _is_hisparse_host_writer(
|
||||
shared_host_region: SharedOffloadRegion | None,
|
||||
) -> bool:
|
||||
return shared_host_region is None or get_tensor_model_parallel_rank() == 0
|
||||
|
||||
|
||||
def _create_hisparse_host_events(
|
||||
shared_host_region: SharedOffloadRegion | None,
|
||||
is_host_writer: bool,
|
||||
@@ -200,15 +206,26 @@ class HiSparseConnectorWorker:
|
||||
raise RuntimeError("HiSparse connector found no hot-cache handles.")
|
||||
hot_backings: dict[int, torch.Tensor] = {}
|
||||
registered_host_pools: dict[int, torch.Tensor] = {}
|
||||
shared_host_regions: dict[int, SharedOffloadRegion] = {}
|
||||
for cache in cache_handles:
|
||||
hot_backing = cache.runtime.hot_backing
|
||||
hot_backings[hot_backing.untyped_storage().data_ptr()] = hot_backing
|
||||
registered_pool = cache.runtime.registered_host_pool
|
||||
registered_host_pools[registered_pool.data_ptr()] = registered_pool
|
||||
if (region := cache.runtime.shared_host_region) is not None:
|
||||
shared_host_regions[id(region)] = region
|
||||
if len(hot_backings) != 1:
|
||||
raise RuntimeError("HiSparse hot tensors must share one GPU backing.")
|
||||
hot_backing = next(iter(hot_backings.values()))
|
||||
pinned_host_pools = list(registered_host_pools.values())
|
||||
if len(shared_host_regions) > 1:
|
||||
raise RuntimeError("HiSparse caches must share one host region.")
|
||||
shared_host_region = next(iter(shared_host_regions.values()), None)
|
||||
pinned_host_pools = (
|
||||
[]
|
||||
if shared_host_region is not None
|
||||
else list(registered_host_pools.values())
|
||||
)
|
||||
is_host_writer = _is_hisparse_host_writer(shared_host_region)
|
||||
|
||||
resident = cache_handles[0].view
|
||||
assert resident is not None
|
||||
@@ -223,11 +240,14 @@ class HiSparseConnectorWorker:
|
||||
host_num_blocks,
|
||||
hot_backing.device,
|
||||
pinned_host_pools,
|
||||
shared_host_region=shared_host_region,
|
||||
is_host_writer=is_host_writer,
|
||||
)
|
||||
except Exception:
|
||||
release_pinned_state(
|
||||
[cache.runtime for cache in cache_handles],
|
||||
pinned_host_pools,
|
||||
shared_host_region,
|
||||
)
|
||||
raise
|
||||
|
||||
@@ -866,6 +886,8 @@ class HiSparseConnectorWorker:
|
||||
if self.dma_stream is not None:
|
||||
self.dma_stream.synchronize()
|
||||
release_pinned_state(
|
||||
[cache.runtime for cache in self.cache_handles], self.pinned_host_pools
|
||||
[cache.runtime for cache in self.cache_handles],
|
||||
self.pinned_host_pools,
|
||||
self.shared_host_region,
|
||||
)
|
||||
self._initialized = False
|
||||
|
||||
@@ -17,6 +17,7 @@ from vllm.v1.kv_cache_interface import (
|
||||
MLAAttentionSpec,
|
||||
SlidingWindowMLASpec,
|
||||
SlidingWindowSpec,
|
||||
UniformTypeKVCacheSpecs,
|
||||
iter_layer_specs,
|
||||
)
|
||||
from vllm.v1.kv_offload.config import (
|
||||
@@ -29,7 +30,7 @@ from vllm.v1.kv_offload.config import (
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.v1.kv_cache_interface import KVCacheConfig
|
||||
from vllm.v1.kv_cache_interface import KVCacheConfig, KVCacheGroupSpec
|
||||
|
||||
|
||||
def get_offloading_group_ids(kv_cache_config: "KVCacheConfig") -> tuple[int, ...]:
|
||||
@@ -42,6 +43,19 @@ def get_offloading_group_ids(kv_cache_config: "KVCacheConfig") -> tuple[int, ...
|
||||
)
|
||||
|
||||
|
||||
def _group_kv_bytes_per_block(group: "KVCacheGroupSpec") -> int:
|
||||
"""Return the physical bytes occupied by one block of a cache group.
|
||||
|
||||
Worker configs may retain ``UniformTypeKVCacheSpecs`` while scheduler
|
||||
configs flatten that wrapper to one representative per-layer spec. Keep
|
||||
the result invariant across those two representations.
|
||||
"""
|
||||
spec = group.kv_cache_spec
|
||||
if isinstance(spec, UniformTypeKVCacheSpecs):
|
||||
return spec.page_size_bytes
|
||||
return spec.page_size_bytes * len(group.layer_names)
|
||||
|
||||
|
||||
def build_offloading_config(
|
||||
vllm_config: "VllmConfig",
|
||||
kv_cache_config: "KVCacheConfig",
|
||||
@@ -133,7 +147,7 @@ def build_offloading_config(
|
||||
worker_kv_bytes_per_block = total_gpu_kv_bytes // kv_cache_config.num_blocks
|
||||
elif kv_cache_config.num_blocks > 0:
|
||||
worker_kv_bytes_per_block = sum(
|
||||
group.kv_cache_spec.page_size_bytes for _, group in selected_groups
|
||||
_group_kv_bytes_per_block(group) for _, group in selected_groups
|
||||
)
|
||||
|
||||
single_group_spec = (
|
||||
|
||||
+44
-27
@@ -120,27 +120,32 @@ def init_hisparse_kv_cache(
|
||||
block_tables: "BlockTables",
|
||||
) -> dict[str, torch.Tensor]:
|
||||
"""Allocate and bind HiSparse caches within the caller's allocation context."""
|
||||
host_pool = HiSparseHostPool()
|
||||
kv_caches = allocate_hisparse_kv_caches(
|
||||
kv_cache_config,
|
||||
device,
|
||||
vllm_config.cache_config.get_resolved_kv_cache_layout(),
|
||||
kernel_block_sizes,
|
||||
host_pool,
|
||||
)
|
||||
cache_handles = bind_hisparse_kv_caches(
|
||||
forward_context=forward_context,
|
||||
kv_cache_config=kv_cache_config,
|
||||
kv_caches=kv_caches,
|
||||
block_tables=block_tables,
|
||||
host_pool=host_pool,
|
||||
)
|
||||
initialize_hisparse_runtime_buffers(
|
||||
cache_handles,
|
||||
max_num_reqs=vllm_config.scheduler_config.max_num_seqs,
|
||||
max_num_batched_tokens=vllm_config.scheduler_config.max_num_batched_tokens,
|
||||
)
|
||||
return kv_caches
|
||||
host_pool = HiSparseHostPool(vllm_config, kv_cache_config)
|
||||
try:
|
||||
kv_caches = allocate_hisparse_kv_caches(
|
||||
kv_cache_config,
|
||||
device,
|
||||
vllm_config.cache_config.get_resolved_kv_cache_layout(),
|
||||
kernel_block_sizes,
|
||||
host_pool,
|
||||
)
|
||||
cache_handles = bind_hisparse_kv_caches(
|
||||
forward_context=forward_context,
|
||||
kv_cache_config=kv_cache_config,
|
||||
kv_caches=kv_caches,
|
||||
block_tables=block_tables,
|
||||
host_pool=host_pool,
|
||||
)
|
||||
initialize_hisparse_runtime_buffers(
|
||||
cache_handles,
|
||||
max_num_reqs=vllm_config.scheduler_config.max_num_seqs,
|
||||
max_num_batched_tokens=vllm_config.scheduler_config.max_num_batched_tokens,
|
||||
)
|
||||
return kv_caches
|
||||
except Exception:
|
||||
if host_pool.shared_region is not None:
|
||||
host_pool.shared_region.cleanup()
|
||||
raise
|
||||
|
||||
|
||||
def _get_hisparse_cache(
|
||||
@@ -166,13 +171,24 @@ def release_hisparse_profiling_cache(forward_context: dict[str, Any]) -> None:
|
||||
if not runtimes:
|
||||
return
|
||||
|
||||
registered_pools = list(
|
||||
{
|
||||
runtime.registered_host_pool.data_ptr(): runtime.registered_host_pool
|
||||
for runtime in runtimes.values()
|
||||
}.values()
|
||||
shared_regions = {
|
||||
id(runtime.shared_host_region): runtime.shared_host_region
|
||||
for runtime in runtimes.values()
|
||||
if runtime.shared_host_region is not None
|
||||
}
|
||||
assert len(shared_regions) <= 1
|
||||
shared_region = next(iter(shared_regions.values()), None)
|
||||
registered_pools = (
|
||||
[]
|
||||
if shared_region is not None
|
||||
else list(
|
||||
{
|
||||
runtime.registered_host_pool.data_ptr(): runtime.registered_host_pool
|
||||
for runtime in runtimes.values()
|
||||
}.values()
|
||||
)
|
||||
)
|
||||
release_pinned_state(list(runtimes.values()), registered_pools)
|
||||
release_pinned_state(list(runtimes.values()), registered_pools, shared_region)
|
||||
for cache in cache_handles:
|
||||
cache.mirror_staging_cache = None
|
||||
cache.mirror_staging_slots = None
|
||||
@@ -257,6 +273,7 @@ def bind_hisparse_kv_caches(
|
||||
assert source_cache.untyped_storage().data_ptr() == (
|
||||
host_pool.registered.untyped_storage().data_ptr()
|
||||
)
|
||||
cache_handle.runtime.shared_host_region = host_pool.shared_region
|
||||
cache_handle.runtime.bind_source_cache(
|
||||
source_cache,
|
||||
registered_host_pool=host_pool.registered,
|
||||
|
||||
@@ -9,7 +9,11 @@ from vllm.config import VllmConfig
|
||||
from vllm.config.kv_transfer import hisparse_host_pool_gib
|
||||
from vllm.logger import init_logger
|
||||
from vllm.utils.math_utils import cdiv
|
||||
from vllm.v1.hisparse.runtime import ResolvedHiSparseConfig
|
||||
from vllm.v1.hisparse.runtime import (
|
||||
ResolvedHiSparseConfig,
|
||||
get_hisparse_host_block_stride,
|
||||
use_shared_hisparse_host_pool,
|
||||
)
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
HiSparseHotSpec,
|
||||
HiSparseResidentSpec,
|
||||
@@ -36,6 +40,8 @@ class HiSparseLayout:
|
||||
source_group: KVCacheGroupSpec
|
||||
device_groups: list[KVCacheGroupSpec]
|
||||
host_num_blocks: int
|
||||
host_block_stride: int
|
||||
shared_host_pool: bool
|
||||
|
||||
|
||||
def get_hisparse_kv_cache_groups(
|
||||
@@ -221,9 +227,12 @@ def create_hisparse_layout(
|
||||
regular_groups = groups[1:]
|
||||
gpu_groups = [indexer_group, *resident_groups, *hot_groups, *regular_groups]
|
||||
|
||||
host_num_blocks = host_budget // sum(
|
||||
spec.page_size_bytes for spec in source_specs.values()
|
||||
shared_host_pool = use_shared_hisparse_host_pool(vllm_config)
|
||||
host_block_stride = get_hisparse_host_block_stride(
|
||||
sum(spec.page_size_bytes for spec in source_specs.values()),
|
||||
use_shared_host_pool=shared_host_pool,
|
||||
)
|
||||
host_num_blocks = host_budget // host_block_stride
|
||||
if host_num_blocks <= 0:
|
||||
raise ValueError("HiSparse has no allocatable host blocks.")
|
||||
|
||||
@@ -231,6 +240,8 @@ def create_hisparse_layout(
|
||||
source_group=source_group,
|
||||
device_groups=gpu_groups,
|
||||
host_num_blocks=host_num_blocks,
|
||||
host_block_stride=host_block_stride,
|
||||
shared_host_pool=shared_host_pool,
|
||||
)
|
||||
|
||||
|
||||
@@ -339,6 +350,8 @@ def get_hisparse_kv_cache_config(
|
||||
kv_cache_tensors=kv_cache_tensors,
|
||||
kv_cache_groups=[*host_groups, *device_groups],
|
||||
hisparse_host_num_blocks=hisparse_layout.host_num_blocks,
|
||||
hisparse_host_block_stride=hisparse_layout.host_block_stride,
|
||||
hisparse_shared_host_pool=hisparse_layout.shared_host_pool,
|
||||
prefix_cache_retention_interval=(
|
||||
vllm_config.cache_config.prefix_cache_retention_interval
|
||||
),
|
||||
|
||||
+189
-6
@@ -4,6 +4,8 @@
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import math
|
||||
import mmap
|
||||
import time
|
||||
from collections.abc import Callable
|
||||
from dataclasses import dataclass
|
||||
@@ -13,17 +15,22 @@ import psutil
|
||||
import torch
|
||||
|
||||
from vllm.config import CUDAGraphMode, VllmConfig
|
||||
from vllm.distributed import get_tp_group
|
||||
from vllm.forward_context import get_forward_context
|
||||
from vllm.logger import init_logger
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.math_utils import round_up
|
||||
from vllm.utils.torch_utils import current_stream
|
||||
from vllm.v1.kv_offload.cpu.shared_offload_region import SharedOffloadRegion
|
||||
from vllm.v1.simple_kv_offload.cuda_mem_ops import pin_tensor
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
HOST_REGISTER_CHUNK_BYTES = 256 * 2**30
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.v1.attention.backends.mla.index_group import HiSparseMLAIndexGroup
|
||||
from vllm.v1.kv_cache_interface import KVCacheConfig
|
||||
|
||||
# fp8_ds_mla KV row: 512 B quantized NoPE + 16 B scales + 128 B RoPE.
|
||||
FP8_DS_MLA_ROW_BYTES = 656
|
||||
@@ -185,22 +192,195 @@ def allocate_pinned_host_pool(size: int) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
class HiSparseHostPool:
|
||||
"""Keep the single host backing and its exact CUDA registration range."""
|
||||
|
||||
def __init__(self) -> None:
|
||||
def __init__(self, vllm_config: VllmConfig, kv_cache_config: KVCacheConfig) -> None:
|
||||
self.vllm_config = vllm_config
|
||||
self.kv_cache_config = kv_cache_config
|
||||
self.backing: torch.Tensor | None = None
|
||||
self.registered: torch.Tensor | None = None
|
||||
self.shared_region: SharedOffloadRegion | None = None
|
||||
|
||||
def allocate(self, size: int) -> torch.Tensor:
|
||||
assert self.backing is None, "HiSparse host pool is already allocated."
|
||||
check_hisparse_host_memory(size)
|
||||
self.backing, self.registered = allocate_pinned_host_pool(size)
|
||||
config = self.kv_cache_config
|
||||
if config.hisparse_shared_host_pool:
|
||||
tensor_sizes = [
|
||||
size
|
||||
for _, size in sorted(
|
||||
(tensor.offset + i * tensor.layer_stride, tensor.layer_stride)
|
||||
for tensor in config.kv_cache_tensors
|
||||
if tensor.host_resident
|
||||
for i in range(len(tensor.layers))
|
||||
)
|
||||
]
|
||||
assert config.hisparse_host_num_blocks is not None
|
||||
assert config.hisparse_host_block_stride is not None
|
||||
_, _, self.shared_region = allocate_hisparse_host_pools(
|
||||
self.vllm_config,
|
||||
tensor_sizes,
|
||||
config.hisparse_host_num_blocks,
|
||||
config.hisparse_host_block_stride,
|
||||
use_shared_host_pool=True,
|
||||
)
|
||||
assert self.shared_region is not None
|
||||
self.registered = self.shared_region.base_tensor
|
||||
self.backing = self.registered[:size]
|
||||
else:
|
||||
check_hisparse_host_memory(size)
|
||||
self.backing, self.registered = allocate_pinned_host_pool(size)
|
||||
return self.backing
|
||||
|
||||
|
||||
def use_shared_hisparse_host_pool(vllm_config: VllmConfig) -> bool:
|
||||
"""Whether replicated MLA host KV can share one local mmap."""
|
||||
parallel = vllm_config.parallel_config
|
||||
return (
|
||||
current_platform.is_cuda_alike()
|
||||
and parallel.tensor_parallel_size > 1
|
||||
and parallel.pipeline_parallel_size == 1
|
||||
and parallel.prefill_context_parallel_size == 1
|
||||
and parallel.decode_context_parallel_size == 1
|
||||
and parallel.world_size == parallel.tensor_parallel_size
|
||||
and parallel.distributed_executor_backend == "mp"
|
||||
and parallel.nnodes_within_dp == 1
|
||||
)
|
||||
|
||||
|
||||
def get_hisparse_host_block_stride(
|
||||
page_size_bytes: int, *, use_shared_host_pool: bool
|
||||
) -> int:
|
||||
if use_shared_host_pool:
|
||||
return round_up(page_size_bytes, SharedOffloadRegion.BLOCK_SIZE_ALIGNMENT)
|
||||
return page_size_bytes
|
||||
|
||||
|
||||
def _hisparse_registration_ranges(
|
||||
tensor_sizes: list[int],
|
||||
num_blocks: int,
|
||||
host_block_stride: int,
|
||||
max_chunk_bytes: int = HOST_REGISTER_CHUNK_BYTES,
|
||||
) -> tuple[tuple[int, int], ...]:
|
||||
"""Split a shared pool without bisecting any tensor's host block.
|
||||
|
||||
CUDA batch copies reject descriptors spanning adjacent registrations.
|
||||
"""
|
||||
total_size = num_blocks * host_block_stride
|
||||
if total_size % mmap.PAGESIZE:
|
||||
raise ValueError("HiSparse host pool must be OS-page aligned.")
|
||||
if max_chunk_bytes < mmap.PAGESIZE:
|
||||
raise ValueError("HiSparse registration chunks must span at least one page.")
|
||||
|
||||
tensors: list[tuple[int, int, int]] = []
|
||||
tensor_start = 0
|
||||
for tensor_size in tensor_sizes:
|
||||
if tensor_size % num_blocks:
|
||||
raise ValueError(
|
||||
f"HiSparse host tensor size {tensor_size} is not divisible by "
|
||||
f"{num_blocks} blocks."
|
||||
)
|
||||
tensor_end = tensor_start + tensor_size
|
||||
tensors.append((tensor_start, tensor_end, tensor_size // num_blocks))
|
||||
tensor_start = tensor_end
|
||||
if tensor_start > total_size:
|
||||
raise ValueError("HiSparse host tensors exceed the shared host pool.")
|
||||
|
||||
def last_safe_boundary(lower: int, upper: int) -> int | None:
|
||||
best = None
|
||||
for start, end, block_bytes in tensors:
|
||||
first_block = max(0, (lower - start) // block_bytes + 1)
|
||||
last_block = min(num_blocks, (min(upper, end) - start) // block_bytes)
|
||||
if first_block > last_block:
|
||||
continue
|
||||
divisor = math.gcd(block_bytes, mmap.PAGESIZE)
|
||||
if start % divisor:
|
||||
continue
|
||||
period = mmap.PAGESIZE // divisor
|
||||
if period == 1:
|
||||
residue = 0
|
||||
else:
|
||||
# Find block boundaries that are also OS-page boundaries.
|
||||
residue = (
|
||||
(-start // divisor) * pow(block_bytes // divisor, -1, period)
|
||||
) % period
|
||||
block = last_block - (last_block - residue) % period
|
||||
if block >= first_block:
|
||||
candidate = start + block * block_bytes
|
||||
best = candidate if best is None else max(best, candidate)
|
||||
|
||||
if upper >= tensor_start:
|
||||
candidate = upper // mmap.PAGESIZE * mmap.PAGESIZE
|
||||
if candidate >= tensor_start and candidate > lower:
|
||||
best = candidate if best is None else max(best, candidate)
|
||||
return best
|
||||
|
||||
ranges: list[tuple[int, int]] = []
|
||||
start = 0
|
||||
while total_size - start > max_chunk_bytes:
|
||||
end = last_safe_boundary(start, start + max_chunk_bytes)
|
||||
if end is None:
|
||||
raise ValueError(
|
||||
"A HiSparse host block is too large for the registration chunk limit."
|
||||
)
|
||||
ranges.append((start, end))
|
||||
start = end
|
||||
ranges.append((start, total_size))
|
||||
return tuple(ranges)
|
||||
|
||||
|
||||
def allocate_hisparse_host_pools(
|
||||
vllm_config: VllmConfig,
|
||||
tensor_sizes: list[int],
|
||||
num_blocks: int,
|
||||
host_block_stride: int,
|
||||
*,
|
||||
use_shared_host_pool: bool,
|
||||
) -> tuple[list[torch.Tensor], list[torch.Tensor], SharedOffloadRegion | None]:
|
||||
"""Allocate private host tensors or one mmap shared by local TP ranks."""
|
||||
if not use_shared_host_pool:
|
||||
check_hisparse_host_memory(sum(tensor_sizes))
|
||||
private_pools = [allocate_pinned_host_pool(size) for size in tensor_sizes]
|
||||
return (
|
||||
[pool for pool, _ in private_pools],
|
||||
[registered for _, registered in private_pools],
|
||||
None,
|
||||
)
|
||||
|
||||
region = SharedOffloadRegion(
|
||||
engine_id=(
|
||||
f"hisparse_{vllm_config.instance_id}_"
|
||||
f"dp{vllm_config.parallel_config.data_parallel_index}"
|
||||
),
|
||||
num_chunks=1,
|
||||
rank=0,
|
||||
kv_bytes_per_chunk=num_blocks * host_block_stride,
|
||||
cpu_page_size=sum(tensor_sizes),
|
||||
barrier=get_tp_group().barrier,
|
||||
creator_memory_check=check_hisparse_host_memory,
|
||||
populate_only_on_creator=True,
|
||||
)
|
||||
try:
|
||||
for start, end in _hisparse_registration_ranges(
|
||||
tensor_sizes, num_blocks, host_block_stride
|
||||
):
|
||||
tensor = region.base_tensor[start:end]
|
||||
pin_tensor(tensor)
|
||||
region.pinned_addresses.append(tensor.data_ptr())
|
||||
region.is_pinned = True
|
||||
pools = [
|
||||
region.create_next_canonical_view(size).view(-1) for size in tensor_sizes
|
||||
]
|
||||
except Exception:
|
||||
region.cleanup()
|
||||
raise
|
||||
return pools, [], region
|
||||
|
||||
|
||||
def release_pinned_state(
|
||||
runtimes: list[HiSparseRuntime], pinned_host_pools: list[torch.Tensor]
|
||||
runtimes: list[HiSparseRuntime],
|
||||
pinned_host_pools: list[torch.Tensor],
|
||||
shared_host_region: SharedOffloadRegion | None = None,
|
||||
) -> None:
|
||||
"""Synchronize and release registered host KV pools."""
|
||||
if pinned_host_pools:
|
||||
if pinned_host_pools or shared_host_region is not None:
|
||||
try:
|
||||
torch.accelerator.synchronize()
|
||||
except RuntimeError as e:
|
||||
@@ -208,7 +388,7 @@ def release_pinned_state(
|
||||
"HiSparse: CUDA context unusable at teardown (%s); leaving "
|
||||
"%d host-pool tensors pinned for kernel exit reclaim.",
|
||||
e,
|
||||
len(pinned_host_pools),
|
||||
len(pinned_host_pools) + int(shared_host_region is not None),
|
||||
)
|
||||
return
|
||||
|
||||
@@ -240,6 +420,8 @@ def release_pinned_state(
|
||||
del runtime._host_cache
|
||||
del runtime.registered_host_pool
|
||||
del runtime.hot_backing
|
||||
if shared_host_region is not None:
|
||||
shared_host_region.cleanup()
|
||||
|
||||
|
||||
def hisparse_prefill_staging_remap(
|
||||
@@ -536,6 +718,7 @@ class HiSparseRuntime:
|
||||
self.eager_host_mirror = config.eager_host_mirror
|
||||
self.resident_source_index = -1
|
||||
self.request_state_indices: torch.Tensor | None = None
|
||||
self.shared_host_region: SharedOffloadRegion | None = None
|
||||
|
||||
@property
|
||||
def host_cache(self) -> torch.Tensor:
|
||||
|
||||
@@ -1435,6 +1435,12 @@ class KVCacheConfig:
|
||||
hisparse_host_num_blocks: int | None = None
|
||||
"""Capacity of the dedicated HiSparse host-block manager, when enabled."""
|
||||
|
||||
hisparse_host_block_stride: int | None = None
|
||||
"""Physical bytes between consecutive HiSparse host blocks."""
|
||||
|
||||
hisparse_shared_host_pool: bool = False
|
||||
"""Whether local TP ranks share one physical HiSparse host pool."""
|
||||
|
||||
@cached_property
|
||||
def transfer_group_ids(self) -> tuple[int, ...]:
|
||||
"""IDs of cache groups that participate in external KV transfer."""
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import contextlib
|
||||
import errno
|
||||
import mmap
|
||||
import os
|
||||
@@ -27,6 +28,10 @@ def _wait_for_file_size(fd: int, expected_size: int, timeout: float = 30.0) -> N
|
||||
while True:
|
||||
if os.fstat(fd).st_size >= expected_size:
|
||||
return
|
||||
if os.fstat(fd).st_nlink == 0:
|
||||
raise RuntimeError(
|
||||
"Shared offload region creator failed during initialization."
|
||||
)
|
||||
if time.monotonic() > deadline:
|
||||
raise TimeoutError(
|
||||
f"Timed out waiting for mmap file to reach {expected_size} bytes"
|
||||
@@ -75,6 +80,9 @@ class SharedOffloadRegion:
|
||||
given, the path is unlinked once every worker has mapped the file, so
|
||||
the kernel reclaims the memory when the last worker exits, no matter
|
||||
how it exits; mappings taken before the unlink stay valid.
|
||||
|
||||
Creator-only population pre-faults the entire region before the barrier
|
||||
and requires that barrier to keep joiners from using unpopulated pages.
|
||||
"""
|
||||
|
||||
BLOCK_SIZE_ALIGNMENT: int = mmap.PAGESIZE
|
||||
@@ -87,7 +95,12 @@ class SharedOffloadRegion:
|
||||
kv_bytes_per_chunk: int,
|
||||
cpu_page_size: int,
|
||||
barrier: Callable[[], None] | None = None,
|
||||
*,
|
||||
creator_memory_check: Callable[[int], None] | None = None,
|
||||
populate_only_on_creator: bool = False,
|
||||
) -> None:
|
||||
if populate_only_on_creator and barrier is None:
|
||||
raise ValueError("Creator-only population requires a barrier.")
|
||||
self.page_size = mmap.PAGESIZE
|
||||
assert kv_bytes_per_chunk % self.page_size == 0
|
||||
|
||||
@@ -112,25 +125,18 @@ class SharedOffloadRegion:
|
||||
# Joiner path — another worker won O_EXCL. Reopen and wait
|
||||
# for the file to reach expected size.
|
||||
self.fd = os.open(self.mmap_path, os.O_RDWR)
|
||||
try:
|
||||
_wait_for_file_size(self.fd, self.total_size_bytes)
|
||||
except (TimeoutError, OSError):
|
||||
os.close(self.fd)
|
||||
raise
|
||||
_wait_for_file_size(self.fd, self.total_size_bytes)
|
||||
logger.info("Opened existing mmap file %s", self.mmap_path)
|
||||
else:
|
||||
# Creator path. We won O_EXCL, so we own the file: any
|
||||
# failure here must clean up so concurrent joiners don't
|
||||
# land on a 0-byte stub and spin in _wait_for_file_size
|
||||
# for the full 30 s timeout.
|
||||
try:
|
||||
check_shm_free_space(self.total_size_bytes)
|
||||
os.ftruncate(self.fd, self.total_size_bytes)
|
||||
except (RuntimeError, OSError):
|
||||
os.unlink(self.mmap_path)
|
||||
os.close(self.fd)
|
||||
raise
|
||||
self._creator = True
|
||||
if creator_memory_check is not None:
|
||||
creator_memory_check(self.total_size_bytes)
|
||||
check_shm_free_space(self.total_size_bytes)
|
||||
os.ftruncate(self.fd, self.total_size_bytes)
|
||||
logger.info(
|
||||
"Created mmap file %s (%.2f GB)",
|
||||
self.mmap_path,
|
||||
@@ -143,10 +149,21 @@ class SharedOffloadRegion:
|
||||
flags=mmap.MAP_SHARED,
|
||||
prot=mmap.PROT_READ | mmap.PROT_WRITE,
|
||||
)
|
||||
|
||||
if populate_only_on_creator and self._creator:
|
||||
populate_write_fn = _get_populate_write_fn(self.mmap_obj)
|
||||
populate_write_fn(self.mmap_obj, 0, self.total_size_bytes)
|
||||
except Exception:
|
||||
if self._creator:
|
||||
os.unlink(self.mmap_path)
|
||||
with contextlib.suppress(FileNotFoundError):
|
||||
os.unlink(self.mmap_path)
|
||||
self._creator = False
|
||||
if hasattr(self, "mmap_obj") and self.mmap_obj is not None:
|
||||
self.mmap_obj.close()
|
||||
self.mmap_obj = None
|
||||
if hasattr(self, "fd") and self.fd is not None:
|
||||
os.close(self.fd)
|
||||
self.fd = None
|
||||
# Peers block inside the barrier until the collective times out if
|
||||
# we die before reaching it. Arrive anyway so every worker calls
|
||||
# barrier() exactly once and they fail on their own errors instead
|
||||
@@ -169,16 +186,28 @@ class SharedOffloadRegion:
|
||||
barrier()
|
||||
except Exception:
|
||||
if self._creator:
|
||||
os.unlink(self.mmap_path)
|
||||
with contextlib.suppress(FileNotFoundError):
|
||||
os.unlink(self.mmap_path)
|
||||
self._creator = False
|
||||
self.mmap_obj.close()
|
||||
os.close(self.fd)
|
||||
self.mmap_obj = None
|
||||
self.fd = None
|
||||
raise
|
||||
if self._creator:
|
||||
os.unlink(self.mmap_path)
|
||||
self._creator = False
|
||||
logger.info("Unlinked mmap file %s", self.mmap_path)
|
||||
|
||||
self._base = torch.frombuffer(memoryview(self.mmap_obj), dtype=torch.int8)
|
||||
self._views: list[torch.Tensor] = []
|
||||
self._canonical_offset = 0
|
||||
self.is_pinned: bool = False
|
||||
self.pinned_addresses: list[int] = []
|
||||
|
||||
if populate_only_on_creator:
|
||||
return
|
||||
|
||||
populate_write_fn = _get_populate_write_fn(self.mmap_obj)
|
||||
|
||||
if rank is not None:
|
||||
@@ -205,10 +234,11 @@ class SharedOffloadRegion:
|
||||
"MADV_POPULATE_WRITE entire region: %.3f s", time.perf_counter() - _t0
|
||||
)
|
||||
|
||||
self._base = torch.frombuffer(memoryview(self.mmap_obj), dtype=torch.int8)
|
||||
self._views: list[torch.Tensor] = []
|
||||
self._canonical_offset = 0
|
||||
self.is_pinned: bool = False
|
||||
@property
|
||||
def base_tensor(self) -> torch.Tensor:
|
||||
if self._base is None:
|
||||
raise RuntimeError("Shared offload region has been released.")
|
||||
return self._base
|
||||
|
||||
def create_next_worker_view(self, tensor_page_size: int) -> torch.Tensor:
|
||||
"""Allocate a strided int8 view for this worker, one canonical tensor.
|
||||
@@ -311,13 +341,18 @@ class SharedOffloadRegion:
|
||||
if self.is_pinned and self._base is not None:
|
||||
if current_platform.is_cuda_alike():
|
||||
base_ptr = self._base.data_ptr()
|
||||
result = torch.cuda.cudart().cudaHostUnregister(base_ptr)
|
||||
if result.value != 0:
|
||||
logger.warning(
|
||||
"cudaHostUnregister failed for rank=%d (code=%d)",
|
||||
self.rank,
|
||||
result,
|
||||
)
|
||||
addresses = self.pinned_addresses or [base_ptr]
|
||||
for address in reversed(addresses):
|
||||
result = torch.cuda.cudart().cudaHostUnregister(address)
|
||||
if result.value != 0:
|
||||
logger.warning(
|
||||
"cudaHostUnregister failed for rank=%d, "
|
||||
"address=%#x (code=%d)",
|
||||
self.rank,
|
||||
address,
|
||||
result.value,
|
||||
)
|
||||
self.pinned_addresses.clear()
|
||||
self.is_pinned = False
|
||||
# Release views before _base: each view holds a _base reference and a
|
||||
# direct StorageImpl reference. Freeing views first lets both refcounts
|
||||
|
||||
Reference in New Issue
Block a user