[5/N] Share HiSparse host cache across TP ranks (reopens #52760) (#56629)

Signed-off-by: Matthew Bonanni <mbonanni@redhat.com>
Signed-off-by: Chao Lei <leichao139636@163.com>
Signed-off-by: Lucas Wilkinson <lwilkinson@neuralmagic.com>
Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com>
Co-authored-by: Chao Lei <leichao139636@163.com>
Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
Co-authored-by: Matthew Bonanni <mbonanni@redhat.com>
Co-authored-by: OpenAI Codex <codex@openai.com>
Co-authored-by: OpenAI Codex <noreply@openai.com>
This commit is contained in:
Lucas Wilkinson
2026-09-12 19:48:25 -04:00
committed by GitHub
co-authored by Chao Lei Claude Fable 5 Matthew Bonanni OpenAI Codex OpenAI Codex
parent ebe1dec2da
commit e19a3e172e
14 changed files with 878 additions and 85 deletions
@@ -147,6 +147,12 @@ def _select_written_row_mirrors(
)
def _is_hisparse_host_writer(
shared_host_region: SharedOffloadRegion | None,
) -> bool:
return shared_host_region is None or get_tensor_model_parallel_rank() == 0
def _create_hisparse_host_events(
shared_host_region: SharedOffloadRegion | None,
is_host_writer: bool,
@@ -200,15 +206,26 @@ class HiSparseConnectorWorker:
raise RuntimeError("HiSparse connector found no hot-cache handles.")
hot_backings: dict[int, torch.Tensor] = {}
registered_host_pools: dict[int, torch.Tensor] = {}
shared_host_regions: dict[int, SharedOffloadRegion] = {}
for cache in cache_handles:
hot_backing = cache.runtime.hot_backing
hot_backings[hot_backing.untyped_storage().data_ptr()] = hot_backing
registered_pool = cache.runtime.registered_host_pool
registered_host_pools[registered_pool.data_ptr()] = registered_pool
if (region := cache.runtime.shared_host_region) is not None:
shared_host_regions[id(region)] = region
if len(hot_backings) != 1:
raise RuntimeError("HiSparse hot tensors must share one GPU backing.")
hot_backing = next(iter(hot_backings.values()))
pinned_host_pools = list(registered_host_pools.values())
if len(shared_host_regions) > 1:
raise RuntimeError("HiSparse caches must share one host region.")
shared_host_region = next(iter(shared_host_regions.values()), None)
pinned_host_pools = (
[]
if shared_host_region is not None
else list(registered_host_pools.values())
)
is_host_writer = _is_hisparse_host_writer(shared_host_region)
resident = cache_handles[0].view
assert resident is not None
@@ -223,11 +240,14 @@ class HiSparseConnectorWorker:
host_num_blocks,
hot_backing.device,
pinned_host_pools,
shared_host_region=shared_host_region,
is_host_writer=is_host_writer,
)
except Exception:
release_pinned_state(
[cache.runtime for cache in cache_handles],
pinned_host_pools,
shared_host_region,
)
raise
@@ -866,6 +886,8 @@ class HiSparseConnectorWorker:
if self.dma_stream is not None:
self.dma_stream.synchronize()
release_pinned_state(
[cache.runtime for cache in self.cache_handles], self.pinned_host_pools
[cache.runtime for cache in self.cache_handles],
self.pinned_host_pools,
self.shared_host_region,
)
self._initialized = False
@@ -17,6 +17,7 @@ from vllm.v1.kv_cache_interface import (
MLAAttentionSpec,
SlidingWindowMLASpec,
SlidingWindowSpec,
UniformTypeKVCacheSpecs,
iter_layer_specs,
)
from vllm.v1.kv_offload.config import (
@@ -29,7 +30,7 @@ from vllm.v1.kv_offload.config import (
if TYPE_CHECKING:
from vllm.config import VllmConfig
from vllm.v1.kv_cache_interface import KVCacheConfig
from vllm.v1.kv_cache_interface import KVCacheConfig, KVCacheGroupSpec
def get_offloading_group_ids(kv_cache_config: "KVCacheConfig") -> tuple[int, ...]:
@@ -42,6 +43,19 @@ def get_offloading_group_ids(kv_cache_config: "KVCacheConfig") -> tuple[int, ...
)
def _group_kv_bytes_per_block(group: "KVCacheGroupSpec") -> int:
"""Return the physical bytes occupied by one block of a cache group.
Worker configs may retain ``UniformTypeKVCacheSpecs`` while scheduler
configs flatten that wrapper to one representative per-layer spec. Keep
the result invariant across those two representations.
"""
spec = group.kv_cache_spec
if isinstance(spec, UniformTypeKVCacheSpecs):
return spec.page_size_bytes
return spec.page_size_bytes * len(group.layer_names)
def build_offloading_config(
vllm_config: "VllmConfig",
kv_cache_config: "KVCacheConfig",
@@ -133,7 +147,7 @@ def build_offloading_config(
worker_kv_bytes_per_block = total_gpu_kv_bytes // kv_cache_config.num_blocks
elif kv_cache_config.num_blocks > 0:
worker_kv_bytes_per_block = sum(
group.kv_cache_spec.page_size_bytes for _, group in selected_groups
_group_kv_bytes_per_block(group) for _, group in selected_groups
)
single_group_spec = (
+44 -27
View File
@@ -120,27 +120,32 @@ def init_hisparse_kv_cache(
block_tables: "BlockTables",
) -> dict[str, torch.Tensor]:
"""Allocate and bind HiSparse caches within the caller's allocation context."""
host_pool = HiSparseHostPool()
kv_caches = allocate_hisparse_kv_caches(
kv_cache_config,
device,
vllm_config.cache_config.get_resolved_kv_cache_layout(),
kernel_block_sizes,
host_pool,
)
cache_handles = bind_hisparse_kv_caches(
forward_context=forward_context,
kv_cache_config=kv_cache_config,
kv_caches=kv_caches,
block_tables=block_tables,
host_pool=host_pool,
)
initialize_hisparse_runtime_buffers(
cache_handles,
max_num_reqs=vllm_config.scheduler_config.max_num_seqs,
max_num_batched_tokens=vllm_config.scheduler_config.max_num_batched_tokens,
)
return kv_caches
host_pool = HiSparseHostPool(vllm_config, kv_cache_config)
try:
kv_caches = allocate_hisparse_kv_caches(
kv_cache_config,
device,
vllm_config.cache_config.get_resolved_kv_cache_layout(),
kernel_block_sizes,
host_pool,
)
cache_handles = bind_hisparse_kv_caches(
forward_context=forward_context,
kv_cache_config=kv_cache_config,
kv_caches=kv_caches,
block_tables=block_tables,
host_pool=host_pool,
)
initialize_hisparse_runtime_buffers(
cache_handles,
max_num_reqs=vllm_config.scheduler_config.max_num_seqs,
max_num_batched_tokens=vllm_config.scheduler_config.max_num_batched_tokens,
)
return kv_caches
except Exception:
if host_pool.shared_region is not None:
host_pool.shared_region.cleanup()
raise
def _get_hisparse_cache(
@@ -166,13 +171,24 @@ def release_hisparse_profiling_cache(forward_context: dict[str, Any]) -> None:
if not runtimes:
return
registered_pools = list(
{
runtime.registered_host_pool.data_ptr(): runtime.registered_host_pool
for runtime in runtimes.values()
}.values()
shared_regions = {
id(runtime.shared_host_region): runtime.shared_host_region
for runtime in runtimes.values()
if runtime.shared_host_region is not None
}
assert len(shared_regions) <= 1
shared_region = next(iter(shared_regions.values()), None)
registered_pools = (
[]
if shared_region is not None
else list(
{
runtime.registered_host_pool.data_ptr(): runtime.registered_host_pool
for runtime in runtimes.values()
}.values()
)
)
release_pinned_state(list(runtimes.values()), registered_pools)
release_pinned_state(list(runtimes.values()), registered_pools, shared_region)
for cache in cache_handles:
cache.mirror_staging_cache = None
cache.mirror_staging_slots = None
@@ -257,6 +273,7 @@ def bind_hisparse_kv_caches(
assert source_cache.untyped_storage().data_ptr() == (
host_pool.registered.untyped_storage().data_ptr()
)
cache_handle.runtime.shared_host_region = host_pool.shared_region
cache_handle.runtime.bind_source_cache(
source_cache,
registered_host_pool=host_pool.registered,
+16 -3
View File
@@ -9,7 +9,11 @@ from vllm.config import VllmConfig
from vllm.config.kv_transfer import hisparse_host_pool_gib
from vllm.logger import init_logger
from vllm.utils.math_utils import cdiv
from vllm.v1.hisparse.runtime import ResolvedHiSparseConfig
from vllm.v1.hisparse.runtime import (
ResolvedHiSparseConfig,
get_hisparse_host_block_stride,
use_shared_hisparse_host_pool,
)
from vllm.v1.kv_cache_interface import (
HiSparseHotSpec,
HiSparseResidentSpec,
@@ -36,6 +40,8 @@ class HiSparseLayout:
source_group: KVCacheGroupSpec
device_groups: list[KVCacheGroupSpec]
host_num_blocks: int
host_block_stride: int
shared_host_pool: bool
def get_hisparse_kv_cache_groups(
@@ -221,9 +227,12 @@ def create_hisparse_layout(
regular_groups = groups[1:]
gpu_groups = [indexer_group, *resident_groups, *hot_groups, *regular_groups]
host_num_blocks = host_budget // sum(
spec.page_size_bytes for spec in source_specs.values()
shared_host_pool = use_shared_hisparse_host_pool(vllm_config)
host_block_stride = get_hisparse_host_block_stride(
sum(spec.page_size_bytes for spec in source_specs.values()),
use_shared_host_pool=shared_host_pool,
)
host_num_blocks = host_budget // host_block_stride
if host_num_blocks <= 0:
raise ValueError("HiSparse has no allocatable host blocks.")
@@ -231,6 +240,8 @@ def create_hisparse_layout(
source_group=source_group,
device_groups=gpu_groups,
host_num_blocks=host_num_blocks,
host_block_stride=host_block_stride,
shared_host_pool=shared_host_pool,
)
@@ -339,6 +350,8 @@ def get_hisparse_kv_cache_config(
kv_cache_tensors=kv_cache_tensors,
kv_cache_groups=[*host_groups, *device_groups],
hisparse_host_num_blocks=hisparse_layout.host_num_blocks,
hisparse_host_block_stride=hisparse_layout.host_block_stride,
hisparse_shared_host_pool=hisparse_layout.shared_host_pool,
prefix_cache_retention_interval=(
vllm_config.cache_config.prefix_cache_retention_interval
),
+189 -6
View File
@@ -4,6 +4,8 @@
from __future__ import annotations
import math
import mmap
import time
from collections.abc import Callable
from dataclasses import dataclass
@@ -13,17 +15,22 @@ import psutil
import torch
from vllm.config import CUDAGraphMode, VllmConfig
from vllm.distributed import get_tp_group
from vllm.forward_context import get_forward_context
from vllm.logger import init_logger
from vllm.platforms import current_platform
from vllm.utils.math_utils import round_up
from vllm.utils.torch_utils import current_stream
from vllm.v1.kv_offload.cpu.shared_offload_region import SharedOffloadRegion
from vllm.v1.simple_kv_offload.cuda_mem_ops import pin_tensor
logger = init_logger(__name__)
HOST_REGISTER_CHUNK_BYTES = 256 * 2**30
if TYPE_CHECKING:
from vllm.v1.attention.backends.mla.index_group import HiSparseMLAIndexGroup
from vllm.v1.kv_cache_interface import KVCacheConfig
# fp8_ds_mla KV row: 512 B quantized NoPE + 16 B scales + 128 B RoPE.
FP8_DS_MLA_ROW_BYTES = 656
@@ -185,22 +192,195 @@ def allocate_pinned_host_pool(size: int) -> tuple[torch.Tensor, torch.Tensor]:
class HiSparseHostPool:
"""Keep the single host backing and its exact CUDA registration range."""
def __init__(self) -> None:
def __init__(self, vllm_config: VllmConfig, kv_cache_config: KVCacheConfig) -> None:
self.vllm_config = vllm_config
self.kv_cache_config = kv_cache_config
self.backing: torch.Tensor | None = None
self.registered: torch.Tensor | None = None
self.shared_region: SharedOffloadRegion | None = None
def allocate(self, size: int) -> torch.Tensor:
assert self.backing is None, "HiSparse host pool is already allocated."
check_hisparse_host_memory(size)
self.backing, self.registered = allocate_pinned_host_pool(size)
config = self.kv_cache_config
if config.hisparse_shared_host_pool:
tensor_sizes = [
size
for _, size in sorted(
(tensor.offset + i * tensor.layer_stride, tensor.layer_stride)
for tensor in config.kv_cache_tensors
if tensor.host_resident
for i in range(len(tensor.layers))
)
]
assert config.hisparse_host_num_blocks is not None
assert config.hisparse_host_block_stride is not None
_, _, self.shared_region = allocate_hisparse_host_pools(
self.vllm_config,
tensor_sizes,
config.hisparse_host_num_blocks,
config.hisparse_host_block_stride,
use_shared_host_pool=True,
)
assert self.shared_region is not None
self.registered = self.shared_region.base_tensor
self.backing = self.registered[:size]
else:
check_hisparse_host_memory(size)
self.backing, self.registered = allocate_pinned_host_pool(size)
return self.backing
def use_shared_hisparse_host_pool(vllm_config: VllmConfig) -> bool:
"""Whether replicated MLA host KV can share one local mmap."""
parallel = vllm_config.parallel_config
return (
current_platform.is_cuda_alike()
and parallel.tensor_parallel_size > 1
and parallel.pipeline_parallel_size == 1
and parallel.prefill_context_parallel_size == 1
and parallel.decode_context_parallel_size == 1
and parallel.world_size == parallel.tensor_parallel_size
and parallel.distributed_executor_backend == "mp"
and parallel.nnodes_within_dp == 1
)
def get_hisparse_host_block_stride(
page_size_bytes: int, *, use_shared_host_pool: bool
) -> int:
if use_shared_host_pool:
return round_up(page_size_bytes, SharedOffloadRegion.BLOCK_SIZE_ALIGNMENT)
return page_size_bytes
def _hisparse_registration_ranges(
tensor_sizes: list[int],
num_blocks: int,
host_block_stride: int,
max_chunk_bytes: int = HOST_REGISTER_CHUNK_BYTES,
) -> tuple[tuple[int, int], ...]:
"""Split a shared pool without bisecting any tensor's host block.
CUDA batch copies reject descriptors spanning adjacent registrations.
"""
total_size = num_blocks * host_block_stride
if total_size % mmap.PAGESIZE:
raise ValueError("HiSparse host pool must be OS-page aligned.")
if max_chunk_bytes < mmap.PAGESIZE:
raise ValueError("HiSparse registration chunks must span at least one page.")
tensors: list[tuple[int, int, int]] = []
tensor_start = 0
for tensor_size in tensor_sizes:
if tensor_size % num_blocks:
raise ValueError(
f"HiSparse host tensor size {tensor_size} is not divisible by "
f"{num_blocks} blocks."
)
tensor_end = tensor_start + tensor_size
tensors.append((tensor_start, tensor_end, tensor_size // num_blocks))
tensor_start = tensor_end
if tensor_start > total_size:
raise ValueError("HiSparse host tensors exceed the shared host pool.")
def last_safe_boundary(lower: int, upper: int) -> int | None:
best = None
for start, end, block_bytes in tensors:
first_block = max(0, (lower - start) // block_bytes + 1)
last_block = min(num_blocks, (min(upper, end) - start) // block_bytes)
if first_block > last_block:
continue
divisor = math.gcd(block_bytes, mmap.PAGESIZE)
if start % divisor:
continue
period = mmap.PAGESIZE // divisor
if period == 1:
residue = 0
else:
# Find block boundaries that are also OS-page boundaries.
residue = (
(-start // divisor) * pow(block_bytes // divisor, -1, period)
) % period
block = last_block - (last_block - residue) % period
if block >= first_block:
candidate = start + block * block_bytes
best = candidate if best is None else max(best, candidate)
if upper >= tensor_start:
candidate = upper // mmap.PAGESIZE * mmap.PAGESIZE
if candidate >= tensor_start and candidate > lower:
best = candidate if best is None else max(best, candidate)
return best
ranges: list[tuple[int, int]] = []
start = 0
while total_size - start > max_chunk_bytes:
end = last_safe_boundary(start, start + max_chunk_bytes)
if end is None:
raise ValueError(
"A HiSparse host block is too large for the registration chunk limit."
)
ranges.append((start, end))
start = end
ranges.append((start, total_size))
return tuple(ranges)
def allocate_hisparse_host_pools(
vllm_config: VllmConfig,
tensor_sizes: list[int],
num_blocks: int,
host_block_stride: int,
*,
use_shared_host_pool: bool,
) -> tuple[list[torch.Tensor], list[torch.Tensor], SharedOffloadRegion | None]:
"""Allocate private host tensors or one mmap shared by local TP ranks."""
if not use_shared_host_pool:
check_hisparse_host_memory(sum(tensor_sizes))
private_pools = [allocate_pinned_host_pool(size) for size in tensor_sizes]
return (
[pool for pool, _ in private_pools],
[registered for _, registered in private_pools],
None,
)
region = SharedOffloadRegion(
engine_id=(
f"hisparse_{vllm_config.instance_id}_"
f"dp{vllm_config.parallel_config.data_parallel_index}"
),
num_chunks=1,
rank=0,
kv_bytes_per_chunk=num_blocks * host_block_stride,
cpu_page_size=sum(tensor_sizes),
barrier=get_tp_group().barrier,
creator_memory_check=check_hisparse_host_memory,
populate_only_on_creator=True,
)
try:
for start, end in _hisparse_registration_ranges(
tensor_sizes, num_blocks, host_block_stride
):
tensor = region.base_tensor[start:end]
pin_tensor(tensor)
region.pinned_addresses.append(tensor.data_ptr())
region.is_pinned = True
pools = [
region.create_next_canonical_view(size).view(-1) for size in tensor_sizes
]
except Exception:
region.cleanup()
raise
return pools, [], region
def release_pinned_state(
runtimes: list[HiSparseRuntime], pinned_host_pools: list[torch.Tensor]
runtimes: list[HiSparseRuntime],
pinned_host_pools: list[torch.Tensor],
shared_host_region: SharedOffloadRegion | None = None,
) -> None:
"""Synchronize and release registered host KV pools."""
if pinned_host_pools:
if pinned_host_pools or shared_host_region is not None:
try:
torch.accelerator.synchronize()
except RuntimeError as e:
@@ -208,7 +388,7 @@ def release_pinned_state(
"HiSparse: CUDA context unusable at teardown (%s); leaving "
"%d host-pool tensors pinned for kernel exit reclaim.",
e,
len(pinned_host_pools),
len(pinned_host_pools) + int(shared_host_region is not None),
)
return
@@ -240,6 +420,8 @@ def release_pinned_state(
del runtime._host_cache
del runtime.registered_host_pool
del runtime.hot_backing
if shared_host_region is not None:
shared_host_region.cleanup()
def hisparse_prefill_staging_remap(
@@ -536,6 +718,7 @@ class HiSparseRuntime:
self.eager_host_mirror = config.eager_host_mirror
self.resident_source_index = -1
self.request_state_indices: torch.Tensor | None = None
self.shared_host_region: SharedOffloadRegion | None = None
@property
def host_cache(self) -> torch.Tensor:
+6
View File
@@ -1435,6 +1435,12 @@ class KVCacheConfig:
hisparse_host_num_blocks: int | None = None
"""Capacity of the dedicated HiSparse host-block manager, when enabled."""
hisparse_host_block_stride: int | None = None
"""Physical bytes between consecutive HiSparse host blocks."""
hisparse_shared_host_pool: bool = False
"""Whether local TP ranks share one physical HiSparse host pool."""
@cached_property
def transfer_group_ids(self) -> tuple[int, ...]:
"""IDs of cache groups that participate in external KV transfer."""
+60 -25
View File
@@ -1,5 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import contextlib
import errno
import mmap
import os
@@ -27,6 +28,10 @@ def _wait_for_file_size(fd: int, expected_size: int, timeout: float = 30.0) -> N
while True:
if os.fstat(fd).st_size >= expected_size:
return
if os.fstat(fd).st_nlink == 0:
raise RuntimeError(
"Shared offload region creator failed during initialization."
)
if time.monotonic() > deadline:
raise TimeoutError(
f"Timed out waiting for mmap file to reach {expected_size} bytes"
@@ -75,6 +80,9 @@ class SharedOffloadRegion:
given, the path is unlinked once every worker has mapped the file, so
the kernel reclaims the memory when the last worker exits, no matter
how it exits; mappings taken before the unlink stay valid.
Creator-only population pre-faults the entire region before the barrier
and requires that barrier to keep joiners from using unpopulated pages.
"""
BLOCK_SIZE_ALIGNMENT: int = mmap.PAGESIZE
@@ -87,7 +95,12 @@ class SharedOffloadRegion:
kv_bytes_per_chunk: int,
cpu_page_size: int,
barrier: Callable[[], None] | None = None,
*,
creator_memory_check: Callable[[int], None] | None = None,
populate_only_on_creator: bool = False,
) -> None:
if populate_only_on_creator and barrier is None:
raise ValueError("Creator-only population requires a barrier.")
self.page_size = mmap.PAGESIZE
assert kv_bytes_per_chunk % self.page_size == 0
@@ -112,25 +125,18 @@ class SharedOffloadRegion:
# Joiner path — another worker won O_EXCL. Reopen and wait
# for the file to reach expected size.
self.fd = os.open(self.mmap_path, os.O_RDWR)
try:
_wait_for_file_size(self.fd, self.total_size_bytes)
except (TimeoutError, OSError):
os.close(self.fd)
raise
_wait_for_file_size(self.fd, self.total_size_bytes)
logger.info("Opened existing mmap file %s", self.mmap_path)
else:
# Creator path. We won O_EXCL, so we own the file: any
# failure here must clean up so concurrent joiners don't
# land on a 0-byte stub and spin in _wait_for_file_size
# for the full 30 s timeout.
try:
check_shm_free_space(self.total_size_bytes)
os.ftruncate(self.fd, self.total_size_bytes)
except (RuntimeError, OSError):
os.unlink(self.mmap_path)
os.close(self.fd)
raise
self._creator = True
if creator_memory_check is not None:
creator_memory_check(self.total_size_bytes)
check_shm_free_space(self.total_size_bytes)
os.ftruncate(self.fd, self.total_size_bytes)
logger.info(
"Created mmap file %s (%.2f GB)",
self.mmap_path,
@@ -143,10 +149,21 @@ class SharedOffloadRegion:
flags=mmap.MAP_SHARED,
prot=mmap.PROT_READ | mmap.PROT_WRITE,
)
if populate_only_on_creator and self._creator:
populate_write_fn = _get_populate_write_fn(self.mmap_obj)
populate_write_fn(self.mmap_obj, 0, self.total_size_bytes)
except Exception:
if self._creator:
os.unlink(self.mmap_path)
with contextlib.suppress(FileNotFoundError):
os.unlink(self.mmap_path)
self._creator = False
if hasattr(self, "mmap_obj") and self.mmap_obj is not None:
self.mmap_obj.close()
self.mmap_obj = None
if hasattr(self, "fd") and self.fd is not None:
os.close(self.fd)
self.fd = None
# Peers block inside the barrier until the collective times out if
# we die before reaching it. Arrive anyway so every worker calls
# barrier() exactly once and they fail on their own errors instead
@@ -169,16 +186,28 @@ class SharedOffloadRegion:
barrier()
except Exception:
if self._creator:
os.unlink(self.mmap_path)
with contextlib.suppress(FileNotFoundError):
os.unlink(self.mmap_path)
self._creator = False
self.mmap_obj.close()
os.close(self.fd)
self.mmap_obj = None
self.fd = None
raise
if self._creator:
os.unlink(self.mmap_path)
self._creator = False
logger.info("Unlinked mmap file %s", self.mmap_path)
self._base = torch.frombuffer(memoryview(self.mmap_obj), dtype=torch.int8)
self._views: list[torch.Tensor] = []
self._canonical_offset = 0
self.is_pinned: bool = False
self.pinned_addresses: list[int] = []
if populate_only_on_creator:
return
populate_write_fn = _get_populate_write_fn(self.mmap_obj)
if rank is not None:
@@ -205,10 +234,11 @@ class SharedOffloadRegion:
"MADV_POPULATE_WRITE entire region: %.3f s", time.perf_counter() - _t0
)
self._base = torch.frombuffer(memoryview(self.mmap_obj), dtype=torch.int8)
self._views: list[torch.Tensor] = []
self._canonical_offset = 0
self.is_pinned: bool = False
@property
def base_tensor(self) -> torch.Tensor:
if self._base is None:
raise RuntimeError("Shared offload region has been released.")
return self._base
def create_next_worker_view(self, tensor_page_size: int) -> torch.Tensor:
"""Allocate a strided int8 view for this worker, one canonical tensor.
@@ -311,13 +341,18 @@ class SharedOffloadRegion:
if self.is_pinned and self._base is not None:
if current_platform.is_cuda_alike():
base_ptr = self._base.data_ptr()
result = torch.cuda.cudart().cudaHostUnregister(base_ptr)
if result.value != 0:
logger.warning(
"cudaHostUnregister failed for rank=%d (code=%d)",
self.rank,
result,
)
addresses = self.pinned_addresses or [base_ptr]
for address in reversed(addresses):
result = torch.cuda.cudart().cudaHostUnregister(address)
if result.value != 0:
logger.warning(
"cudaHostUnregister failed for rank=%d, "
"address=%#x (code=%d)",
self.rank,
address,
result.value,
)
self.pinned_addresses.clear()
self.is_pinned = False
# Release views before _base: each view holds a _base reference and a
# direct StorageImpl reference. Freeing views first lets both refcounts