Signed-off-by: Matthew Bonanni <mbonanni@redhat.com> Signed-off-by: Chao Lei <leichao139636@163.com> Signed-off-by: Lucas Wilkinson <lwilkinson@neuralmagic.com> Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com> Co-authored-by: Chao Lei <leichao139636@163.com> Co-authored-by: Claude Fable 5 <noreply@anthropic.com> Co-authored-by: Matthew Bonanni <mbonanni@redhat.com> Co-authored-by: OpenAI Codex <codex@openai.com> Co-authored-by: OpenAI Codex <noreply@openai.com>
This commit is contained in:
co-authored by
Chao Lei
Claude Fable 5
Matthew Bonanni
OpenAI Codex
OpenAI Codex
parent
ebe1dec2da
commit
e19a3e172e
+189
-6
@@ -4,6 +4,8 @@
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import math
|
||||
import mmap
|
||||
import time
|
||||
from collections.abc import Callable
|
||||
from dataclasses import dataclass
|
||||
@@ -13,17 +15,22 @@ import psutil
|
||||
import torch
|
||||
|
||||
from vllm.config import CUDAGraphMode, VllmConfig
|
||||
from vllm.distributed import get_tp_group
|
||||
from vllm.forward_context import get_forward_context
|
||||
from vllm.logger import init_logger
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.math_utils import round_up
|
||||
from vllm.utils.torch_utils import current_stream
|
||||
from vllm.v1.kv_offload.cpu.shared_offload_region import SharedOffloadRegion
|
||||
from vllm.v1.simple_kv_offload.cuda_mem_ops import pin_tensor
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
HOST_REGISTER_CHUNK_BYTES = 256 * 2**30
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.v1.attention.backends.mla.index_group import HiSparseMLAIndexGroup
|
||||
from vllm.v1.kv_cache_interface import KVCacheConfig
|
||||
|
||||
# fp8_ds_mla KV row: 512 B quantized NoPE + 16 B scales + 128 B RoPE.
|
||||
FP8_DS_MLA_ROW_BYTES = 656
|
||||
@@ -185,22 +192,195 @@ def allocate_pinned_host_pool(size: int) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
class HiSparseHostPool:
|
||||
"""Keep the single host backing and its exact CUDA registration range."""
|
||||
|
||||
def __init__(self) -> None:
|
||||
def __init__(self, vllm_config: VllmConfig, kv_cache_config: KVCacheConfig) -> None:
|
||||
self.vllm_config = vllm_config
|
||||
self.kv_cache_config = kv_cache_config
|
||||
self.backing: torch.Tensor | None = None
|
||||
self.registered: torch.Tensor | None = None
|
||||
self.shared_region: SharedOffloadRegion | None = None
|
||||
|
||||
def allocate(self, size: int) -> torch.Tensor:
|
||||
assert self.backing is None, "HiSparse host pool is already allocated."
|
||||
check_hisparse_host_memory(size)
|
||||
self.backing, self.registered = allocate_pinned_host_pool(size)
|
||||
config = self.kv_cache_config
|
||||
if config.hisparse_shared_host_pool:
|
||||
tensor_sizes = [
|
||||
size
|
||||
for _, size in sorted(
|
||||
(tensor.offset + i * tensor.layer_stride, tensor.layer_stride)
|
||||
for tensor in config.kv_cache_tensors
|
||||
if tensor.host_resident
|
||||
for i in range(len(tensor.layers))
|
||||
)
|
||||
]
|
||||
assert config.hisparse_host_num_blocks is not None
|
||||
assert config.hisparse_host_block_stride is not None
|
||||
_, _, self.shared_region = allocate_hisparse_host_pools(
|
||||
self.vllm_config,
|
||||
tensor_sizes,
|
||||
config.hisparse_host_num_blocks,
|
||||
config.hisparse_host_block_stride,
|
||||
use_shared_host_pool=True,
|
||||
)
|
||||
assert self.shared_region is not None
|
||||
self.registered = self.shared_region.base_tensor
|
||||
self.backing = self.registered[:size]
|
||||
else:
|
||||
check_hisparse_host_memory(size)
|
||||
self.backing, self.registered = allocate_pinned_host_pool(size)
|
||||
return self.backing
|
||||
|
||||
|
||||
def use_shared_hisparse_host_pool(vllm_config: VllmConfig) -> bool:
|
||||
"""Whether replicated MLA host KV can share one local mmap."""
|
||||
parallel = vllm_config.parallel_config
|
||||
return (
|
||||
current_platform.is_cuda_alike()
|
||||
and parallel.tensor_parallel_size > 1
|
||||
and parallel.pipeline_parallel_size == 1
|
||||
and parallel.prefill_context_parallel_size == 1
|
||||
and parallel.decode_context_parallel_size == 1
|
||||
and parallel.world_size == parallel.tensor_parallel_size
|
||||
and parallel.distributed_executor_backend == "mp"
|
||||
and parallel.nnodes_within_dp == 1
|
||||
)
|
||||
|
||||
|
||||
def get_hisparse_host_block_stride(
|
||||
page_size_bytes: int, *, use_shared_host_pool: bool
|
||||
) -> int:
|
||||
if use_shared_host_pool:
|
||||
return round_up(page_size_bytes, SharedOffloadRegion.BLOCK_SIZE_ALIGNMENT)
|
||||
return page_size_bytes
|
||||
|
||||
|
||||
def _hisparse_registration_ranges(
|
||||
tensor_sizes: list[int],
|
||||
num_blocks: int,
|
||||
host_block_stride: int,
|
||||
max_chunk_bytes: int = HOST_REGISTER_CHUNK_BYTES,
|
||||
) -> tuple[tuple[int, int], ...]:
|
||||
"""Split a shared pool without bisecting any tensor's host block.
|
||||
|
||||
CUDA batch copies reject descriptors spanning adjacent registrations.
|
||||
"""
|
||||
total_size = num_blocks * host_block_stride
|
||||
if total_size % mmap.PAGESIZE:
|
||||
raise ValueError("HiSparse host pool must be OS-page aligned.")
|
||||
if max_chunk_bytes < mmap.PAGESIZE:
|
||||
raise ValueError("HiSparse registration chunks must span at least one page.")
|
||||
|
||||
tensors: list[tuple[int, int, int]] = []
|
||||
tensor_start = 0
|
||||
for tensor_size in tensor_sizes:
|
||||
if tensor_size % num_blocks:
|
||||
raise ValueError(
|
||||
f"HiSparse host tensor size {tensor_size} is not divisible by "
|
||||
f"{num_blocks} blocks."
|
||||
)
|
||||
tensor_end = tensor_start + tensor_size
|
||||
tensors.append((tensor_start, tensor_end, tensor_size // num_blocks))
|
||||
tensor_start = tensor_end
|
||||
if tensor_start > total_size:
|
||||
raise ValueError("HiSparse host tensors exceed the shared host pool.")
|
||||
|
||||
def last_safe_boundary(lower: int, upper: int) -> int | None:
|
||||
best = None
|
||||
for start, end, block_bytes in tensors:
|
||||
first_block = max(0, (lower - start) // block_bytes + 1)
|
||||
last_block = min(num_blocks, (min(upper, end) - start) // block_bytes)
|
||||
if first_block > last_block:
|
||||
continue
|
||||
divisor = math.gcd(block_bytes, mmap.PAGESIZE)
|
||||
if start % divisor:
|
||||
continue
|
||||
period = mmap.PAGESIZE // divisor
|
||||
if period == 1:
|
||||
residue = 0
|
||||
else:
|
||||
# Find block boundaries that are also OS-page boundaries.
|
||||
residue = (
|
||||
(-start // divisor) * pow(block_bytes // divisor, -1, period)
|
||||
) % period
|
||||
block = last_block - (last_block - residue) % period
|
||||
if block >= first_block:
|
||||
candidate = start + block * block_bytes
|
||||
best = candidate if best is None else max(best, candidate)
|
||||
|
||||
if upper >= tensor_start:
|
||||
candidate = upper // mmap.PAGESIZE * mmap.PAGESIZE
|
||||
if candidate >= tensor_start and candidate > lower:
|
||||
best = candidate if best is None else max(best, candidate)
|
||||
return best
|
||||
|
||||
ranges: list[tuple[int, int]] = []
|
||||
start = 0
|
||||
while total_size - start > max_chunk_bytes:
|
||||
end = last_safe_boundary(start, start + max_chunk_bytes)
|
||||
if end is None:
|
||||
raise ValueError(
|
||||
"A HiSparse host block is too large for the registration chunk limit."
|
||||
)
|
||||
ranges.append((start, end))
|
||||
start = end
|
||||
ranges.append((start, total_size))
|
||||
return tuple(ranges)
|
||||
|
||||
|
||||
def allocate_hisparse_host_pools(
|
||||
vllm_config: VllmConfig,
|
||||
tensor_sizes: list[int],
|
||||
num_blocks: int,
|
||||
host_block_stride: int,
|
||||
*,
|
||||
use_shared_host_pool: bool,
|
||||
) -> tuple[list[torch.Tensor], list[torch.Tensor], SharedOffloadRegion | None]:
|
||||
"""Allocate private host tensors or one mmap shared by local TP ranks."""
|
||||
if not use_shared_host_pool:
|
||||
check_hisparse_host_memory(sum(tensor_sizes))
|
||||
private_pools = [allocate_pinned_host_pool(size) for size in tensor_sizes]
|
||||
return (
|
||||
[pool for pool, _ in private_pools],
|
||||
[registered for _, registered in private_pools],
|
||||
None,
|
||||
)
|
||||
|
||||
region = SharedOffloadRegion(
|
||||
engine_id=(
|
||||
f"hisparse_{vllm_config.instance_id}_"
|
||||
f"dp{vllm_config.parallel_config.data_parallel_index}"
|
||||
),
|
||||
num_chunks=1,
|
||||
rank=0,
|
||||
kv_bytes_per_chunk=num_blocks * host_block_stride,
|
||||
cpu_page_size=sum(tensor_sizes),
|
||||
barrier=get_tp_group().barrier,
|
||||
creator_memory_check=check_hisparse_host_memory,
|
||||
populate_only_on_creator=True,
|
||||
)
|
||||
try:
|
||||
for start, end in _hisparse_registration_ranges(
|
||||
tensor_sizes, num_blocks, host_block_stride
|
||||
):
|
||||
tensor = region.base_tensor[start:end]
|
||||
pin_tensor(tensor)
|
||||
region.pinned_addresses.append(tensor.data_ptr())
|
||||
region.is_pinned = True
|
||||
pools = [
|
||||
region.create_next_canonical_view(size).view(-1) for size in tensor_sizes
|
||||
]
|
||||
except Exception:
|
||||
region.cleanup()
|
||||
raise
|
||||
return pools, [], region
|
||||
|
||||
|
||||
def release_pinned_state(
|
||||
runtimes: list[HiSparseRuntime], pinned_host_pools: list[torch.Tensor]
|
||||
runtimes: list[HiSparseRuntime],
|
||||
pinned_host_pools: list[torch.Tensor],
|
||||
shared_host_region: SharedOffloadRegion | None = None,
|
||||
) -> None:
|
||||
"""Synchronize and release registered host KV pools."""
|
||||
if pinned_host_pools:
|
||||
if pinned_host_pools or shared_host_region is not None:
|
||||
try:
|
||||
torch.accelerator.synchronize()
|
||||
except RuntimeError as e:
|
||||
@@ -208,7 +388,7 @@ def release_pinned_state(
|
||||
"HiSparse: CUDA context unusable at teardown (%s); leaving "
|
||||
"%d host-pool tensors pinned for kernel exit reclaim.",
|
||||
e,
|
||||
len(pinned_host_pools),
|
||||
len(pinned_host_pools) + int(shared_host_region is not None),
|
||||
)
|
||||
return
|
||||
|
||||
@@ -240,6 +420,8 @@ def release_pinned_state(
|
||||
del runtime._host_cache
|
||||
del runtime.registered_host_pool
|
||||
del runtime.hot_backing
|
||||
if shared_host_region is not None:
|
||||
shared_host_region.cleanup()
|
||||
|
||||
|
||||
def hisparse_prefill_staging_remap(
|
||||
@@ -536,6 +718,7 @@ class HiSparseRuntime:
|
||||
self.eager_host_mirror = config.eager_host_mirror
|
||||
self.resident_source_index = -1
|
||||
self.request_state_indices: torch.Tensor | None = None
|
||||
self.shared_host_region: SharedOffloadRegion | None = None
|
||||
|
||||
@property
|
||||
def host_cache(self) -> torch.Tensor:
|
||||
|
||||
Reference in New Issue
Block a user