[KV Offload] Define clean backend configuration boundary (#48150)
Signed-off-by: Change72 <changg@nvidia.com> Signed-off-by: Chang Guo <cguo51@asu.edu> Co-authored-by: Codex <codex@openai.com> Co-authored-by: Claude Fable 5 <noreply@anthropic.com> Co-authored-by: Or Ozeri <or@ozery.com> Co-authored-by: OpenAI Codex <noreply@openai.com>
This commit is contained in:
co-authored by
Codex
Claude Fable 5
Or Ozeri
OpenAI Codex
parent
8c3393f373
commit
a9531edfa6
@@ -72,7 +72,7 @@ class Transfer:
|
||||
|
||||
def compute_sub_block_ptrs(
|
||||
block_ids: np.ndarray,
|
||||
block_size_factor: int,
|
||||
blocks_per_chunk: int,
|
||||
output: np.ndarray,
|
||||
tensor: torch.Tensor,
|
||||
skip_count: int = 0,
|
||||
@@ -80,38 +80,38 @@ def compute_sub_block_ptrs(
|
||||
"""
|
||||
Compute byte pointers for sub-blocks of the given block IDs.
|
||||
|
||||
Each block in block_ids contains block_size_factor sub-blocks.
|
||||
Each block in block_ids contains blocks_per_chunk sub-blocks.
|
||||
The pointer for sub-block j of block b is:
|
||||
base_ptr + b * row_stride + j * sub_block_size
|
||||
base_ptr + b * row_stride + j * block_page_size
|
||||
|
||||
where sub_block_size = tensor.shape[1] // block_size_factor (gpu page size).
|
||||
where block_page_size = tensor.shape[1] // blocks_per_chunk (gpu page size).
|
||||
|
||||
This handles tensors where row_stride != block_size_factor * sub_block_size
|
||||
This handles tensors where row_stride != blocks_per_chunk * block_page_size
|
||||
(e.g. non-contiguous CPU tensors).
|
||||
|
||||
Args:
|
||||
block_ids: array of block IDs at the tensor's native granularity.
|
||||
block_size_factor: number of sub-blocks per block.
|
||||
blocks_per_chunk: number of sub-blocks per block.
|
||||
output: pre-allocated pointer array to write pointers into.
|
||||
tensor: the source or destination tensor.
|
||||
skip_count: sub-blocks to skip in the first block.
|
||||
"""
|
||||
assert skip_count < block_size_factor
|
||||
assert skip_count < blocks_per_chunk
|
||||
|
||||
num_sub_blocks = len(output)
|
||||
base_ptr = tensor.data_ptr()
|
||||
row_stride = tensor.stride(0)
|
||||
|
||||
if block_size_factor == 1:
|
||||
if blocks_per_chunk == 1:
|
||||
# Fast path: 1:1 mapping, no sub-block expansion needed.
|
||||
output[:] = base_ptr + block_ids.astype(np.uint64)[:num_sub_blocks] * row_stride
|
||||
return
|
||||
|
||||
# Vectorized expansion for block_size_factor > 1.
|
||||
assert tensor.shape[1] % block_size_factor == 0
|
||||
sub_block_size = tensor.shape[1] // block_size_factor
|
||||
sub_offsets = np.arange(block_size_factor, dtype=np.uint64) * sub_block_size
|
||||
# (num_blocks, 1) + (1, block_size_factor) -> (num_blocks, block_size_factor)
|
||||
# Vectorized expansion for blocks_per_chunk > 1.
|
||||
assert tensor.shape[1] % blocks_per_chunk == 0
|
||||
block_page_size = tensor.shape[1] // blocks_per_chunk
|
||||
sub_offsets = np.arange(blocks_per_chunk, dtype=np.uint64) * block_page_size
|
||||
# (num_blocks, 1) + (1, blocks_per_chunk) -> (num_blocks, blocks_per_chunk)
|
||||
all_ptrs = (
|
||||
base_ptr + block_ids.astype(np.uint64)[:, np.newaxis] * row_stride
|
||||
) + sub_offsets[np.newaxis, :]
|
||||
@@ -175,7 +175,7 @@ class SingleDirectionOffloadingHandler:
|
||||
self,
|
||||
gpu_tensors: list[torch.Tensor],
|
||||
cpu_tensors: list[torch.Tensor],
|
||||
block_size_factor: int,
|
||||
blocks_per_chunk: int,
|
||||
kv_cache_groups_data_refs: list[list[CanonicalKVCacheRef]],
|
||||
gpu_to_cpu: bool,
|
||||
mmap_region: SharedOffloadRegion | None = None,
|
||||
@@ -205,7 +205,7 @@ class SingleDirectionOffloadingHandler:
|
||||
assert cpu_tensor.device.type == "cpu"
|
||||
_, gpu_page_size = gpu_tensor.shape
|
||||
_, cpu_page_size = cpu_tensor.shape
|
||||
assert cpu_page_size == gpu_page_size * block_size_factor
|
||||
assert cpu_page_size == gpu_page_size * blocks_per_chunk
|
||||
|
||||
self.src_tensors: list[torch.Tensor] = (
|
||||
gpu_tensors if gpu_to_cpu else cpu_tensors
|
||||
@@ -220,9 +220,9 @@ class SingleDirectionOffloadingHandler:
|
||||
)
|
||||
|
||||
# GPU blocks may be smaller
|
||||
# cpu_page_size = gpu_page_size * block_size_factor.
|
||||
self.src_block_size_factor = 1 if self.gpu_to_cpu else block_size_factor
|
||||
self.dst_block_size_factor = block_size_factor if self.gpu_to_cpu else 1
|
||||
# cpu_page_size = gpu_page_size * blocks_per_chunk.
|
||||
self.src_blocks_per_chunk = 1 if self.gpu_to_cpu else blocks_per_chunk
|
||||
self.dst_blocks_per_chunk = blocks_per_chunk if self.gpu_to_cpu else 1
|
||||
|
||||
# mmap_region to clean up on shutdown (gpu_to_cpu handler owns it)
|
||||
self._mmap_region = mmap_region
|
||||
@@ -313,20 +313,16 @@ class SingleDirectionOffloadingHandler:
|
||||
if group_size == 0:
|
||||
continue
|
||||
|
||||
src_logical_blocks_to_skip = block_idx % self.src_block_size_factor
|
||||
dst_logical_blocks_to_skip = block_idx % self.dst_block_size_factor
|
||||
src_logical_blocks_to_skip = block_idx % self.src_blocks_per_chunk
|
||||
dst_logical_blocks_to_skip = block_idx % self.dst_blocks_per_chunk
|
||||
src_logical_blocks_count = group_size + src_logical_blocks_to_skip
|
||||
dst_logical_blocks_count = group_size + dst_logical_blocks_to_skip
|
||||
|
||||
dst_blocks_count = cdiv(
|
||||
dst_logical_blocks_count, self.dst_block_size_factor
|
||||
)
|
||||
dst_blocks_count = cdiv(dst_logical_blocks_count, self.dst_blocks_per_chunk)
|
||||
dst_end_offset = dst_offset + dst_blocks_count
|
||||
assert dst_end_offset <= num_dst_blocks
|
||||
|
||||
src_blocks_count = cdiv(
|
||||
src_logical_blocks_count, self.src_block_size_factor
|
||||
)
|
||||
src_blocks_count = cdiv(src_logical_blocks_count, self.src_blocks_per_chunk)
|
||||
src_end_offset = src_offset + src_blocks_count
|
||||
assert src_end_offset <= num_src_blocks
|
||||
|
||||
@@ -339,14 +335,14 @@ class SingleDirectionOffloadingHandler:
|
||||
|
||||
compute_sub_block_ptrs(
|
||||
group_src,
|
||||
self.src_block_size_factor,
|
||||
self.src_blocks_per_chunk,
|
||||
all_src[op_idx:end_idx],
|
||||
self.src_tensors[t_idx],
|
||||
skip_count=src_logical_blocks_to_skip,
|
||||
)
|
||||
compute_sub_block_ptrs(
|
||||
group_dst,
|
||||
self.dst_block_size_factor,
|
||||
self.dst_blocks_per_chunk,
|
||||
all_dst[op_idx:end_idx],
|
||||
self.dst_tensors[t_idx],
|
||||
skip_count=dst_logical_blocks_to_skip,
|
||||
@@ -476,7 +472,7 @@ class CPUOffloadingWorker(OffloadingWorker):
|
||||
def __init__(
|
||||
self,
|
||||
kv_caches: CanonicalKVCaches,
|
||||
block_size_factor: int,
|
||||
blocks_per_chunk: int,
|
||||
num_cpu_blocks: int,
|
||||
mmap_region: SharedOffloadRegion | None = None,
|
||||
):
|
||||
@@ -492,7 +488,7 @@ class CPUOffloadingWorker(OffloadingWorker):
|
||||
gpu_tensor = kv_cache_tensor.tensor.view(torch.int8).view(
|
||||
(-1, gpu_page_size_bytes)
|
||||
)
|
||||
cpu_page_size_bytes = gpu_page_size_bytes * block_size_factor
|
||||
cpu_page_size_bytes = gpu_page_size_bytes * blocks_per_chunk
|
||||
|
||||
if mmap_region is not None:
|
||||
cpu_tensor = mmap_region.create_next_view(cpu_page_size_bytes)
|
||||
@@ -518,7 +514,7 @@ class CPUOffloadingWorker(OffloadingWorker):
|
||||
self._store_handler = SingleDirectionOffloadingHandler(
|
||||
gpu_tensors=gpu_tensors,
|
||||
cpu_tensors=cpu_tensors,
|
||||
block_size_factor=block_size_factor,
|
||||
blocks_per_chunk=blocks_per_chunk,
|
||||
kv_cache_groups_data_refs=kv_caches.group_data_refs,
|
||||
gpu_to_cpu=True,
|
||||
mmap_region=mmap_region,
|
||||
@@ -527,7 +523,7 @@ class CPUOffloadingWorker(OffloadingWorker):
|
||||
self._load_handler = SingleDirectionOffloadingHandler(
|
||||
gpu_tensors=gpu_tensors,
|
||||
cpu_tensors=cpu_tensors,
|
||||
block_size_factor=block_size_factor,
|
||||
blocks_per_chunk=blocks_per_chunk,
|
||||
kv_cache_groups_data_refs=kv_caches.group_data_refs,
|
||||
gpu_to_cpu=False,
|
||||
)
|
||||
|
||||
@@ -4,10 +4,8 @@ from typing import Any
|
||||
|
||||
from typing_extensions import override
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.math_utils import round_up
|
||||
from vllm.v1.kv_cache_interface import KVCacheConfig
|
||||
from vllm.v1.kv_offload.base import (
|
||||
CanonicalKVCaches,
|
||||
OffloadingCounterMetadata,
|
||||
@@ -18,6 +16,7 @@ from vllm.v1.kv_offload.base import (
|
||||
OffloadingSpec,
|
||||
OffloadingWorker,
|
||||
)
|
||||
from vllm.v1.kv_offload.config import OffloadingConfig
|
||||
from vllm.v1.kv_offload.cpu.common import CPUOffloadingMetrics
|
||||
from vllm.v1.kv_offload.cpu.gpu_worker import CPUOffloadingWorker
|
||||
from vllm.v1.kv_offload.cpu.manager import CPUOffloadingManager
|
||||
@@ -73,8 +72,8 @@ class CPUOffloadingSpec(OffloadingSpec):
|
||||
)
|
||||
return definitions
|
||||
|
||||
def __init__(self, vllm_config: VllmConfig, kv_cache_config: KVCacheConfig):
|
||||
super().__init__(vllm_config, kv_cache_config)
|
||||
def __init__(self, config: OffloadingConfig):
|
||||
super().__init__(config)
|
||||
|
||||
cpu_bytes_to_use = self.extra_config.get("cpu_bytes_to_use")
|
||||
if not cpu_bytes_to_use:
|
||||
@@ -82,42 +81,28 @@ class CPUOffloadingSpec(OffloadingSpec):
|
||||
"cpu_bytes_to_use must be specified in kv_connector_extra_config"
|
||||
)
|
||||
|
||||
world_size = vllm_config.parallel_config.world_size
|
||||
world_size = config.parallel.world_size
|
||||
self.num_blocks = 0
|
||||
self.kv_bytes_per_offloaded_block = 0
|
||||
self.kv_bytes_per_chunk = 0
|
||||
self.cpu_page_size_per_worker = 0
|
||||
assert kv_cache_config is not None
|
||||
if kv_cache_config.num_blocks > 0 and world_size > 0:
|
||||
is_packed = any(t.block_stride for t in kv_cache_config.kv_cache_tensors)
|
||||
assert not is_packed or all(
|
||||
t.block_stride for t in kv_cache_config.kv_cache_tensors
|
||||
)
|
||||
total_gpu_kv_bytes = (
|
||||
kv_cache_config.kv_cache_tensors[0].size
|
||||
if is_packed
|
||||
else sum(t.size for t in kv_cache_config.kv_cache_tensors)
|
||||
)
|
||||
kv_bytes_per_block = (
|
||||
total_gpu_kv_bytes // kv_cache_config.num_blocks
|
||||
) * world_size
|
||||
kv_bytes_per_offloaded_block = kv_bytes_per_block * self.block_size_factor
|
||||
if config.worker_kv_bytes_per_block > 0 and world_size > 0:
|
||||
kv_bytes_per_block = config.worker_kv_bytes_per_block * world_size
|
||||
kv_bytes_per_chunk = kv_bytes_per_block * self.blocks_per_chunk
|
||||
|
||||
# calculate cpu_page_size_per_worker
|
||||
self.cpu_page_size_per_worker = kv_bytes_per_offloaded_block // world_size
|
||||
self.cpu_page_size_per_worker = kv_bytes_per_chunk // world_size
|
||||
|
||||
# calculate num_blocks
|
||||
aligned_kv_bytes_per_offloaded_block = round_up(
|
||||
kv_bytes_per_offloaded_block, self.BLOCK_SIZE_ALIGNMENT
|
||||
)
|
||||
self.num_blocks = (
|
||||
int(cpu_bytes_to_use) // aligned_kv_bytes_per_offloaded_block
|
||||
aligned_kv_bytes_per_chunk = round_up(
|
||||
kv_bytes_per_chunk, self.BLOCK_SIZE_ALIGNMENT
|
||||
)
|
||||
self.num_blocks = int(cpu_bytes_to_use) // aligned_kv_bytes_per_chunk
|
||||
|
||||
# Expose aligned_kv_bytes_per_offloaded_block as
|
||||
# kv_bytes_per_offloaded_block. Note that this might contain
|
||||
# Expose aligned_kv_bytes_per_chunk as
|
||||
# kv_bytes_per_chunk. Note that this might contain
|
||||
# some padding. i.e. each offloaded block is of the form,
|
||||
# |--- W0-B0---|---- W1-B0---| ... |---- Wn-B0---| *** maybe-pad *** |
|
||||
self.kv_bytes_per_offloaded_block = aligned_kv_bytes_per_offloaded_block
|
||||
self.kv_bytes_per_chunk = aligned_kv_bytes_per_chunk
|
||||
|
||||
# scheduler-side
|
||||
self._manager: OffloadingManager | None = None
|
||||
@@ -150,7 +135,7 @@ class CPUOffloadingSpec(OffloadingSpec):
|
||||
def create_worker(self, kv_caches: CanonicalKVCaches) -> CPUOffloadingWorker:
|
||||
return CPUOffloadingWorker(
|
||||
kv_caches=kv_caches,
|
||||
block_size_factor=self.block_size_factor,
|
||||
blocks_per_chunk=self.blocks_per_chunk,
|
||||
num_cpu_blocks=self.num_blocks,
|
||||
)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user