[KV Offload] Replace OffloadingHandler with OffloadingWorker (#45053)

Signed-off-by: Martin Hickey <martin.hickey@ie.ibm.com>
This commit is contained in:
Martin Hickey
2026-06-24 14:44:24 +03:00
committed by GitHub
parent 70749fdcca
commit f237e16b41
13 changed files with 176 additions and 465 deletions
@@ -98,7 +98,7 @@ def _make_worker(kv_cache_config: KVCacheConfig):
spec = MagicMock(spec=OffloadingSpec)
spec.kv_cache_config = kv_cache_config
spec.vllm_config = MagicMock()
spec.get_handlers.return_value = iter([])
spec.get_worker.return_value = MagicMock()
worker = OffloadingConnectorWorker(spec=spec)
worker.worker = MagicMock()
@@ -279,7 +279,7 @@ def test_register_kv_caches(backend):
worker, spec = _make_worker(kv_cache_config)
worker.register_kv_caches(kv_caches)
canonical = spec.get_handlers.call_args[0][0]
canonical = spec.get_worker.call_args[0][0]
assert isinstance(canonical, CanonicalKVCaches)
# -- Expected block tensors ----------------------------------------------
@@ -422,7 +422,7 @@ def test_register_kv_caches_uniform_type(backend):
worker, spec = _make_worker(kv_cache_config)
worker.register_kv_caches(kv_caches)
canonical = spec.get_handlers.call_args[0][0]
canonical = spec.get_worker.call_args[0][0]
assert isinstance(canonical, CanonicalKVCaches)
for block_tensor in canonical.tensors:
@@ -1,6 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from collections.abc import Callable, Iterable, Iterator
from collections.abc import Callable, Iterable
from dataclasses import dataclass
from typing import Any
from unittest.mock import MagicMock
@@ -43,19 +43,17 @@ from vllm.v1.kv_cache_interface import (
KVCacheGroupSpec,
)
from vllm.v1.kv_offload.base import (
CanonicalKVCaches,
GPULoadStoreSpec,
LoadStoreSpec,
OffloadingManager,
OffloadingSpec,
OffloadingWorker,
OffloadKey,
PrepareStoreOutput,
RequestOffloadingContext,
make_offload_key,
)
from vllm.v1.kv_offload.worker.worker import (
OffloadingHandler,
TransferResult,
TransferSpec,
make_offload_key,
)
from vllm.v1.request import Request
from vllm.v1.structured_output import StructuredOutputManager
@@ -81,9 +79,9 @@ class MockLoadStoreSpec(LoadStoreSpec):
return repr(self.offload_keys)
class MockOffloadingHandler(OffloadingHandler):
class MockOffloadingWorker(OffloadingWorker):
def __init__(self):
self.transfer_specs: dict[int, TransferSpec] = {}
self.transfer_specs: dict[int, tuple[LoadStoreSpec, LoadStoreSpec]] = {}
self.completed_transfers: list[TransferResult] = []
self.waiting_jobs: set[int] = set()
self.completed_jobs: list[int] = []
@@ -94,8 +92,17 @@ class MockOffloadingHandler(OffloadingHandler):
self.completed_transfers = []
return finished
def transfer_async(self, job_id: int, spec: TransferSpec) -> bool:
self.transfer_specs[job_id] = spec
def submit_store(
self, job_id: int, src_spec: LoadStoreSpec, dst_spec: LoadStoreSpec
) -> bool: # type: ignore[override]
self.transfer_specs[job_id] = (src_spec, dst_spec)
self.waiting_jobs.add(job_id)
return True
def submit_load(
self, job_id: int, src_spec: LoadStoreSpec, dst_spec: LoadStoreSpec
) -> bool: # type: ignore[override]
self.transfer_specs[job_id] = (src_spec, dst_spec)
self.waiting_jobs.add(job_id)
return True
@@ -109,7 +116,6 @@ class MockOffloadingHandler(OffloadingHandler):
success=True,
transfer_size=None,
transfer_time=None,
transfer_type=None,
)
self.completed_transfers.append(result)
@@ -127,21 +133,18 @@ class MockOffloadingSpec(OffloadingSpec):
self.manager.prepare_load = lambda keys, req_context: MockLoadStoreSpec(keys)
self.manager.lookup.return_value = False
self.manager.on_new_request.return_value = RequestOffloadingContext()
self.handler = MockOffloadingHandler()
self.handler = MockOffloadingWorker()
def get_manager(self) -> OffloadingManager:
return self.manager
def get_handlers(
self, _
) -> Iterator[tuple[type[LoadStoreSpec], type[LoadStoreSpec], OffloadingHandler]]:
yield GPULoadStoreSpec, MockLoadStoreSpec, self.handler
yield MockLoadStoreSpec, GPULoadStoreSpec, self.handler
def get_worker(self, _: CanonicalKVCaches) -> OffloadingWorker:
return self.handler
def complete_transfers(self):
self.handler.complete_jobs(self.handler.waiting_jobs.copy())
def get_completed_transfers(self) -> list[TransferSpec]:
def get_completed_transfers(self) -> list[tuple[LoadStoreSpec, LoadStoreSpec]]:
specs = [
self.handler.transfer_specs[job_id]
for job_id in self.handler.completed_jobs
@@ -149,7 +152,7 @@ class MockOffloadingSpec(OffloadingSpec):
self.handler.completed_jobs.clear()
return specs
def get_flushed_transfers(self):
def get_flushed_transfers(self) -> list[tuple[LoadStoreSpec, LoadStoreSpec]]:
specs = [
self.handler.transfer_specs[job_id] for job_id in self.handler.flushed_jobs
]
@@ -362,8 +365,7 @@ class RequestRunner:
self.scheduler.add_request(req)
def _parse_transfers(self):
for transfer_spec in self.offloading_spec.get_flushed_transfers():
src_spec, dst_spec = transfer_spec
for src_spec, dst_spec in self.offloading_spec.get_flushed_transfers():
if isinstance(src_spec, GPULoadStoreSpec):
# store flush
for block_id in src_spec.block_ids:
@@ -375,9 +377,7 @@ class RequestRunner:
block_size_factor = self.block_size_factor
for transfer_spec in self.offloading_spec.get_completed_transfers():
src_spec, dst_spec = transfer_spec
for src_spec, dst_spec in self.offloading_spec.get_completed_transfers():
if isinstance(src_spec, GPULoadStoreSpec):
store = True
gpu_spec = src_spec
+20 -21
View File
@@ -17,7 +17,7 @@ from vllm.v1.kv_offload.base import (
GPULoadStoreSpec,
)
from vllm.v1.kv_offload.cpu.common import CPULoadStoreSpec
from vllm.v1.kv_offload.cpu.gpu_worker import CpuGpuOffloadingHandlers
from vllm.v1.kv_offload.cpu.gpu_worker import CPUOffloadingWorker
from vllm.v1.kv_offload.cpu.shared_offload_region import SharedOffloadRegion
NUM_GPU_BLOCKS = [64]
@@ -103,7 +103,7 @@ def test_transfer(
cpu_page_size=cpu_page_size,
)
handlers = CpuGpuOffloadingHandlers(
worker = CPUOffloadingWorker(
kv_caches=kv_caches,
block_size_factor=block_size_factor,
num_cpu_blocks=num_cpu_blocks,
@@ -130,7 +130,7 @@ def test_transfer(
# set transfer direction
if gpu_to_cpu:
handler = handlers.gpu_to_cpu_handler
handler = worker._store_handler
src_spec = GPULoadStoreSpec(
gpu_blocks, group_sizes=(len(gpu_blocks),), block_indices=(blocks_to_skip,)
)
@@ -138,7 +138,7 @@ def test_transfer(
dst_to_src = dict(zip(cpu_blocks_expanded, gpu_blocks))
num_dst_sub_blocks = num_gpu_blocks
else:
handler = handlers.cpu_to_gpu_handler
handler = worker._load_handler
src_spec = CPULoadStoreSpec(cpu_blocks)
dst_spec = GPULoadStoreSpec(
gpu_blocks, group_sizes=(len(gpu_blocks),), block_indices=(blocks_to_skip,)
@@ -156,23 +156,21 @@ def test_transfer(
orig_src_tensors = [x.clone() for x in handler.src_tensors]
orig_dst_tensors = [x.clone() for x in handler.dst_tensors]
# call transfer function
# call transfer function via public API
start_time = time.time()
assert handler.transfer_async(1, (src_spec, dst_spec))
if gpu_to_cpu:
assert worker.submit_store(1, src_spec, dst_spec)
else:
assert worker.submit_load(1, src_spec, dst_spec)
assert {x.job_id for x in handler._transfers} == {1}
# wait for transfer to complete
end_time = time.time() + 10
while time.time() < end_time:
finished = handler.get_finished()
finished = worker.get_finished()
if finished:
assert finished[0].job_id == 1
assert finished[0].success
assert (
finished[0].transfer_type == ("GPU", "CPU")
if gpu_to_cpu
else ("CPU", "GPU")
)
assert finished[0].transfer_size == (
len(gpu_blocks)
* sum([x.page_size_bytes for x in handler.kv_cache_groups_data_refs[0]])
@@ -208,8 +206,7 @@ def test_transfer(
del orig_tensor, tensor, src_tensor, dst_tensor, orig_dst_tensor
del src_view, dst_view, orig_dst_view, expected
handlers.cpu_to_gpu_handler.shutdown()
handlers.gpu_to_cpu_handler.shutdown()
worker.shutdown()
if mmap_region:
mmap_region.cleanup()
@@ -276,7 +273,7 @@ def test_transfer_multi_group(
tensors=kv_cache_tensors, group_data_refs=kv_cache_groups_data_refs
)
handlers = CpuGpuOffloadingHandlers(
worker = CPUOffloadingWorker(
kv_caches=canonical_kv_caches,
block_size_factor=block_size_factor,
num_cpu_blocks=num_cpu_blocks,
@@ -338,7 +335,7 @@ def test_transfer_multi_group(
block_indices: list[int] = [0, 0, sub_blocks_to_skip]
if gpu_to_cpu:
handler = handlers.gpu_to_cpu_handler
handler = worker._store_handler
src_spec = GPULoadStoreSpec(
gpu_blocks, group_sizes=group_sizes, block_indices=block_indices
)
@@ -352,7 +349,7 @@ def test_transfer_multi_group(
]
num_dst_sub_blocks = num_cpu_blocks * block_size_factor
else:
handler = handlers.cpu_to_gpu_handler
handler = worker._load_handler
src_spec = CPULoadStoreSpec(cpu_blocks)
dst_spec = GPULoadStoreSpec(
gpu_blocks, group_sizes=group_sizes, block_indices=block_indices
@@ -375,12 +372,15 @@ def test_transfer_multi_group(
orig_src_tensors = [x.clone() for x in handler.src_tensors]
orig_dst_tensors = [x.clone() for x in handler.dst_tensors]
assert handler.transfer_async(1, (src_spec, dst_spec))
if gpu_to_cpu:
assert worker.submit_store(1, src_spec, dst_spec)
else:
assert worker.submit_load(1, src_spec, dst_spec)
assert {x.job_id for x in handler._transfers} == {1}
end_time = time.time() + 10
while time.time() < end_time:
finished = handler.get_finished()
finished = worker.get_finished()
if finished:
assert finished[0].job_id == 1
assert finished[0].success
@@ -418,5 +418,4 @@ def test_transfer_multi_group(
dst_view[dst_sub_block].cpu(), expected.cpu()
)
handlers.cpu_to_gpu_handler.shutdown()
handlers.gpu_to_cpu_handler.shutdown()
worker.shutdown()
-165
View File
@@ -1,165 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from vllm.v1.kv_offload.base import LoadStoreSpec
from vllm.v1.kv_offload.worker.worker import (
OffloadingHandler,
OffloadingWorker,
TransferResult,
TransferSpec,
)
class LoadStoreSpec1(LoadStoreSpec):
def __init__(
self,
submit_success: bool = True,
async_success: bool = True,
exception: bool = False,
):
self.finished = False
self.submit_success = submit_success
self.async_success = async_success
self.exception = exception
@staticmethod
def medium() -> str:
return "1"
def __repr__(self):
return f"{self.medium()}: {id(self)}"
class LoadStoreSpec2(LoadStoreSpec):
@staticmethod
def medium() -> str:
return "2"
def __repr__(self):
return f"{self.medium()}: {id(self)}"
class OffloadingHandler1To2(OffloadingHandler):
def __init__(self):
self.transfers: dict[int, LoadStoreSpec1] = {}
def transfer_async(self, job_id: int, spec: TransferSpec) -> bool:
src, dst = spec
assert isinstance(src, LoadStoreSpec1)
assert isinstance(dst, LoadStoreSpec2)
if src.exception:
raise Exception("An expected exception. Don't worry!")
if not src.submit_success:
return False
self.transfers[job_id] = src
return True
def get_finished(self) -> list[TransferResult]:
finished = []
for job_id, spec in list(self.transfers.items()):
if spec.finished:
finished.append((job_id, spec.async_success))
del self.transfers[job_id]
return finished
def wait(self, job_ids: set[int]) -> None:
for job_id in job_ids:
spec = self.transfers.get(job_id)
if spec:
assert spec.finished
class OffloadingHandler2To1(OffloadingHandler):
def __init__(self):
self.transfers: dict[int, LoadStoreSpec1] = {}
def transfer_async(self, job_id: int, spec: TransferSpec) -> bool:
src, dst = spec
assert isinstance(src, LoadStoreSpec2)
assert isinstance(dst, LoadStoreSpec1)
self.transfers[job_id] = dst
return True
def get_finished(self) -> list[TransferResult]:
finished = []
for job_id, spec in list(self.transfers.items()):
if spec.finished:
finished.append((job_id, spec.async_success))
del self.transfers[job_id]
return finished
def wait(self, job_ids: set[int]) -> None:
for job_id in job_ids:
spec = self.transfers.get(job_id)
if spec:
assert spec.finished
def test_offloading_worker():
"""
Tests OffloadingWorker with 2 handlers.
One handler performs 1->2 transfers, and the other handles 2->1.
"""
worker = OffloadingWorker()
handler1to2 = OffloadingHandler1To2()
handler2to1 = OffloadingHandler2To1()
worker.register_handler(LoadStoreSpec1, LoadStoreSpec2, handler1to2)
worker.register_handler(LoadStoreSpec2, LoadStoreSpec1, handler2to1)
# 1st transfer 1->2 (exception)
src1 = LoadStoreSpec1(exception=True)
dst1 = LoadStoreSpec2()
assert not worker.transfer_async(1, (src1, dst1))
# 2ed transfer 1->2 (failure to submit)
src2 = LoadStoreSpec1(submit_success=False)
dst2 = LoadStoreSpec2()
assert not worker.transfer_async(2, (src2, dst2))
# 3rd transfer 1->2 (failure)
src3 = LoadStoreSpec1(async_success=False)
dst3 = LoadStoreSpec2()
assert worker.transfer_async(3, (src3, dst3))
# 4th transfer 1->2 (success)
src4 = LoadStoreSpec1()
dst4 = LoadStoreSpec2()
worker.transfer_async(4, (src4, dst4))
assert set(handler1to2.transfers.keys()) == {3, 4}
# 5th transfer 2->1
src5 = LoadStoreSpec2()
dst5 = LoadStoreSpec1()
worker.transfer_async(5, (src5, dst5))
assert set(handler2to1.transfers.keys()) == {5}
# no transfer completed yet
assert worker.get_finished() == []
# complete 3rd, 4th
src3.finished = True
src4.finished = True
# 6th transfer 1->2
src6 = LoadStoreSpec1()
dst6 = LoadStoreSpec2()
worker.transfer_async(6, (src6, dst6))
# 7th transfer 2->1
src7 = LoadStoreSpec2()
dst7 = LoadStoreSpec1()
worker.transfer_async(7, (src7, dst7))
# 6th and 7th transfers started
assert 6 in handler1to2.transfers
assert 7 in handler2to1.transfers
# verify result of 3rd and 4th transfers
assert sorted(worker.get_finished()) == [(3, False), (4, True)]
# complete 6th and 7th transfers
src6.finished = True
dst7.finished = True
assert sorted(worker.get_finished()) == [(6, True), (7, True)]
@@ -6,7 +6,7 @@ from vllm.distributed.kv_transfer.kv_connector.v1.base import (
KVConnectorMetadata,
KVConnectorWorkerMetadata,
)
from vllm.v1.kv_offload.worker.worker import TransferSpec
from vllm.v1.kv_offload.base import LoadStoreSpec
ReqId = str
@@ -60,7 +60,8 @@ class TransferJob:
"""
req_id: ReqId
transfer_spec: TransferSpec
src_spec: LoadStoreSpec
dst_spec: LoadStoreSpec
@dataclass
@@ -762,7 +762,8 @@ class OffloadingConnectorScheduler:
load_job_id = self._generate_job_id()
self._current_batch_load_jobs[load_job_id] = TransferJob(
req_id=request.request_id,
transfer_spec=(src_spec, dst_spec),
src_spec=src_spec,
dst_spec=dst_spec,
)
# a load can only be issued when no other jobs are pending.
assert not req_status.transfer_jobs
@@ -998,7 +999,7 @@ class OffloadingConnectorScheduler:
)
store_jobs[job_id] = TransferJob(
req_id=req_id, transfer_spec=(src_spec, dst_spec)
req_id=req_id, src_spec=src_spec, dst_spec=dst_spec
)
logger.debug(
@@ -21,11 +21,10 @@ from vllm.v1.kv_offload.base import (
CanonicalKVCacheRef,
CanonicalKVCaches,
CanonicalKVCacheTensor,
GPULoadStoreSpec,
LoadStoreSpec,
OffloadingSpec,
)
from vllm.v1.kv_offload.worker.worker import (
OffloadingWorker,
TransferSpec,
)
logger = init_logger(__name__)
@@ -36,16 +35,17 @@ class OffloadingConnectorWorker:
def __init__(self, spec: OffloadingSpec):
self.spec = spec
self.worker = OffloadingWorker()
self.worker: OffloadingWorker | None = None
# job_id -> req_id for in-flight loads.
self._load_jobs: dict[int, ReqId] = {}
self._unsubmitted_store_jobs: list[tuple[int, TransferSpec]] = []
self._unsubmitted_store_jobs: list[
tuple[int, GPULoadStoreSpec, LoadStoreSpec]
] = []
self._connector_worker_meta = OffloadingWorkerMetadata()
def _register_handlers(self, kv_caches: CanonicalKVCaches):
for src_cls, dst_cls, handler in self.spec.get_handlers(kv_caches):
self.worker.register_handler(src_cls, dst_cls, handler)
def _init_worker(self, kv_caches: CanonicalKVCaches) -> None:
self.worker = self.spec.get_worker(kv_caches)
def register_kv_caches(
self, kv_caches: dict[str, torch.Tensor | list[torch.Tensor]]
@@ -138,7 +138,7 @@ class OffloadingConnectorWorker:
(block_stride, 1),
storage_offset=0,
)
self._register_handlers(
self._init_worker(
CanonicalKVCaches(
[CanonicalKVCacheTensor(packed_tensor, block_stride)],
[
@@ -204,7 +204,7 @@ class OffloadingConnectorWorker:
group_data_refs=group_data_refs,
)
self._register_handlers(canonical_kv_caches)
self._init_worker(canonical_kv_caches)
def register_cross_layers_kv_cache(
self, kv_cache: torch.Tensor, attn_backend: type[AttentionBackend]
@@ -251,11 +251,12 @@ class OffloadingConnectorWorker:
tensors=[kv_cache_tensor], group_data_refs=[[kv_cache_data_ref]]
)
self._register_handlers(canonical_kv_caches)
self._init_worker(canonical_kv_caches)
def handle_preemptions(self, kv_connector_metadata: OffloadingConnectorMetadata):
for job_id, transfer_spec in self._unsubmitted_store_jobs:
success = self.worker.transfer_async(job_id, transfer_spec)
assert self.worker is not None
for job_id, src_spec, dst_spec in self._unsubmitted_store_jobs:
success = self.worker.submit_store(job_id, src_spec, dst_spec)
assert success
self._unsubmitted_store_jobs.clear()
@@ -263,14 +264,16 @@ class OffloadingConnectorWorker:
self.worker.wait(kv_connector_metadata.jobs_to_flush)
def start_kv_transfers(self, metadata: OffloadingConnectorMetadata):
for job_id, transfer_spec in self._unsubmitted_store_jobs:
success = self.worker.transfer_async(job_id, transfer_spec)
assert self.worker is not None
for job_id, src_spec, dst_spec in self._unsubmitted_store_jobs:
success = self.worker.submit_store(job_id, src_spec, dst_spec)
assert success
self._unsubmitted_store_jobs.clear()
for job_id, entry in metadata.load_jobs.items():
self._load_jobs[job_id] = entry.req_id
success = self.worker.transfer_async(job_id, entry.transfer_spec)
assert isinstance(entry.dst_spec, GPULoadStoreSpec)
success = self.worker.submit_load(job_id, entry.src_spec, entry.dst_spec)
assert success
def prepare_store_kv(self, metadata: OffloadingConnectorMetadata):
@@ -278,7 +281,10 @@ class OffloadingConnectorWorker:
# NOTE(orozery): defer the store to the beginning of the next
# engine step, so that offloading starts AFTER transfers related
# to token sampling, thereby avoiding delays to token generation.
self._unsubmitted_store_jobs.append((job_id, entry.transfer_spec))
assert isinstance(entry.src_spec, GPULoadStoreSpec)
self._unsubmitted_store_jobs.append(
(job_id, entry.src_spec, entry.dst_spec)
)
def get_finished(self, finished_req_ids: set[str]) -> tuple[set[str], set[str]]:
"""
@@ -290,6 +296,7 @@ class OffloadingConnectorWorker:
finished_recving so the base scheduler can resume requests
blocked on remote KV (and free aborted-during-load reqs).
"""
assert self.worker is not None
finished_recving: set[str] = set()
for transfer_result in self.worker.get_finished():
# we currently do not support job failures
@@ -328,4 +335,5 @@ class OffloadingConnectorWorker:
self._unsubmitted_store_jobs.clear()
self._load_jobs.clear()
self._connector_worker_meta = OffloadingWorkerMetadata()
self.worker.shutdown()
if self.worker is not None:
self.worker.shutdown()
+40 -8
View File
@@ -5,7 +5,7 @@ Core abstractions for KV cache offloading in vLLM v1.
"""
from abc import ABC, abstractmethod
from collections.abc import Collection, Iterable, Iterator, Sequence
from collections.abc import Collection, Iterable, Sequence
from dataclasses import dataclass
from enum import Enum
from typing import TYPE_CHECKING, Any, NewType
@@ -23,7 +23,6 @@ if TYPE_CHECKING:
OffloadingConnectorStats,
)
from vllm.v1.kv_cache_interface import KVCacheConfig
from vllm.v1.kv_offload.worker.worker import OffloadingHandler
# `OffloadKey` identifies an offloaded block. It combines a block hash with
# its KV cache group index, encoded as raw bytes to avoid tuple GC overhead.
@@ -432,6 +431,41 @@ class CanonicalKVCaches:
group_data_refs: list[list[CanonicalKVCacheRef]]
@dataclass
class TransferResult:
job_id: int
success: bool
transfer_size: int | None = None
transfer_time: float | None = None
class OffloadingWorker(ABC):
"""Runs in the worker process. Performs async KV transfers for ONE
offloaded medium (e.g. CPU). Direction is explicit via submit_store /
submit_load, so there is no (src_medium, dst_medium) routing."""
@abstractmethod
def submit_store(
self, job_id: int, src_spec: GPULoadStoreSpec, dst_spec: LoadStoreSpec
) -> bool:
"""Async GPU -> offloaded medium."""
@abstractmethod
def submit_load(
self, job_id: int, src_spec: LoadStoreSpec, dst_spec: GPULoadStoreSpec
) -> bool:
"""Async offloaded medium -> GPU."""
@abstractmethod
def get_finished(self) -> list[TransferResult]: ...
@abstractmethod
def wait(self, job_ids: set[int]) -> None: ...
def shutdown(self) -> None:
return
class OffloadingSpec(ABC):
"""Spec for an offloading connector"""
@@ -524,16 +558,14 @@ class OffloadingSpec(ABC):
pass
@abstractmethod
def get_handlers(
self, kv_caches: CanonicalKVCaches
) -> Iterator[tuple[type[LoadStoreSpec], type[LoadStoreSpec], "OffloadingHandler"]]:
def get_worker(self, kv_caches: CanonicalKVCaches) -> OffloadingWorker:
"""
Get offloading handlers along with their respective src and dst types.
Get an OffloadingWorker that handles async KV transfers for this spec.
Args:
kv_caches: Canonicalized KV caches.
Yields:
Tuples of (src_type, dst_type, offloading_handler).
Returns:
An OffloadingWorker instance for this medium.
"""
pass
+41 -21
View File
@@ -7,7 +7,6 @@ from dataclasses import dataclass
import numpy as np
import torch
from typing_extensions import override
from vllm import _custom_ops as ops
from vllm.logger import init_logger
@@ -20,17 +19,15 @@ from vllm.v1.kv_offload.base import (
CanonicalKVCacheRef,
CanonicalKVCaches,
GPULoadStoreSpec,
LoadStoreSpec,
OffloadingWorker,
TransferResult,
)
from vllm.v1.kv_offload.cpu.shared_offload_region import SharedOffloadRegion
from vllm.v1.kv_offload.cpu.swap_blocks_triton import (
THRESHOLD_BYTES,
swap_blocks_batch,
)
from vllm.v1.kv_offload.worker.worker import (
OffloadingHandler,
TransferResult,
TransferSpec,
)
logger = init_logger(__name__)
@@ -166,10 +163,9 @@ def _new_descriptor_buffers(
)
class SingleDirectionOffloadingHandler(OffloadingHandler):
class SingleDirectionOffloadingHandler:
"""
SingleDirectionOffloadingHandler handles transfers for a single direction,
either CPU->GPU or GPU->CPU.
Handles transfers for a single direction, either CPU->GPU or GPU->CPU.
Transfers are guaranteed to be executed in order of their submission.
Each transfer uses a unique CUDA stream, and its stream will start
executing only after the streams of previous transfers have finished.
@@ -228,7 +224,6 @@ class SingleDirectionOffloadingHandler(OffloadingHandler):
self.src_block_size_factor = 1 if self.gpu_to_cpu else block_size_factor
self.dst_block_size_factor = block_size_factor if self.gpu_to_cpu else 1
self.transfer_type = ("GPU", "CPU") if self.gpu_to_cpu else ("CPU", "GPU")
# mmap_region to clean up on shutdown (gpu_to_cpu handler owns it)
self._mmap_region = mmap_region
# job_id -> event
@@ -242,9 +237,9 @@ class SingleDirectionOffloadingHandler(OffloadingHandler):
# list of pinned descriptor buffer sets available for re-use
self._buffer_pool: list[tuple[torch.Tensor, torch.Tensor, torch.Tensor]] = []
@override
def transfer_async(self, job_id: int, transfer_spec: TransferSpec) -> bool:
src_spec, dst_spec = transfer_spec
def transfer_async(
self, job_id: int, src_spec: LoadStoreSpec, dst_spec: LoadStoreSpec
) -> bool:
assert isinstance(src_spec, BlockIDsLoadStoreSpec)
assert isinstance(dst_spec, BlockIDsLoadStoreSpec)
@@ -425,7 +420,6 @@ class SingleDirectionOffloadingHandler(OffloadingHandler):
# success
return True
@override
def get_finished(self) -> list[TransferResult]:
results: list[TransferResult] = []
while self._transfers and self._transfers[0].end_event.query():
@@ -438,7 +432,6 @@ class SingleDirectionOffloadingHandler(OffloadingHandler):
success=True,
transfer_size=transfer.num_bytes,
transfer_time=transfer_time,
transfer_type=self.transfer_type,
)
results.append(result)
@@ -451,14 +444,12 @@ class SingleDirectionOffloadingHandler(OffloadingHandler):
del self._transfer_events[transfer.job_id]
return results
@override
def wait(self, job_ids: set[int]):
for job_id in job_ids:
event = self._transfer_events.get(job_id)
if event is not None:
event.synchronize()
@override
def shutdown(self) -> None:
while self._transfers:
transfer = self._transfers.popleft()
@@ -474,7 +465,14 @@ class SingleDirectionOffloadingHandler(OffloadingHandler):
self._mmap_region = None
class CpuGpuOffloadingHandlers:
class CPUOffloadingWorker(OffloadingWorker):
"""OffloadingWorker for CPU offloading.
Composes two SingleDirectionOffloadingHandler instances (one for each
direction) and exposes them through the explicit submit_store /
submit_load API.
"""
def __init__(
self,
kv_caches: CanonicalKVCaches,
@@ -484,7 +482,6 @@ class CpuGpuOffloadingHandlers:
):
pin_memory = PIN_MEMORY
logger.info("Allocating %d CPU tensors...", len(kv_caches.tensors))
self._mmap_region = mmap_region
if mmap_region is not None and pin_memory:
pin_mmap_region(mmap_region)
@@ -518,7 +515,7 @@ class CpuGpuOffloadingHandlers:
gpu_tensors.append(gpu_tensor)
cpu_tensors.append(cpu_tensor)
self.gpu_to_cpu_handler = SingleDirectionOffloadingHandler(
self._store_handler = SingleDirectionOffloadingHandler(
gpu_tensors=gpu_tensors,
cpu_tensors=cpu_tensors,
block_size_factor=block_size_factor,
@@ -527,10 +524,33 @@ class CpuGpuOffloadingHandlers:
mmap_region=mmap_region,
)
self.cpu_to_gpu_handler = SingleDirectionOffloadingHandler(
self._load_handler = SingleDirectionOffloadingHandler(
gpu_tensors=gpu_tensors,
cpu_tensors=cpu_tensors,
block_size_factor=block_size_factor,
kv_cache_groups_data_refs=kv_caches.group_data_refs,
gpu_to_cpu=False,
)
def submit_store(
self, job_id: int, src_spec: GPULoadStoreSpec, dst_spec: LoadStoreSpec
) -> bool:
"""Async GPU -> CPU."""
return self._store_handler.transfer_async(job_id, src_spec, dst_spec)
def submit_load(
self, job_id: int, src_spec: LoadStoreSpec, dst_spec: GPULoadStoreSpec
) -> bool:
"""Async CPU -> GPU."""
return self._load_handler.transfer_async(job_id, src_spec, dst_spec)
def get_finished(self) -> list[TransferResult]:
return self._store_handler.get_finished() + self._load_handler.get_finished()
def wait(self, job_ids: set[int]) -> None:
self._store_handler.wait(job_ids)
self._load_handler.wait(job_ids)
def shutdown(self) -> None:
self._store_handler.shutdown()
self._load_handler.shutdown()
+11 -20
View File
@@ -1,6 +1,5 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from collections.abc import Iterator
from typing import Any
from typing_extensions import override
@@ -11,21 +10,16 @@ from vllm.utils.math_utils import round_up
from vllm.v1.kv_cache_interface import KVCacheConfig
from vllm.v1.kv_offload.base import (
CanonicalKVCaches,
GPULoadStoreSpec,
LoadStoreSpec,
OffloadingCounterMetadata,
OffloadingGaugeMetadata,
OffloadingManager,
OffloadingMetricMetadata,
OffloadingSpec,
OffloadingWorker,
)
from vllm.v1.kv_offload.cpu.common import (
CPULoadStoreSpec,
CPUOffloadingMetrics,
)
from vllm.v1.kv_offload.cpu.gpu_worker import CpuGpuOffloadingHandlers
from vllm.v1.kv_offload.cpu.common import CPUOffloadingMetrics
from vllm.v1.kv_offload.cpu.gpu_worker import CPUOffloadingWorker
from vllm.v1.kv_offload.cpu.manager import CPUOffloadingManager
from vllm.v1.kv_offload.worker.worker import OffloadingHandler
class CPUOffloadingSpec(OffloadingSpec):
@@ -107,7 +101,7 @@ class CPUOffloadingSpec(OffloadingSpec):
self._manager: OffloadingManager | None = None
# worker-side
self._handlers: CpuGpuOffloadingHandlers | None = None
self._worker: CPUOffloadingWorker | None = None
self.eviction_policy: str = self.extra_config.get("eviction_policy", "lru")
@@ -131,25 +125,22 @@ class CPUOffloadingSpec(OffloadingSpec):
)
return self._manager
def create_handlers(self, kv_caches: CanonicalKVCaches) -> CpuGpuOffloadingHandlers:
return CpuGpuOffloadingHandlers(
def create_worker(self, kv_caches: CanonicalKVCaches) -> CPUOffloadingWorker:
return CPUOffloadingWorker(
kv_caches=kv_caches,
block_size_factor=self.block_size_factor,
num_cpu_blocks=self.num_blocks,
)
@override
def get_handlers(
self, kv_caches: CanonicalKVCaches
) -> Iterator[tuple[type[LoadStoreSpec], type[LoadStoreSpec], OffloadingHandler]]:
if not self._handlers:
def get_worker(self, kv_caches: CanonicalKVCaches) -> OffloadingWorker:
if not self._worker:
if not (current_platform.is_cuda_alike() or current_platform.is_xpu()):
raise Exception(
"CPU Offloading is currently only supported on CUDA-alike "
"and XPU GPUs"
)
self._handlers = self.create_handlers(kv_caches)
self._worker = self.create_worker(kv_caches)
assert self._handlers is not None
yield GPULoadStoreSpec, CPULoadStoreSpec, self._handlers.gpu_to_cpu_handler
yield CPULoadStoreSpec, GPULoadStoreSpec, self._handlers.cpu_to_gpu_handler
assert self._worker is not None
return self._worker
+5 -5
View File
@@ -44,7 +44,7 @@ from vllm.v1.kv_offload.base import (
OffloadingManager,
OffloadingMetricMetadata,
)
from vllm.v1.kv_offload.cpu.gpu_worker import CpuGpuOffloadingHandlers
from vllm.v1.kv_offload.cpu.gpu_worker import CPUOffloadingWorker
from vllm.v1.kv_offload.cpu.shared_offload_region import SharedOffloadRegion
from vllm.v1.kv_offload.cpu.spec import CPUOffloadingSpec
from vllm.v1.kv_offload.tiering.factory import SecondaryTierFactory
@@ -163,8 +163,8 @@ class TieringOffloadingSpec(CPUOffloadingSpec):
raise
# Create TieringOffloadingManager. GPU↔CPU transfers use the inherited
# get_handlers(); secondary tier transfers are handled by the
# secondary tier managers and need no additional handlers here.
# get_worker(). Secondary tier transfers are handled by the
# secondary tier managers and need no additional workers here.
tiering_manager = TieringOffloadingManager(
primary_tier=primary_tier,
secondary_tiers=secondary_tiers,
@@ -187,7 +187,7 @@ class TieringOffloadingSpec(CPUOffloadingSpec):
return self._manager
@override
def create_handlers(self, kv_caches: CanonicalKVCaches) -> CpuGpuOffloadingHandlers:
def create_worker(self, kv_caches: CanonicalKVCaches) -> CPUOffloadingWorker:
rank = torch.accelerator.current_device_index()
worker_mmap = SharedOffloadRegion(
instance_id=self.vllm_config.instance_id,
@@ -196,7 +196,7 @@ class TieringOffloadingSpec(CPUOffloadingSpec):
kv_bytes_per_block=self.kv_bytes_per_offloaded_block,
cpu_page_size=self.cpu_page_size_per_worker,
)
return CpuGpuOffloadingHandlers(
return CPUOffloadingWorker(
kv_caches=kv_caches,
block_size_factor=self.block_size_factor,
num_cpu_blocks=self.num_blocks,
-176
View File
@@ -1,176 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from abc import ABC, abstractmethod
from dataclasses import dataclass
from vllm.logger import init_logger
from vllm.v1.kv_offload.base import LoadStoreSpec
# a single transfer spec (src_blocks_spec, dst_blocks_spec)
TransferSpec = tuple[LoadStoreSpec, LoadStoreSpec]
# transfers are forwarded to workers by (src_medium, dst_medium)
TransferType = tuple[str, str]
logger = init_logger(__name__)
@dataclass
class TransferResult:
job_id: int
success: bool
transfer_size: int | None = None # Size in bytes
transfer_time: float | None = None
transfer_type: TransferType | None = None
class OffloadingHandler(ABC):
"""
OffloadingHandler class for managing asynchronous KV data transfers
This class runs in the worker.
It kicks off async KV data transfer requests, and allows
collecting back completion statuses.
The class provides the following primitives:
transfer_async() - kicks off a new transfer job
get_finished() - returns a list of newly finished job IDs.
"""
@abstractmethod
def transfer_async(self, job_id: int, spec: TransferSpec) -> bool:
"""
Initiates an asynchronous transfer of KV data.
Args:
job_id: a unique ID that will be used when notifying back on
transfer completion.
spec: the (src, dst) spec of the KV data transfer.
Returns:
True if transfer was submitted successfully.
"""
pass
@abstractmethod
def get_finished(self) -> list[TransferResult]:
"""
Get transfers finished since last call.
Returns:
A list of (job_id, success) of transfers.
"""
pass
@abstractmethod
def wait(self, job_ids: set[int]) -> None:
"""
Wait for jobs to finish (blocking).
Args:
job_ids: The set of job IDs to wait for.
"""
def shutdown(self) -> None:
"""Shutdown the handler and release any resources."""
return
class OffloadingWorker:
"""
OffloadingWorker class for managing asynchronous KV data transfers
using multiple OffloadingHandlers
This class runs in the worker.
It kicks off async KV data transfer requests, by delegating
to one of its registered OffloadingHandlers, based on the transfer type.
The class provides the following primitives:
register_handler() - registers a new handler to handle
a specific transfer type
transfer_async() - kicks off a new transfer job
using one of the registered handlers.
get_finished() - returns a list of newly finished job IDs
from all handlers.
"""
def __init__(self):
self.handlers: set[OffloadingHandler] = set()
self.transfer_type_to_handler: dict[TransferType, OffloadingHandler] = {}
def register_handler(
self,
src_cls: type[LoadStoreSpec],
dst_cls: type[LoadStoreSpec],
handler: OffloadingHandler,
) -> None:
"""
Registers a new handler.
Args:
src_cls: the source type of transfers handled by this handler.
dst_cls: the destination type of transfers handled by this handler.
handler: the handler that will handle transfers.
"""
transfer_type = (src_cls.medium(), dst_cls.medium())
assert transfer_type not in self.transfer_type_to_handler
self.handlers.add(handler)
self.transfer_type_to_handler[transfer_type] = handler
def transfer_async(self, job_id: int, spec: TransferSpec) -> bool:
"""
Initiates an asynchronous transfer of KV data.
Args:
job_id: a unique ID that will be used when notifying back on
transfer completion.
spec: the (src, dst) spec of the KV data transfer.
Returns:
True if transfer was submitted successfully.
"""
src, dst = spec
transfer_type = (src.medium(), dst.medium())
handler = self.transfer_type_to_handler.get(transfer_type)
assert handler is not None
try:
success = handler.transfer_async(job_id, spec)
except Exception as e:
logger.warning(
"Exception in %r transfer %d: %r",
transfer_type,
job_id,
e,
exc_info=True,
)
return False
if not success:
logger.warning("Failed to submit %r transfer %d", transfer_type, job_id)
else:
logger.debug("Submitted %r transfer %d: %r", transfer_type, job_id, spec)
return success
def get_finished(self) -> list[TransferResult]:
"""
Get transfers finished since last call.
Returns:
A list of TransferResults
"""
finished = []
for handler in self.handlers:
finished.extend(handler.get_finished())
return finished
def wait(self, job_ids: set[int]) -> None:
"""
Wait for jobs to finish (blocking).
Args:
job_ids: The set of job IDs to wait for.
"""
for handler in self.handlers:
handler.wait(job_ids)
def shutdown(self) -> None:
for handler in self.handlers:
handler.shutdown()