[KV Offload] Replace OffloadingHandler with OffloadingWorker (#45053)
Signed-off-by: Martin Hickey <martin.hickey@ie.ibm.com>
This commit is contained in:
@@ -98,7 +98,7 @@ def _make_worker(kv_cache_config: KVCacheConfig):
|
||||
spec = MagicMock(spec=OffloadingSpec)
|
||||
spec.kv_cache_config = kv_cache_config
|
||||
spec.vllm_config = MagicMock()
|
||||
spec.get_handlers.return_value = iter([])
|
||||
spec.get_worker.return_value = MagicMock()
|
||||
|
||||
worker = OffloadingConnectorWorker(spec=spec)
|
||||
worker.worker = MagicMock()
|
||||
@@ -279,7 +279,7 @@ def test_register_kv_caches(backend):
|
||||
worker, spec = _make_worker(kv_cache_config)
|
||||
worker.register_kv_caches(kv_caches)
|
||||
|
||||
canonical = spec.get_handlers.call_args[0][0]
|
||||
canonical = spec.get_worker.call_args[0][0]
|
||||
assert isinstance(canonical, CanonicalKVCaches)
|
||||
|
||||
# -- Expected block tensors ----------------------------------------------
|
||||
@@ -422,7 +422,7 @@ def test_register_kv_caches_uniform_type(backend):
|
||||
worker, spec = _make_worker(kv_cache_config)
|
||||
worker.register_kv_caches(kv_caches)
|
||||
|
||||
canonical = spec.get_handlers.call_args[0][0]
|
||||
canonical = spec.get_worker.call_args[0][0]
|
||||
assert isinstance(canonical, CanonicalKVCaches)
|
||||
|
||||
for block_tensor in canonical.tensors:
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from collections.abc import Callable, Iterable, Iterator
|
||||
from collections.abc import Callable, Iterable
|
||||
from dataclasses import dataclass
|
||||
from typing import Any
|
||||
from unittest.mock import MagicMock
|
||||
@@ -43,19 +43,17 @@ from vllm.v1.kv_cache_interface import (
|
||||
KVCacheGroupSpec,
|
||||
)
|
||||
from vllm.v1.kv_offload.base import (
|
||||
CanonicalKVCaches,
|
||||
GPULoadStoreSpec,
|
||||
LoadStoreSpec,
|
||||
OffloadingManager,
|
||||
OffloadingSpec,
|
||||
OffloadingWorker,
|
||||
OffloadKey,
|
||||
PrepareStoreOutput,
|
||||
RequestOffloadingContext,
|
||||
make_offload_key,
|
||||
)
|
||||
from vllm.v1.kv_offload.worker.worker import (
|
||||
OffloadingHandler,
|
||||
TransferResult,
|
||||
TransferSpec,
|
||||
make_offload_key,
|
||||
)
|
||||
from vllm.v1.request import Request
|
||||
from vllm.v1.structured_output import StructuredOutputManager
|
||||
@@ -81,9 +79,9 @@ class MockLoadStoreSpec(LoadStoreSpec):
|
||||
return repr(self.offload_keys)
|
||||
|
||||
|
||||
class MockOffloadingHandler(OffloadingHandler):
|
||||
class MockOffloadingWorker(OffloadingWorker):
|
||||
def __init__(self):
|
||||
self.transfer_specs: dict[int, TransferSpec] = {}
|
||||
self.transfer_specs: dict[int, tuple[LoadStoreSpec, LoadStoreSpec]] = {}
|
||||
self.completed_transfers: list[TransferResult] = []
|
||||
self.waiting_jobs: set[int] = set()
|
||||
self.completed_jobs: list[int] = []
|
||||
@@ -94,8 +92,17 @@ class MockOffloadingHandler(OffloadingHandler):
|
||||
self.completed_transfers = []
|
||||
return finished
|
||||
|
||||
def transfer_async(self, job_id: int, spec: TransferSpec) -> bool:
|
||||
self.transfer_specs[job_id] = spec
|
||||
def submit_store(
|
||||
self, job_id: int, src_spec: LoadStoreSpec, dst_spec: LoadStoreSpec
|
||||
) -> bool: # type: ignore[override]
|
||||
self.transfer_specs[job_id] = (src_spec, dst_spec)
|
||||
self.waiting_jobs.add(job_id)
|
||||
return True
|
||||
|
||||
def submit_load(
|
||||
self, job_id: int, src_spec: LoadStoreSpec, dst_spec: LoadStoreSpec
|
||||
) -> bool: # type: ignore[override]
|
||||
self.transfer_specs[job_id] = (src_spec, dst_spec)
|
||||
self.waiting_jobs.add(job_id)
|
||||
return True
|
||||
|
||||
@@ -109,7 +116,6 @@ class MockOffloadingHandler(OffloadingHandler):
|
||||
success=True,
|
||||
transfer_size=None,
|
||||
transfer_time=None,
|
||||
transfer_type=None,
|
||||
)
|
||||
self.completed_transfers.append(result)
|
||||
|
||||
@@ -127,21 +133,18 @@ class MockOffloadingSpec(OffloadingSpec):
|
||||
self.manager.prepare_load = lambda keys, req_context: MockLoadStoreSpec(keys)
|
||||
self.manager.lookup.return_value = False
|
||||
self.manager.on_new_request.return_value = RequestOffloadingContext()
|
||||
self.handler = MockOffloadingHandler()
|
||||
self.handler = MockOffloadingWorker()
|
||||
|
||||
def get_manager(self) -> OffloadingManager:
|
||||
return self.manager
|
||||
|
||||
def get_handlers(
|
||||
self, _
|
||||
) -> Iterator[tuple[type[LoadStoreSpec], type[LoadStoreSpec], OffloadingHandler]]:
|
||||
yield GPULoadStoreSpec, MockLoadStoreSpec, self.handler
|
||||
yield MockLoadStoreSpec, GPULoadStoreSpec, self.handler
|
||||
def get_worker(self, _: CanonicalKVCaches) -> OffloadingWorker:
|
||||
return self.handler
|
||||
|
||||
def complete_transfers(self):
|
||||
self.handler.complete_jobs(self.handler.waiting_jobs.copy())
|
||||
|
||||
def get_completed_transfers(self) -> list[TransferSpec]:
|
||||
def get_completed_transfers(self) -> list[tuple[LoadStoreSpec, LoadStoreSpec]]:
|
||||
specs = [
|
||||
self.handler.transfer_specs[job_id]
|
||||
for job_id in self.handler.completed_jobs
|
||||
@@ -149,7 +152,7 @@ class MockOffloadingSpec(OffloadingSpec):
|
||||
self.handler.completed_jobs.clear()
|
||||
return specs
|
||||
|
||||
def get_flushed_transfers(self):
|
||||
def get_flushed_transfers(self) -> list[tuple[LoadStoreSpec, LoadStoreSpec]]:
|
||||
specs = [
|
||||
self.handler.transfer_specs[job_id] for job_id in self.handler.flushed_jobs
|
||||
]
|
||||
@@ -362,8 +365,7 @@ class RequestRunner:
|
||||
self.scheduler.add_request(req)
|
||||
|
||||
def _parse_transfers(self):
|
||||
for transfer_spec in self.offloading_spec.get_flushed_transfers():
|
||||
src_spec, dst_spec = transfer_spec
|
||||
for src_spec, dst_spec in self.offloading_spec.get_flushed_transfers():
|
||||
if isinstance(src_spec, GPULoadStoreSpec):
|
||||
# store flush
|
||||
for block_id in src_spec.block_ids:
|
||||
@@ -375,9 +377,7 @@ class RequestRunner:
|
||||
|
||||
block_size_factor = self.block_size_factor
|
||||
|
||||
for transfer_spec in self.offloading_spec.get_completed_transfers():
|
||||
src_spec, dst_spec = transfer_spec
|
||||
|
||||
for src_spec, dst_spec in self.offloading_spec.get_completed_transfers():
|
||||
if isinstance(src_spec, GPULoadStoreSpec):
|
||||
store = True
|
||||
gpu_spec = src_spec
|
||||
|
||||
@@ -17,7 +17,7 @@ from vllm.v1.kv_offload.base import (
|
||||
GPULoadStoreSpec,
|
||||
)
|
||||
from vllm.v1.kv_offload.cpu.common import CPULoadStoreSpec
|
||||
from vllm.v1.kv_offload.cpu.gpu_worker import CpuGpuOffloadingHandlers
|
||||
from vllm.v1.kv_offload.cpu.gpu_worker import CPUOffloadingWorker
|
||||
from vllm.v1.kv_offload.cpu.shared_offload_region import SharedOffloadRegion
|
||||
|
||||
NUM_GPU_BLOCKS = [64]
|
||||
@@ -103,7 +103,7 @@ def test_transfer(
|
||||
cpu_page_size=cpu_page_size,
|
||||
)
|
||||
|
||||
handlers = CpuGpuOffloadingHandlers(
|
||||
worker = CPUOffloadingWorker(
|
||||
kv_caches=kv_caches,
|
||||
block_size_factor=block_size_factor,
|
||||
num_cpu_blocks=num_cpu_blocks,
|
||||
@@ -130,7 +130,7 @@ def test_transfer(
|
||||
|
||||
# set transfer direction
|
||||
if gpu_to_cpu:
|
||||
handler = handlers.gpu_to_cpu_handler
|
||||
handler = worker._store_handler
|
||||
src_spec = GPULoadStoreSpec(
|
||||
gpu_blocks, group_sizes=(len(gpu_blocks),), block_indices=(blocks_to_skip,)
|
||||
)
|
||||
@@ -138,7 +138,7 @@ def test_transfer(
|
||||
dst_to_src = dict(zip(cpu_blocks_expanded, gpu_blocks))
|
||||
num_dst_sub_blocks = num_gpu_blocks
|
||||
else:
|
||||
handler = handlers.cpu_to_gpu_handler
|
||||
handler = worker._load_handler
|
||||
src_spec = CPULoadStoreSpec(cpu_blocks)
|
||||
dst_spec = GPULoadStoreSpec(
|
||||
gpu_blocks, group_sizes=(len(gpu_blocks),), block_indices=(blocks_to_skip,)
|
||||
@@ -156,23 +156,21 @@ def test_transfer(
|
||||
orig_src_tensors = [x.clone() for x in handler.src_tensors]
|
||||
orig_dst_tensors = [x.clone() for x in handler.dst_tensors]
|
||||
|
||||
# call transfer function
|
||||
# call transfer function via public API
|
||||
start_time = time.time()
|
||||
assert handler.transfer_async(1, (src_spec, dst_spec))
|
||||
if gpu_to_cpu:
|
||||
assert worker.submit_store(1, src_spec, dst_spec)
|
||||
else:
|
||||
assert worker.submit_load(1, src_spec, dst_spec)
|
||||
assert {x.job_id for x in handler._transfers} == {1}
|
||||
|
||||
# wait for transfer to complete
|
||||
end_time = time.time() + 10
|
||||
while time.time() < end_time:
|
||||
finished = handler.get_finished()
|
||||
finished = worker.get_finished()
|
||||
if finished:
|
||||
assert finished[0].job_id == 1
|
||||
assert finished[0].success
|
||||
assert (
|
||||
finished[0].transfer_type == ("GPU", "CPU")
|
||||
if gpu_to_cpu
|
||||
else ("CPU", "GPU")
|
||||
)
|
||||
assert finished[0].transfer_size == (
|
||||
len(gpu_blocks)
|
||||
* sum([x.page_size_bytes for x in handler.kv_cache_groups_data_refs[0]])
|
||||
@@ -208,8 +206,7 @@ def test_transfer(
|
||||
del orig_tensor, tensor, src_tensor, dst_tensor, orig_dst_tensor
|
||||
del src_view, dst_view, orig_dst_view, expected
|
||||
|
||||
handlers.cpu_to_gpu_handler.shutdown()
|
||||
handlers.gpu_to_cpu_handler.shutdown()
|
||||
worker.shutdown()
|
||||
if mmap_region:
|
||||
mmap_region.cleanup()
|
||||
|
||||
@@ -276,7 +273,7 @@ def test_transfer_multi_group(
|
||||
tensors=kv_cache_tensors, group_data_refs=kv_cache_groups_data_refs
|
||||
)
|
||||
|
||||
handlers = CpuGpuOffloadingHandlers(
|
||||
worker = CPUOffloadingWorker(
|
||||
kv_caches=canonical_kv_caches,
|
||||
block_size_factor=block_size_factor,
|
||||
num_cpu_blocks=num_cpu_blocks,
|
||||
@@ -338,7 +335,7 @@ def test_transfer_multi_group(
|
||||
block_indices: list[int] = [0, 0, sub_blocks_to_skip]
|
||||
|
||||
if gpu_to_cpu:
|
||||
handler = handlers.gpu_to_cpu_handler
|
||||
handler = worker._store_handler
|
||||
src_spec = GPULoadStoreSpec(
|
||||
gpu_blocks, group_sizes=group_sizes, block_indices=block_indices
|
||||
)
|
||||
@@ -352,7 +349,7 @@ def test_transfer_multi_group(
|
||||
]
|
||||
num_dst_sub_blocks = num_cpu_blocks * block_size_factor
|
||||
else:
|
||||
handler = handlers.cpu_to_gpu_handler
|
||||
handler = worker._load_handler
|
||||
src_spec = CPULoadStoreSpec(cpu_blocks)
|
||||
dst_spec = GPULoadStoreSpec(
|
||||
gpu_blocks, group_sizes=group_sizes, block_indices=block_indices
|
||||
@@ -375,12 +372,15 @@ def test_transfer_multi_group(
|
||||
orig_src_tensors = [x.clone() for x in handler.src_tensors]
|
||||
orig_dst_tensors = [x.clone() for x in handler.dst_tensors]
|
||||
|
||||
assert handler.transfer_async(1, (src_spec, dst_spec))
|
||||
if gpu_to_cpu:
|
||||
assert worker.submit_store(1, src_spec, dst_spec)
|
||||
else:
|
||||
assert worker.submit_load(1, src_spec, dst_spec)
|
||||
assert {x.job_id for x in handler._transfers} == {1}
|
||||
|
||||
end_time = time.time() + 10
|
||||
while time.time() < end_time:
|
||||
finished = handler.get_finished()
|
||||
finished = worker.get_finished()
|
||||
if finished:
|
||||
assert finished[0].job_id == 1
|
||||
assert finished[0].success
|
||||
@@ -418,5 +418,4 @@ def test_transfer_multi_group(
|
||||
dst_view[dst_sub_block].cpu(), expected.cpu()
|
||||
)
|
||||
|
||||
handlers.cpu_to_gpu_handler.shutdown()
|
||||
handlers.gpu_to_cpu_handler.shutdown()
|
||||
worker.shutdown()
|
||||
|
||||
@@ -1,165 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from vllm.v1.kv_offload.base import LoadStoreSpec
|
||||
from vllm.v1.kv_offload.worker.worker import (
|
||||
OffloadingHandler,
|
||||
OffloadingWorker,
|
||||
TransferResult,
|
||||
TransferSpec,
|
||||
)
|
||||
|
||||
|
||||
class LoadStoreSpec1(LoadStoreSpec):
|
||||
def __init__(
|
||||
self,
|
||||
submit_success: bool = True,
|
||||
async_success: bool = True,
|
||||
exception: bool = False,
|
||||
):
|
||||
self.finished = False
|
||||
self.submit_success = submit_success
|
||||
self.async_success = async_success
|
||||
self.exception = exception
|
||||
|
||||
@staticmethod
|
||||
def medium() -> str:
|
||||
return "1"
|
||||
|
||||
def __repr__(self):
|
||||
return f"{self.medium()}: {id(self)}"
|
||||
|
||||
|
||||
class LoadStoreSpec2(LoadStoreSpec):
|
||||
@staticmethod
|
||||
def medium() -> str:
|
||||
return "2"
|
||||
|
||||
def __repr__(self):
|
||||
return f"{self.medium()}: {id(self)}"
|
||||
|
||||
|
||||
class OffloadingHandler1To2(OffloadingHandler):
|
||||
def __init__(self):
|
||||
self.transfers: dict[int, LoadStoreSpec1] = {}
|
||||
|
||||
def transfer_async(self, job_id: int, spec: TransferSpec) -> bool:
|
||||
src, dst = spec
|
||||
assert isinstance(src, LoadStoreSpec1)
|
||||
assert isinstance(dst, LoadStoreSpec2)
|
||||
|
||||
if src.exception:
|
||||
raise Exception("An expected exception. Don't worry!")
|
||||
if not src.submit_success:
|
||||
return False
|
||||
|
||||
self.transfers[job_id] = src
|
||||
return True
|
||||
|
||||
def get_finished(self) -> list[TransferResult]:
|
||||
finished = []
|
||||
for job_id, spec in list(self.transfers.items()):
|
||||
if spec.finished:
|
||||
finished.append((job_id, spec.async_success))
|
||||
del self.transfers[job_id]
|
||||
return finished
|
||||
|
||||
def wait(self, job_ids: set[int]) -> None:
|
||||
for job_id in job_ids:
|
||||
spec = self.transfers.get(job_id)
|
||||
if spec:
|
||||
assert spec.finished
|
||||
|
||||
|
||||
class OffloadingHandler2To1(OffloadingHandler):
|
||||
def __init__(self):
|
||||
self.transfers: dict[int, LoadStoreSpec1] = {}
|
||||
|
||||
def transfer_async(self, job_id: int, spec: TransferSpec) -> bool:
|
||||
src, dst = spec
|
||||
assert isinstance(src, LoadStoreSpec2)
|
||||
assert isinstance(dst, LoadStoreSpec1)
|
||||
|
||||
self.transfers[job_id] = dst
|
||||
return True
|
||||
|
||||
def get_finished(self) -> list[TransferResult]:
|
||||
finished = []
|
||||
for job_id, spec in list(self.transfers.items()):
|
||||
if spec.finished:
|
||||
finished.append((job_id, spec.async_success))
|
||||
del self.transfers[job_id]
|
||||
return finished
|
||||
|
||||
def wait(self, job_ids: set[int]) -> None:
|
||||
for job_id in job_ids:
|
||||
spec = self.transfers.get(job_id)
|
||||
if spec:
|
||||
assert spec.finished
|
||||
|
||||
|
||||
def test_offloading_worker():
|
||||
"""
|
||||
Tests OffloadingWorker with 2 handlers.
|
||||
One handler performs 1->2 transfers, and the other handles 2->1.
|
||||
"""
|
||||
worker = OffloadingWorker()
|
||||
handler1to2 = OffloadingHandler1To2()
|
||||
handler2to1 = OffloadingHandler2To1()
|
||||
worker.register_handler(LoadStoreSpec1, LoadStoreSpec2, handler1to2)
|
||||
worker.register_handler(LoadStoreSpec2, LoadStoreSpec1, handler2to1)
|
||||
|
||||
# 1st transfer 1->2 (exception)
|
||||
src1 = LoadStoreSpec1(exception=True)
|
||||
dst1 = LoadStoreSpec2()
|
||||
assert not worker.transfer_async(1, (src1, dst1))
|
||||
|
||||
# 2ed transfer 1->2 (failure to submit)
|
||||
src2 = LoadStoreSpec1(submit_success=False)
|
||||
dst2 = LoadStoreSpec2()
|
||||
assert not worker.transfer_async(2, (src2, dst2))
|
||||
|
||||
# 3rd transfer 1->2 (failure)
|
||||
src3 = LoadStoreSpec1(async_success=False)
|
||||
dst3 = LoadStoreSpec2()
|
||||
assert worker.transfer_async(3, (src3, dst3))
|
||||
|
||||
# 4th transfer 1->2 (success)
|
||||
src4 = LoadStoreSpec1()
|
||||
dst4 = LoadStoreSpec2()
|
||||
worker.transfer_async(4, (src4, dst4))
|
||||
assert set(handler1to2.transfers.keys()) == {3, 4}
|
||||
|
||||
# 5th transfer 2->1
|
||||
src5 = LoadStoreSpec2()
|
||||
dst5 = LoadStoreSpec1()
|
||||
worker.transfer_async(5, (src5, dst5))
|
||||
assert set(handler2to1.transfers.keys()) == {5}
|
||||
|
||||
# no transfer completed yet
|
||||
assert worker.get_finished() == []
|
||||
|
||||
# complete 3rd, 4th
|
||||
src3.finished = True
|
||||
src4.finished = True
|
||||
|
||||
# 6th transfer 1->2
|
||||
src6 = LoadStoreSpec1()
|
||||
dst6 = LoadStoreSpec2()
|
||||
worker.transfer_async(6, (src6, dst6))
|
||||
|
||||
# 7th transfer 2->1
|
||||
src7 = LoadStoreSpec2()
|
||||
dst7 = LoadStoreSpec1()
|
||||
worker.transfer_async(7, (src7, dst7))
|
||||
|
||||
# 6th and 7th transfers started
|
||||
assert 6 in handler1to2.transfers
|
||||
assert 7 in handler2to1.transfers
|
||||
|
||||
# verify result of 3rd and 4th transfers
|
||||
assert sorted(worker.get_finished()) == [(3, False), (4, True)]
|
||||
|
||||
# complete 6th and 7th transfers
|
||||
src6.finished = True
|
||||
dst7.finished = True
|
||||
assert sorted(worker.get_finished()) == [(6, True), (7, True)]
|
||||
@@ -6,7 +6,7 @@ from vllm.distributed.kv_transfer.kv_connector.v1.base import (
|
||||
KVConnectorMetadata,
|
||||
KVConnectorWorkerMetadata,
|
||||
)
|
||||
from vllm.v1.kv_offload.worker.worker import TransferSpec
|
||||
from vllm.v1.kv_offload.base import LoadStoreSpec
|
||||
|
||||
ReqId = str
|
||||
|
||||
@@ -60,7 +60,8 @@ class TransferJob:
|
||||
"""
|
||||
|
||||
req_id: ReqId
|
||||
transfer_spec: TransferSpec
|
||||
src_spec: LoadStoreSpec
|
||||
dst_spec: LoadStoreSpec
|
||||
|
||||
|
||||
@dataclass
|
||||
|
||||
@@ -762,7 +762,8 @@ class OffloadingConnectorScheduler:
|
||||
load_job_id = self._generate_job_id()
|
||||
self._current_batch_load_jobs[load_job_id] = TransferJob(
|
||||
req_id=request.request_id,
|
||||
transfer_spec=(src_spec, dst_spec),
|
||||
src_spec=src_spec,
|
||||
dst_spec=dst_spec,
|
||||
)
|
||||
# a load can only be issued when no other jobs are pending.
|
||||
assert not req_status.transfer_jobs
|
||||
@@ -998,7 +999,7 @@ class OffloadingConnectorScheduler:
|
||||
)
|
||||
|
||||
store_jobs[job_id] = TransferJob(
|
||||
req_id=req_id, transfer_spec=(src_spec, dst_spec)
|
||||
req_id=req_id, src_spec=src_spec, dst_spec=dst_spec
|
||||
)
|
||||
|
||||
logger.debug(
|
||||
|
||||
@@ -21,11 +21,10 @@ from vllm.v1.kv_offload.base import (
|
||||
CanonicalKVCacheRef,
|
||||
CanonicalKVCaches,
|
||||
CanonicalKVCacheTensor,
|
||||
GPULoadStoreSpec,
|
||||
LoadStoreSpec,
|
||||
OffloadingSpec,
|
||||
)
|
||||
from vllm.v1.kv_offload.worker.worker import (
|
||||
OffloadingWorker,
|
||||
TransferSpec,
|
||||
)
|
||||
|
||||
logger = init_logger(__name__)
|
||||
@@ -36,16 +35,17 @@ class OffloadingConnectorWorker:
|
||||
|
||||
def __init__(self, spec: OffloadingSpec):
|
||||
self.spec = spec
|
||||
self.worker = OffloadingWorker()
|
||||
self.worker: OffloadingWorker | None = None
|
||||
|
||||
# job_id -> req_id for in-flight loads.
|
||||
self._load_jobs: dict[int, ReqId] = {}
|
||||
self._unsubmitted_store_jobs: list[tuple[int, TransferSpec]] = []
|
||||
self._unsubmitted_store_jobs: list[
|
||||
tuple[int, GPULoadStoreSpec, LoadStoreSpec]
|
||||
] = []
|
||||
self._connector_worker_meta = OffloadingWorkerMetadata()
|
||||
|
||||
def _register_handlers(self, kv_caches: CanonicalKVCaches):
|
||||
for src_cls, dst_cls, handler in self.spec.get_handlers(kv_caches):
|
||||
self.worker.register_handler(src_cls, dst_cls, handler)
|
||||
def _init_worker(self, kv_caches: CanonicalKVCaches) -> None:
|
||||
self.worker = self.spec.get_worker(kv_caches)
|
||||
|
||||
def register_kv_caches(
|
||||
self, kv_caches: dict[str, torch.Tensor | list[torch.Tensor]]
|
||||
@@ -138,7 +138,7 @@ class OffloadingConnectorWorker:
|
||||
(block_stride, 1),
|
||||
storage_offset=0,
|
||||
)
|
||||
self._register_handlers(
|
||||
self._init_worker(
|
||||
CanonicalKVCaches(
|
||||
[CanonicalKVCacheTensor(packed_tensor, block_stride)],
|
||||
[
|
||||
@@ -204,7 +204,7 @@ class OffloadingConnectorWorker:
|
||||
group_data_refs=group_data_refs,
|
||||
)
|
||||
|
||||
self._register_handlers(canonical_kv_caches)
|
||||
self._init_worker(canonical_kv_caches)
|
||||
|
||||
def register_cross_layers_kv_cache(
|
||||
self, kv_cache: torch.Tensor, attn_backend: type[AttentionBackend]
|
||||
@@ -251,11 +251,12 @@ class OffloadingConnectorWorker:
|
||||
tensors=[kv_cache_tensor], group_data_refs=[[kv_cache_data_ref]]
|
||||
)
|
||||
|
||||
self._register_handlers(canonical_kv_caches)
|
||||
self._init_worker(canonical_kv_caches)
|
||||
|
||||
def handle_preemptions(self, kv_connector_metadata: OffloadingConnectorMetadata):
|
||||
for job_id, transfer_spec in self._unsubmitted_store_jobs:
|
||||
success = self.worker.transfer_async(job_id, transfer_spec)
|
||||
assert self.worker is not None
|
||||
for job_id, src_spec, dst_spec in self._unsubmitted_store_jobs:
|
||||
success = self.worker.submit_store(job_id, src_spec, dst_spec)
|
||||
assert success
|
||||
self._unsubmitted_store_jobs.clear()
|
||||
|
||||
@@ -263,14 +264,16 @@ class OffloadingConnectorWorker:
|
||||
self.worker.wait(kv_connector_metadata.jobs_to_flush)
|
||||
|
||||
def start_kv_transfers(self, metadata: OffloadingConnectorMetadata):
|
||||
for job_id, transfer_spec in self._unsubmitted_store_jobs:
|
||||
success = self.worker.transfer_async(job_id, transfer_spec)
|
||||
assert self.worker is not None
|
||||
for job_id, src_spec, dst_spec in self._unsubmitted_store_jobs:
|
||||
success = self.worker.submit_store(job_id, src_spec, dst_spec)
|
||||
assert success
|
||||
self._unsubmitted_store_jobs.clear()
|
||||
|
||||
for job_id, entry in metadata.load_jobs.items():
|
||||
self._load_jobs[job_id] = entry.req_id
|
||||
success = self.worker.transfer_async(job_id, entry.transfer_spec)
|
||||
assert isinstance(entry.dst_spec, GPULoadStoreSpec)
|
||||
success = self.worker.submit_load(job_id, entry.src_spec, entry.dst_spec)
|
||||
assert success
|
||||
|
||||
def prepare_store_kv(self, metadata: OffloadingConnectorMetadata):
|
||||
@@ -278,7 +281,10 @@ class OffloadingConnectorWorker:
|
||||
# NOTE(orozery): defer the store to the beginning of the next
|
||||
# engine step, so that offloading starts AFTER transfers related
|
||||
# to token sampling, thereby avoiding delays to token generation.
|
||||
self._unsubmitted_store_jobs.append((job_id, entry.transfer_spec))
|
||||
assert isinstance(entry.src_spec, GPULoadStoreSpec)
|
||||
self._unsubmitted_store_jobs.append(
|
||||
(job_id, entry.src_spec, entry.dst_spec)
|
||||
)
|
||||
|
||||
def get_finished(self, finished_req_ids: set[str]) -> tuple[set[str], set[str]]:
|
||||
"""
|
||||
@@ -290,6 +296,7 @@ class OffloadingConnectorWorker:
|
||||
finished_recving so the base scheduler can resume requests
|
||||
blocked on remote KV (and free aborted-during-load reqs).
|
||||
"""
|
||||
assert self.worker is not None
|
||||
finished_recving: set[str] = set()
|
||||
for transfer_result in self.worker.get_finished():
|
||||
# we currently do not support job failures
|
||||
@@ -328,4 +335,5 @@ class OffloadingConnectorWorker:
|
||||
self._unsubmitted_store_jobs.clear()
|
||||
self._load_jobs.clear()
|
||||
self._connector_worker_meta = OffloadingWorkerMetadata()
|
||||
self.worker.shutdown()
|
||||
if self.worker is not None:
|
||||
self.worker.shutdown()
|
||||
|
||||
@@ -5,7 +5,7 @@ Core abstractions for KV cache offloading in vLLM v1.
|
||||
"""
|
||||
|
||||
from abc import ABC, abstractmethod
|
||||
from collections.abc import Collection, Iterable, Iterator, Sequence
|
||||
from collections.abc import Collection, Iterable, Sequence
|
||||
from dataclasses import dataclass
|
||||
from enum import Enum
|
||||
from typing import TYPE_CHECKING, Any, NewType
|
||||
@@ -23,7 +23,6 @@ if TYPE_CHECKING:
|
||||
OffloadingConnectorStats,
|
||||
)
|
||||
from vllm.v1.kv_cache_interface import KVCacheConfig
|
||||
from vllm.v1.kv_offload.worker.worker import OffloadingHandler
|
||||
|
||||
# `OffloadKey` identifies an offloaded block. It combines a block hash with
|
||||
# its KV cache group index, encoded as raw bytes to avoid tuple GC overhead.
|
||||
@@ -432,6 +431,41 @@ class CanonicalKVCaches:
|
||||
group_data_refs: list[list[CanonicalKVCacheRef]]
|
||||
|
||||
|
||||
@dataclass
|
||||
class TransferResult:
|
||||
job_id: int
|
||||
success: bool
|
||||
transfer_size: int | None = None
|
||||
transfer_time: float | None = None
|
||||
|
||||
|
||||
class OffloadingWorker(ABC):
|
||||
"""Runs in the worker process. Performs async KV transfers for ONE
|
||||
offloaded medium (e.g. CPU). Direction is explicit via submit_store /
|
||||
submit_load, so there is no (src_medium, dst_medium) routing."""
|
||||
|
||||
@abstractmethod
|
||||
def submit_store(
|
||||
self, job_id: int, src_spec: GPULoadStoreSpec, dst_spec: LoadStoreSpec
|
||||
) -> bool:
|
||||
"""Async GPU -> offloaded medium."""
|
||||
|
||||
@abstractmethod
|
||||
def submit_load(
|
||||
self, job_id: int, src_spec: LoadStoreSpec, dst_spec: GPULoadStoreSpec
|
||||
) -> bool:
|
||||
"""Async offloaded medium -> GPU."""
|
||||
|
||||
@abstractmethod
|
||||
def get_finished(self) -> list[TransferResult]: ...
|
||||
|
||||
@abstractmethod
|
||||
def wait(self, job_ids: set[int]) -> None: ...
|
||||
|
||||
def shutdown(self) -> None:
|
||||
return
|
||||
|
||||
|
||||
class OffloadingSpec(ABC):
|
||||
"""Spec for an offloading connector"""
|
||||
|
||||
@@ -524,16 +558,14 @@ class OffloadingSpec(ABC):
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
def get_handlers(
|
||||
self, kv_caches: CanonicalKVCaches
|
||||
) -> Iterator[tuple[type[LoadStoreSpec], type[LoadStoreSpec], "OffloadingHandler"]]:
|
||||
def get_worker(self, kv_caches: CanonicalKVCaches) -> OffloadingWorker:
|
||||
"""
|
||||
Get offloading handlers along with their respective src and dst types.
|
||||
Get an OffloadingWorker that handles async KV transfers for this spec.
|
||||
|
||||
Args:
|
||||
kv_caches: Canonicalized KV caches.
|
||||
|
||||
Yields:
|
||||
Tuples of (src_type, dst_type, offloading_handler).
|
||||
Returns:
|
||||
An OffloadingWorker instance for this medium.
|
||||
"""
|
||||
pass
|
||||
|
||||
@@ -7,7 +7,6 @@ from dataclasses import dataclass
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
from typing_extensions import override
|
||||
|
||||
from vllm import _custom_ops as ops
|
||||
from vllm.logger import init_logger
|
||||
@@ -20,17 +19,15 @@ from vllm.v1.kv_offload.base import (
|
||||
CanonicalKVCacheRef,
|
||||
CanonicalKVCaches,
|
||||
GPULoadStoreSpec,
|
||||
LoadStoreSpec,
|
||||
OffloadingWorker,
|
||||
TransferResult,
|
||||
)
|
||||
from vllm.v1.kv_offload.cpu.shared_offload_region import SharedOffloadRegion
|
||||
from vllm.v1.kv_offload.cpu.swap_blocks_triton import (
|
||||
THRESHOLD_BYTES,
|
||||
swap_blocks_batch,
|
||||
)
|
||||
from vllm.v1.kv_offload.worker.worker import (
|
||||
OffloadingHandler,
|
||||
TransferResult,
|
||||
TransferSpec,
|
||||
)
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
@@ -166,10 +163,9 @@ def _new_descriptor_buffers(
|
||||
)
|
||||
|
||||
|
||||
class SingleDirectionOffloadingHandler(OffloadingHandler):
|
||||
class SingleDirectionOffloadingHandler:
|
||||
"""
|
||||
SingleDirectionOffloadingHandler handles transfers for a single direction,
|
||||
either CPU->GPU or GPU->CPU.
|
||||
Handles transfers for a single direction, either CPU->GPU or GPU->CPU.
|
||||
Transfers are guaranteed to be executed in order of their submission.
|
||||
Each transfer uses a unique CUDA stream, and its stream will start
|
||||
executing only after the streams of previous transfers have finished.
|
||||
@@ -228,7 +224,6 @@ class SingleDirectionOffloadingHandler(OffloadingHandler):
|
||||
self.src_block_size_factor = 1 if self.gpu_to_cpu else block_size_factor
|
||||
self.dst_block_size_factor = block_size_factor if self.gpu_to_cpu else 1
|
||||
|
||||
self.transfer_type = ("GPU", "CPU") if self.gpu_to_cpu else ("CPU", "GPU")
|
||||
# mmap_region to clean up on shutdown (gpu_to_cpu handler owns it)
|
||||
self._mmap_region = mmap_region
|
||||
# job_id -> event
|
||||
@@ -242,9 +237,9 @@ class SingleDirectionOffloadingHandler(OffloadingHandler):
|
||||
# list of pinned descriptor buffer sets available for re-use
|
||||
self._buffer_pool: list[tuple[torch.Tensor, torch.Tensor, torch.Tensor]] = []
|
||||
|
||||
@override
|
||||
def transfer_async(self, job_id: int, transfer_spec: TransferSpec) -> bool:
|
||||
src_spec, dst_spec = transfer_spec
|
||||
def transfer_async(
|
||||
self, job_id: int, src_spec: LoadStoreSpec, dst_spec: LoadStoreSpec
|
||||
) -> bool:
|
||||
assert isinstance(src_spec, BlockIDsLoadStoreSpec)
|
||||
assert isinstance(dst_spec, BlockIDsLoadStoreSpec)
|
||||
|
||||
@@ -425,7 +420,6 @@ class SingleDirectionOffloadingHandler(OffloadingHandler):
|
||||
# success
|
||||
return True
|
||||
|
||||
@override
|
||||
def get_finished(self) -> list[TransferResult]:
|
||||
results: list[TransferResult] = []
|
||||
while self._transfers and self._transfers[0].end_event.query():
|
||||
@@ -438,7 +432,6 @@ class SingleDirectionOffloadingHandler(OffloadingHandler):
|
||||
success=True,
|
||||
transfer_size=transfer.num_bytes,
|
||||
transfer_time=transfer_time,
|
||||
transfer_type=self.transfer_type,
|
||||
)
|
||||
|
||||
results.append(result)
|
||||
@@ -451,14 +444,12 @@ class SingleDirectionOffloadingHandler(OffloadingHandler):
|
||||
del self._transfer_events[transfer.job_id]
|
||||
return results
|
||||
|
||||
@override
|
||||
def wait(self, job_ids: set[int]):
|
||||
for job_id in job_ids:
|
||||
event = self._transfer_events.get(job_id)
|
||||
if event is not None:
|
||||
event.synchronize()
|
||||
|
||||
@override
|
||||
def shutdown(self) -> None:
|
||||
while self._transfers:
|
||||
transfer = self._transfers.popleft()
|
||||
@@ -474,7 +465,14 @@ class SingleDirectionOffloadingHandler(OffloadingHandler):
|
||||
self._mmap_region = None
|
||||
|
||||
|
||||
class CpuGpuOffloadingHandlers:
|
||||
class CPUOffloadingWorker(OffloadingWorker):
|
||||
"""OffloadingWorker for CPU offloading.
|
||||
|
||||
Composes two SingleDirectionOffloadingHandler instances (one for each
|
||||
direction) and exposes them through the explicit submit_store /
|
||||
submit_load API.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
kv_caches: CanonicalKVCaches,
|
||||
@@ -484,7 +482,6 @@ class CpuGpuOffloadingHandlers:
|
||||
):
|
||||
pin_memory = PIN_MEMORY
|
||||
logger.info("Allocating %d CPU tensors...", len(kv_caches.tensors))
|
||||
self._mmap_region = mmap_region
|
||||
if mmap_region is not None and pin_memory:
|
||||
pin_mmap_region(mmap_region)
|
||||
|
||||
@@ -518,7 +515,7 @@ class CpuGpuOffloadingHandlers:
|
||||
gpu_tensors.append(gpu_tensor)
|
||||
cpu_tensors.append(cpu_tensor)
|
||||
|
||||
self.gpu_to_cpu_handler = SingleDirectionOffloadingHandler(
|
||||
self._store_handler = SingleDirectionOffloadingHandler(
|
||||
gpu_tensors=gpu_tensors,
|
||||
cpu_tensors=cpu_tensors,
|
||||
block_size_factor=block_size_factor,
|
||||
@@ -527,10 +524,33 @@ class CpuGpuOffloadingHandlers:
|
||||
mmap_region=mmap_region,
|
||||
)
|
||||
|
||||
self.cpu_to_gpu_handler = SingleDirectionOffloadingHandler(
|
||||
self._load_handler = SingleDirectionOffloadingHandler(
|
||||
gpu_tensors=gpu_tensors,
|
||||
cpu_tensors=cpu_tensors,
|
||||
block_size_factor=block_size_factor,
|
||||
kv_cache_groups_data_refs=kv_caches.group_data_refs,
|
||||
gpu_to_cpu=False,
|
||||
)
|
||||
|
||||
def submit_store(
|
||||
self, job_id: int, src_spec: GPULoadStoreSpec, dst_spec: LoadStoreSpec
|
||||
) -> bool:
|
||||
"""Async GPU -> CPU."""
|
||||
return self._store_handler.transfer_async(job_id, src_spec, dst_spec)
|
||||
|
||||
def submit_load(
|
||||
self, job_id: int, src_spec: LoadStoreSpec, dst_spec: GPULoadStoreSpec
|
||||
) -> bool:
|
||||
"""Async CPU -> GPU."""
|
||||
return self._load_handler.transfer_async(job_id, src_spec, dst_spec)
|
||||
|
||||
def get_finished(self) -> list[TransferResult]:
|
||||
return self._store_handler.get_finished() + self._load_handler.get_finished()
|
||||
|
||||
def wait(self, job_ids: set[int]) -> None:
|
||||
self._store_handler.wait(job_ids)
|
||||
self._load_handler.wait(job_ids)
|
||||
|
||||
def shutdown(self) -> None:
|
||||
self._store_handler.shutdown()
|
||||
self._load_handler.shutdown()
|
||||
|
||||
@@ -1,6 +1,5 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from collections.abc import Iterator
|
||||
from typing import Any
|
||||
|
||||
from typing_extensions import override
|
||||
@@ -11,21 +10,16 @@ from vllm.utils.math_utils import round_up
|
||||
from vllm.v1.kv_cache_interface import KVCacheConfig
|
||||
from vllm.v1.kv_offload.base import (
|
||||
CanonicalKVCaches,
|
||||
GPULoadStoreSpec,
|
||||
LoadStoreSpec,
|
||||
OffloadingCounterMetadata,
|
||||
OffloadingGaugeMetadata,
|
||||
OffloadingManager,
|
||||
OffloadingMetricMetadata,
|
||||
OffloadingSpec,
|
||||
OffloadingWorker,
|
||||
)
|
||||
from vllm.v1.kv_offload.cpu.common import (
|
||||
CPULoadStoreSpec,
|
||||
CPUOffloadingMetrics,
|
||||
)
|
||||
from vllm.v1.kv_offload.cpu.gpu_worker import CpuGpuOffloadingHandlers
|
||||
from vllm.v1.kv_offload.cpu.common import CPUOffloadingMetrics
|
||||
from vllm.v1.kv_offload.cpu.gpu_worker import CPUOffloadingWorker
|
||||
from vllm.v1.kv_offload.cpu.manager import CPUOffloadingManager
|
||||
from vllm.v1.kv_offload.worker.worker import OffloadingHandler
|
||||
|
||||
|
||||
class CPUOffloadingSpec(OffloadingSpec):
|
||||
@@ -107,7 +101,7 @@ class CPUOffloadingSpec(OffloadingSpec):
|
||||
self._manager: OffloadingManager | None = None
|
||||
|
||||
# worker-side
|
||||
self._handlers: CpuGpuOffloadingHandlers | None = None
|
||||
self._worker: CPUOffloadingWorker | None = None
|
||||
|
||||
self.eviction_policy: str = self.extra_config.get("eviction_policy", "lru")
|
||||
|
||||
@@ -131,25 +125,22 @@ class CPUOffloadingSpec(OffloadingSpec):
|
||||
)
|
||||
return self._manager
|
||||
|
||||
def create_handlers(self, kv_caches: CanonicalKVCaches) -> CpuGpuOffloadingHandlers:
|
||||
return CpuGpuOffloadingHandlers(
|
||||
def create_worker(self, kv_caches: CanonicalKVCaches) -> CPUOffloadingWorker:
|
||||
return CPUOffloadingWorker(
|
||||
kv_caches=kv_caches,
|
||||
block_size_factor=self.block_size_factor,
|
||||
num_cpu_blocks=self.num_blocks,
|
||||
)
|
||||
|
||||
@override
|
||||
def get_handlers(
|
||||
self, kv_caches: CanonicalKVCaches
|
||||
) -> Iterator[tuple[type[LoadStoreSpec], type[LoadStoreSpec], OffloadingHandler]]:
|
||||
if not self._handlers:
|
||||
def get_worker(self, kv_caches: CanonicalKVCaches) -> OffloadingWorker:
|
||||
if not self._worker:
|
||||
if not (current_platform.is_cuda_alike() or current_platform.is_xpu()):
|
||||
raise Exception(
|
||||
"CPU Offloading is currently only supported on CUDA-alike "
|
||||
"and XPU GPUs"
|
||||
)
|
||||
self._handlers = self.create_handlers(kv_caches)
|
||||
self._worker = self.create_worker(kv_caches)
|
||||
|
||||
assert self._handlers is not None
|
||||
yield GPULoadStoreSpec, CPULoadStoreSpec, self._handlers.gpu_to_cpu_handler
|
||||
yield CPULoadStoreSpec, GPULoadStoreSpec, self._handlers.cpu_to_gpu_handler
|
||||
assert self._worker is not None
|
||||
return self._worker
|
||||
|
||||
@@ -44,7 +44,7 @@ from vllm.v1.kv_offload.base import (
|
||||
OffloadingManager,
|
||||
OffloadingMetricMetadata,
|
||||
)
|
||||
from vllm.v1.kv_offload.cpu.gpu_worker import CpuGpuOffloadingHandlers
|
||||
from vllm.v1.kv_offload.cpu.gpu_worker import CPUOffloadingWorker
|
||||
from vllm.v1.kv_offload.cpu.shared_offload_region import SharedOffloadRegion
|
||||
from vllm.v1.kv_offload.cpu.spec import CPUOffloadingSpec
|
||||
from vllm.v1.kv_offload.tiering.factory import SecondaryTierFactory
|
||||
@@ -163,8 +163,8 @@ class TieringOffloadingSpec(CPUOffloadingSpec):
|
||||
raise
|
||||
|
||||
# Create TieringOffloadingManager. GPU↔CPU transfers use the inherited
|
||||
# get_handlers(); secondary tier transfers are handled by the
|
||||
# secondary tier managers and need no additional handlers here.
|
||||
# get_worker(). Secondary tier transfers are handled by the
|
||||
# secondary tier managers and need no additional workers here.
|
||||
tiering_manager = TieringOffloadingManager(
|
||||
primary_tier=primary_tier,
|
||||
secondary_tiers=secondary_tiers,
|
||||
@@ -187,7 +187,7 @@ class TieringOffloadingSpec(CPUOffloadingSpec):
|
||||
return self._manager
|
||||
|
||||
@override
|
||||
def create_handlers(self, kv_caches: CanonicalKVCaches) -> CpuGpuOffloadingHandlers:
|
||||
def create_worker(self, kv_caches: CanonicalKVCaches) -> CPUOffloadingWorker:
|
||||
rank = torch.accelerator.current_device_index()
|
||||
worker_mmap = SharedOffloadRegion(
|
||||
instance_id=self.vllm_config.instance_id,
|
||||
@@ -196,7 +196,7 @@ class TieringOffloadingSpec(CPUOffloadingSpec):
|
||||
kv_bytes_per_block=self.kv_bytes_per_offloaded_block,
|
||||
cpu_page_size=self.cpu_page_size_per_worker,
|
||||
)
|
||||
return CpuGpuOffloadingHandlers(
|
||||
return CPUOffloadingWorker(
|
||||
kv_caches=kv_caches,
|
||||
block_size_factor=self.block_size_factor,
|
||||
num_cpu_blocks=self.num_blocks,
|
||||
|
||||
@@ -1,176 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from abc import ABC, abstractmethod
|
||||
from dataclasses import dataclass
|
||||
|
||||
from vllm.logger import init_logger
|
||||
from vllm.v1.kv_offload.base import LoadStoreSpec
|
||||
|
||||
# a single transfer spec (src_blocks_spec, dst_blocks_spec)
|
||||
TransferSpec = tuple[LoadStoreSpec, LoadStoreSpec]
|
||||
# transfers are forwarded to workers by (src_medium, dst_medium)
|
||||
TransferType = tuple[str, str]
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
@dataclass
|
||||
class TransferResult:
|
||||
job_id: int
|
||||
success: bool
|
||||
transfer_size: int | None = None # Size in bytes
|
||||
transfer_time: float | None = None
|
||||
transfer_type: TransferType | None = None
|
||||
|
||||
|
||||
class OffloadingHandler(ABC):
|
||||
"""
|
||||
OffloadingHandler class for managing asynchronous KV data transfers
|
||||
|
||||
This class runs in the worker.
|
||||
It kicks off async KV data transfer requests, and allows
|
||||
collecting back completion statuses.
|
||||
|
||||
The class provides the following primitives:
|
||||
transfer_async() - kicks off a new transfer job
|
||||
get_finished() - returns a list of newly finished job IDs.
|
||||
"""
|
||||
|
||||
@abstractmethod
|
||||
def transfer_async(self, job_id: int, spec: TransferSpec) -> bool:
|
||||
"""
|
||||
Initiates an asynchronous transfer of KV data.
|
||||
|
||||
Args:
|
||||
job_id: a unique ID that will be used when notifying back on
|
||||
transfer completion.
|
||||
spec: the (src, dst) spec of the KV data transfer.
|
||||
|
||||
Returns:
|
||||
True if transfer was submitted successfully.
|
||||
"""
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
def get_finished(self) -> list[TransferResult]:
|
||||
"""
|
||||
Get transfers finished since last call.
|
||||
|
||||
Returns:
|
||||
A list of (job_id, success) of transfers.
|
||||
"""
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
def wait(self, job_ids: set[int]) -> None:
|
||||
"""
|
||||
Wait for jobs to finish (blocking).
|
||||
Args:
|
||||
job_ids: The set of job IDs to wait for.
|
||||
"""
|
||||
|
||||
def shutdown(self) -> None:
|
||||
"""Shutdown the handler and release any resources."""
|
||||
return
|
||||
|
||||
|
||||
class OffloadingWorker:
|
||||
"""
|
||||
OffloadingWorker class for managing asynchronous KV data transfers
|
||||
using multiple OffloadingHandlers
|
||||
|
||||
This class runs in the worker.
|
||||
It kicks off async KV data transfer requests, by delegating
|
||||
to one of its registered OffloadingHandlers, based on the transfer type.
|
||||
|
||||
The class provides the following primitives:
|
||||
register_handler() - registers a new handler to handle
|
||||
a specific transfer type
|
||||
transfer_async() - kicks off a new transfer job
|
||||
using one of the registered handlers.
|
||||
get_finished() - returns a list of newly finished job IDs
|
||||
from all handlers.
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
self.handlers: set[OffloadingHandler] = set()
|
||||
self.transfer_type_to_handler: dict[TransferType, OffloadingHandler] = {}
|
||||
|
||||
def register_handler(
|
||||
self,
|
||||
src_cls: type[LoadStoreSpec],
|
||||
dst_cls: type[LoadStoreSpec],
|
||||
handler: OffloadingHandler,
|
||||
) -> None:
|
||||
"""
|
||||
Registers a new handler.
|
||||
|
||||
Args:
|
||||
src_cls: the source type of transfers handled by this handler.
|
||||
dst_cls: the destination type of transfers handled by this handler.
|
||||
handler: the handler that will handle transfers.
|
||||
"""
|
||||
transfer_type = (src_cls.medium(), dst_cls.medium())
|
||||
assert transfer_type not in self.transfer_type_to_handler
|
||||
self.handlers.add(handler)
|
||||
self.transfer_type_to_handler[transfer_type] = handler
|
||||
|
||||
def transfer_async(self, job_id: int, spec: TransferSpec) -> bool:
|
||||
"""
|
||||
Initiates an asynchronous transfer of KV data.
|
||||
|
||||
Args:
|
||||
job_id: a unique ID that will be used when notifying back on
|
||||
transfer completion.
|
||||
spec: the (src, dst) spec of the KV data transfer.
|
||||
|
||||
Returns:
|
||||
True if transfer was submitted successfully.
|
||||
"""
|
||||
src, dst = spec
|
||||
transfer_type = (src.medium(), dst.medium())
|
||||
handler = self.transfer_type_to_handler.get(transfer_type)
|
||||
assert handler is not None
|
||||
try:
|
||||
success = handler.transfer_async(job_id, spec)
|
||||
except Exception as e:
|
||||
logger.warning(
|
||||
"Exception in %r transfer %d: %r",
|
||||
transfer_type,
|
||||
job_id,
|
||||
e,
|
||||
exc_info=True,
|
||||
)
|
||||
return False
|
||||
|
||||
if not success:
|
||||
logger.warning("Failed to submit %r transfer %d", transfer_type, job_id)
|
||||
else:
|
||||
logger.debug("Submitted %r transfer %d: %r", transfer_type, job_id, spec)
|
||||
return success
|
||||
|
||||
def get_finished(self) -> list[TransferResult]:
|
||||
"""
|
||||
Get transfers finished since last call.
|
||||
|
||||
Returns:
|
||||
A list of TransferResults
|
||||
"""
|
||||
finished = []
|
||||
for handler in self.handlers:
|
||||
finished.extend(handler.get_finished())
|
||||
return finished
|
||||
|
||||
def wait(self, job_ids: set[int]) -> None:
|
||||
"""
|
||||
Wait for jobs to finish (blocking).
|
||||
|
||||
Args:
|
||||
job_ids: The set of job IDs to wait for.
|
||||
"""
|
||||
for handler in self.handlers:
|
||||
handler.wait(job_ids)
|
||||
|
||||
def shutdown(self) -> None:
|
||||
for handler in self.handlers:
|
||||
handler.shutdown()
|
||||
Reference in New Issue
Block a user