From f237e16b41bb444b3c9994260a36f9c2388bd019 Mon Sep 17 00:00:00 2001 From: Martin Hickey Date: Wed, 24 Jun 2026 12:44:24 +0100 Subject: [PATCH] [KV Offload] Replace OffloadingHandler with OffloadingWorker (#45053) Signed-off-by: Martin Hickey --- .../unit/offloading_connector/test_worker.py | 6 +- .../unit/offloading_connector/utils.py | 48 ++--- tests/v1/kv_offload/cpu/test_gpu_worker.py | 41 ++-- tests/v1/kv_offload/test_worker.py | 165 ---------------- .../kv_connector/v1/offloading/common.py | 5 +- .../kv_connector/v1/offloading/scheduler.py | 5 +- .../kv_connector/v1/offloading/worker.py | 44 +++-- vllm/v1/kv_offload/base.py | 48 ++++- vllm/v1/kv_offload/cpu/gpu_worker.py | 62 +++--- vllm/v1/kv_offload/cpu/spec.py | 31 ++- vllm/v1/kv_offload/tiering/spec.py | 10 +- vllm/v1/kv_offload/worker/__init__.py | 0 vllm/v1/kv_offload/worker/worker.py | 176 ------------------ 13 files changed, 176 insertions(+), 465 deletions(-) delete mode 100644 tests/v1/kv_offload/test_worker.py delete mode 100644 vllm/v1/kv_offload/worker/__init__.py delete mode 100644 vllm/v1/kv_offload/worker/worker.py diff --git a/tests/v1/kv_connector/unit/offloading_connector/test_worker.py b/tests/v1/kv_connector/unit/offloading_connector/test_worker.py index 833d4fe0a4..81c00266cf 100644 --- a/tests/v1/kv_connector/unit/offloading_connector/test_worker.py +++ b/tests/v1/kv_connector/unit/offloading_connector/test_worker.py @@ -98,7 +98,7 @@ def _make_worker(kv_cache_config: KVCacheConfig): spec = MagicMock(spec=OffloadingSpec) spec.kv_cache_config = kv_cache_config spec.vllm_config = MagicMock() - spec.get_handlers.return_value = iter([]) + spec.get_worker.return_value = MagicMock() worker = OffloadingConnectorWorker(spec=spec) worker.worker = MagicMock() @@ -279,7 +279,7 @@ def test_register_kv_caches(backend): worker, spec = _make_worker(kv_cache_config) worker.register_kv_caches(kv_caches) - canonical = spec.get_handlers.call_args[0][0] + canonical = spec.get_worker.call_args[0][0] assert isinstance(canonical, CanonicalKVCaches) # -- Expected block tensors ---------------------------------------------- @@ -422,7 +422,7 @@ def test_register_kv_caches_uniform_type(backend): worker, spec = _make_worker(kv_cache_config) worker.register_kv_caches(kv_caches) - canonical = spec.get_handlers.call_args[0][0] + canonical = spec.get_worker.call_args[0][0] assert isinstance(canonical, CanonicalKVCaches) for block_tensor in canonical.tensors: diff --git a/tests/v1/kv_connector/unit/offloading_connector/utils.py b/tests/v1/kv_connector/unit/offloading_connector/utils.py index 95980690d6..a232082879 100644 --- a/tests/v1/kv_connector/unit/offloading_connector/utils.py +++ b/tests/v1/kv_connector/unit/offloading_connector/utils.py @@ -1,6 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -from collections.abc import Callable, Iterable, Iterator +from collections.abc import Callable, Iterable from dataclasses import dataclass from typing import Any from unittest.mock import MagicMock @@ -43,19 +43,17 @@ from vllm.v1.kv_cache_interface import ( KVCacheGroupSpec, ) from vllm.v1.kv_offload.base import ( + CanonicalKVCaches, GPULoadStoreSpec, LoadStoreSpec, OffloadingManager, OffloadingSpec, + OffloadingWorker, OffloadKey, PrepareStoreOutput, RequestOffloadingContext, - make_offload_key, -) -from vllm.v1.kv_offload.worker.worker import ( - OffloadingHandler, TransferResult, - TransferSpec, + make_offload_key, ) from vllm.v1.request import Request from vllm.v1.structured_output import StructuredOutputManager @@ -81,9 +79,9 @@ class MockLoadStoreSpec(LoadStoreSpec): return repr(self.offload_keys) -class MockOffloadingHandler(OffloadingHandler): +class MockOffloadingWorker(OffloadingWorker): def __init__(self): - self.transfer_specs: dict[int, TransferSpec] = {} + self.transfer_specs: dict[int, tuple[LoadStoreSpec, LoadStoreSpec]] = {} self.completed_transfers: list[TransferResult] = [] self.waiting_jobs: set[int] = set() self.completed_jobs: list[int] = [] @@ -94,8 +92,17 @@ class MockOffloadingHandler(OffloadingHandler): self.completed_transfers = [] return finished - def transfer_async(self, job_id: int, spec: TransferSpec) -> bool: - self.transfer_specs[job_id] = spec + def submit_store( + self, job_id: int, src_spec: LoadStoreSpec, dst_spec: LoadStoreSpec + ) -> bool: # type: ignore[override] + self.transfer_specs[job_id] = (src_spec, dst_spec) + self.waiting_jobs.add(job_id) + return True + + def submit_load( + self, job_id: int, src_spec: LoadStoreSpec, dst_spec: LoadStoreSpec + ) -> bool: # type: ignore[override] + self.transfer_specs[job_id] = (src_spec, dst_spec) self.waiting_jobs.add(job_id) return True @@ -109,7 +116,6 @@ class MockOffloadingHandler(OffloadingHandler): success=True, transfer_size=None, transfer_time=None, - transfer_type=None, ) self.completed_transfers.append(result) @@ -127,21 +133,18 @@ class MockOffloadingSpec(OffloadingSpec): self.manager.prepare_load = lambda keys, req_context: MockLoadStoreSpec(keys) self.manager.lookup.return_value = False self.manager.on_new_request.return_value = RequestOffloadingContext() - self.handler = MockOffloadingHandler() + self.handler = MockOffloadingWorker() def get_manager(self) -> OffloadingManager: return self.manager - def get_handlers( - self, _ - ) -> Iterator[tuple[type[LoadStoreSpec], type[LoadStoreSpec], OffloadingHandler]]: - yield GPULoadStoreSpec, MockLoadStoreSpec, self.handler - yield MockLoadStoreSpec, GPULoadStoreSpec, self.handler + def get_worker(self, _: CanonicalKVCaches) -> OffloadingWorker: + return self.handler def complete_transfers(self): self.handler.complete_jobs(self.handler.waiting_jobs.copy()) - def get_completed_transfers(self) -> list[TransferSpec]: + def get_completed_transfers(self) -> list[tuple[LoadStoreSpec, LoadStoreSpec]]: specs = [ self.handler.transfer_specs[job_id] for job_id in self.handler.completed_jobs @@ -149,7 +152,7 @@ class MockOffloadingSpec(OffloadingSpec): self.handler.completed_jobs.clear() return specs - def get_flushed_transfers(self): + def get_flushed_transfers(self) -> list[tuple[LoadStoreSpec, LoadStoreSpec]]: specs = [ self.handler.transfer_specs[job_id] for job_id in self.handler.flushed_jobs ] @@ -362,8 +365,7 @@ class RequestRunner: self.scheduler.add_request(req) def _parse_transfers(self): - for transfer_spec in self.offloading_spec.get_flushed_transfers(): - src_spec, dst_spec = transfer_spec + for src_spec, dst_spec in self.offloading_spec.get_flushed_transfers(): if isinstance(src_spec, GPULoadStoreSpec): # store flush for block_id in src_spec.block_ids: @@ -375,9 +377,7 @@ class RequestRunner: block_size_factor = self.block_size_factor - for transfer_spec in self.offloading_spec.get_completed_transfers(): - src_spec, dst_spec = transfer_spec - + for src_spec, dst_spec in self.offloading_spec.get_completed_transfers(): if isinstance(src_spec, GPULoadStoreSpec): store = True gpu_spec = src_spec diff --git a/tests/v1/kv_offload/cpu/test_gpu_worker.py b/tests/v1/kv_offload/cpu/test_gpu_worker.py index d192b04a07..d8ce9093c2 100644 --- a/tests/v1/kv_offload/cpu/test_gpu_worker.py +++ b/tests/v1/kv_offload/cpu/test_gpu_worker.py @@ -17,7 +17,7 @@ from vllm.v1.kv_offload.base import ( GPULoadStoreSpec, ) from vllm.v1.kv_offload.cpu.common import CPULoadStoreSpec -from vllm.v1.kv_offload.cpu.gpu_worker import CpuGpuOffloadingHandlers +from vllm.v1.kv_offload.cpu.gpu_worker import CPUOffloadingWorker from vllm.v1.kv_offload.cpu.shared_offload_region import SharedOffloadRegion NUM_GPU_BLOCKS = [64] @@ -103,7 +103,7 @@ def test_transfer( cpu_page_size=cpu_page_size, ) - handlers = CpuGpuOffloadingHandlers( + worker = CPUOffloadingWorker( kv_caches=kv_caches, block_size_factor=block_size_factor, num_cpu_blocks=num_cpu_blocks, @@ -130,7 +130,7 @@ def test_transfer( # set transfer direction if gpu_to_cpu: - handler = handlers.gpu_to_cpu_handler + handler = worker._store_handler src_spec = GPULoadStoreSpec( gpu_blocks, group_sizes=(len(gpu_blocks),), block_indices=(blocks_to_skip,) ) @@ -138,7 +138,7 @@ def test_transfer( dst_to_src = dict(zip(cpu_blocks_expanded, gpu_blocks)) num_dst_sub_blocks = num_gpu_blocks else: - handler = handlers.cpu_to_gpu_handler + handler = worker._load_handler src_spec = CPULoadStoreSpec(cpu_blocks) dst_spec = GPULoadStoreSpec( gpu_blocks, group_sizes=(len(gpu_blocks),), block_indices=(blocks_to_skip,) @@ -156,23 +156,21 @@ def test_transfer( orig_src_tensors = [x.clone() for x in handler.src_tensors] orig_dst_tensors = [x.clone() for x in handler.dst_tensors] - # call transfer function + # call transfer function via public API start_time = time.time() - assert handler.transfer_async(1, (src_spec, dst_spec)) + if gpu_to_cpu: + assert worker.submit_store(1, src_spec, dst_spec) + else: + assert worker.submit_load(1, src_spec, dst_spec) assert {x.job_id for x in handler._transfers} == {1} # wait for transfer to complete end_time = time.time() + 10 while time.time() < end_time: - finished = handler.get_finished() + finished = worker.get_finished() if finished: assert finished[0].job_id == 1 assert finished[0].success - assert ( - finished[0].transfer_type == ("GPU", "CPU") - if gpu_to_cpu - else ("CPU", "GPU") - ) assert finished[0].transfer_size == ( len(gpu_blocks) * sum([x.page_size_bytes for x in handler.kv_cache_groups_data_refs[0]]) @@ -208,8 +206,7 @@ def test_transfer( del orig_tensor, tensor, src_tensor, dst_tensor, orig_dst_tensor del src_view, dst_view, orig_dst_view, expected - handlers.cpu_to_gpu_handler.shutdown() - handlers.gpu_to_cpu_handler.shutdown() + worker.shutdown() if mmap_region: mmap_region.cleanup() @@ -276,7 +273,7 @@ def test_transfer_multi_group( tensors=kv_cache_tensors, group_data_refs=kv_cache_groups_data_refs ) - handlers = CpuGpuOffloadingHandlers( + worker = CPUOffloadingWorker( kv_caches=canonical_kv_caches, block_size_factor=block_size_factor, num_cpu_blocks=num_cpu_blocks, @@ -338,7 +335,7 @@ def test_transfer_multi_group( block_indices: list[int] = [0, 0, sub_blocks_to_skip] if gpu_to_cpu: - handler = handlers.gpu_to_cpu_handler + handler = worker._store_handler src_spec = GPULoadStoreSpec( gpu_blocks, group_sizes=group_sizes, block_indices=block_indices ) @@ -352,7 +349,7 @@ def test_transfer_multi_group( ] num_dst_sub_blocks = num_cpu_blocks * block_size_factor else: - handler = handlers.cpu_to_gpu_handler + handler = worker._load_handler src_spec = CPULoadStoreSpec(cpu_blocks) dst_spec = GPULoadStoreSpec( gpu_blocks, group_sizes=group_sizes, block_indices=block_indices @@ -375,12 +372,15 @@ def test_transfer_multi_group( orig_src_tensors = [x.clone() for x in handler.src_tensors] orig_dst_tensors = [x.clone() for x in handler.dst_tensors] - assert handler.transfer_async(1, (src_spec, dst_spec)) + if gpu_to_cpu: + assert worker.submit_store(1, src_spec, dst_spec) + else: + assert worker.submit_load(1, src_spec, dst_spec) assert {x.job_id for x in handler._transfers} == {1} end_time = time.time() + 10 while time.time() < end_time: - finished = handler.get_finished() + finished = worker.get_finished() if finished: assert finished[0].job_id == 1 assert finished[0].success @@ -418,5 +418,4 @@ def test_transfer_multi_group( dst_view[dst_sub_block].cpu(), expected.cpu() ) - handlers.cpu_to_gpu_handler.shutdown() - handlers.gpu_to_cpu_handler.shutdown() + worker.shutdown() diff --git a/tests/v1/kv_offload/test_worker.py b/tests/v1/kv_offload/test_worker.py deleted file mode 100644 index b291fcf1b8..0000000000 --- a/tests/v1/kv_offload/test_worker.py +++ /dev/null @@ -1,165 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -from vllm.v1.kv_offload.base import LoadStoreSpec -from vllm.v1.kv_offload.worker.worker import ( - OffloadingHandler, - OffloadingWorker, - TransferResult, - TransferSpec, -) - - -class LoadStoreSpec1(LoadStoreSpec): - def __init__( - self, - submit_success: bool = True, - async_success: bool = True, - exception: bool = False, - ): - self.finished = False - self.submit_success = submit_success - self.async_success = async_success - self.exception = exception - - @staticmethod - def medium() -> str: - return "1" - - def __repr__(self): - return f"{self.medium()}: {id(self)}" - - -class LoadStoreSpec2(LoadStoreSpec): - @staticmethod - def medium() -> str: - return "2" - - def __repr__(self): - return f"{self.medium()}: {id(self)}" - - -class OffloadingHandler1To2(OffloadingHandler): - def __init__(self): - self.transfers: dict[int, LoadStoreSpec1] = {} - - def transfer_async(self, job_id: int, spec: TransferSpec) -> bool: - src, dst = spec - assert isinstance(src, LoadStoreSpec1) - assert isinstance(dst, LoadStoreSpec2) - - if src.exception: - raise Exception("An expected exception. Don't worry!") - if not src.submit_success: - return False - - self.transfers[job_id] = src - return True - - def get_finished(self) -> list[TransferResult]: - finished = [] - for job_id, spec in list(self.transfers.items()): - if spec.finished: - finished.append((job_id, spec.async_success)) - del self.transfers[job_id] - return finished - - def wait(self, job_ids: set[int]) -> None: - for job_id in job_ids: - spec = self.transfers.get(job_id) - if spec: - assert spec.finished - - -class OffloadingHandler2To1(OffloadingHandler): - def __init__(self): - self.transfers: dict[int, LoadStoreSpec1] = {} - - def transfer_async(self, job_id: int, spec: TransferSpec) -> bool: - src, dst = spec - assert isinstance(src, LoadStoreSpec2) - assert isinstance(dst, LoadStoreSpec1) - - self.transfers[job_id] = dst - return True - - def get_finished(self) -> list[TransferResult]: - finished = [] - for job_id, spec in list(self.transfers.items()): - if spec.finished: - finished.append((job_id, spec.async_success)) - del self.transfers[job_id] - return finished - - def wait(self, job_ids: set[int]) -> None: - for job_id in job_ids: - spec = self.transfers.get(job_id) - if spec: - assert spec.finished - - -def test_offloading_worker(): - """ - Tests OffloadingWorker with 2 handlers. - One handler performs 1->2 transfers, and the other handles 2->1. - """ - worker = OffloadingWorker() - handler1to2 = OffloadingHandler1To2() - handler2to1 = OffloadingHandler2To1() - worker.register_handler(LoadStoreSpec1, LoadStoreSpec2, handler1to2) - worker.register_handler(LoadStoreSpec2, LoadStoreSpec1, handler2to1) - - # 1st transfer 1->2 (exception) - src1 = LoadStoreSpec1(exception=True) - dst1 = LoadStoreSpec2() - assert not worker.transfer_async(1, (src1, dst1)) - - # 2ed transfer 1->2 (failure to submit) - src2 = LoadStoreSpec1(submit_success=False) - dst2 = LoadStoreSpec2() - assert not worker.transfer_async(2, (src2, dst2)) - - # 3rd transfer 1->2 (failure) - src3 = LoadStoreSpec1(async_success=False) - dst3 = LoadStoreSpec2() - assert worker.transfer_async(3, (src3, dst3)) - - # 4th transfer 1->2 (success) - src4 = LoadStoreSpec1() - dst4 = LoadStoreSpec2() - worker.transfer_async(4, (src4, dst4)) - assert set(handler1to2.transfers.keys()) == {3, 4} - - # 5th transfer 2->1 - src5 = LoadStoreSpec2() - dst5 = LoadStoreSpec1() - worker.transfer_async(5, (src5, dst5)) - assert set(handler2to1.transfers.keys()) == {5} - - # no transfer completed yet - assert worker.get_finished() == [] - - # complete 3rd, 4th - src3.finished = True - src4.finished = True - - # 6th transfer 1->2 - src6 = LoadStoreSpec1() - dst6 = LoadStoreSpec2() - worker.transfer_async(6, (src6, dst6)) - - # 7th transfer 2->1 - src7 = LoadStoreSpec2() - dst7 = LoadStoreSpec1() - worker.transfer_async(7, (src7, dst7)) - - # 6th and 7th transfers started - assert 6 in handler1to2.transfers - assert 7 in handler2to1.transfers - - # verify result of 3rd and 4th transfers - assert sorted(worker.get_finished()) == [(3, False), (4, True)] - - # complete 6th and 7th transfers - src6.finished = True - dst7.finished = True - assert sorted(worker.get_finished()) == [(6, True), (7, True)] diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/offloading/common.py b/vllm/distributed/kv_transfer/kv_connector/v1/offloading/common.py index 928fec639c..939a2b08ff 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/offloading/common.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/offloading/common.py @@ -6,7 +6,7 @@ from vllm.distributed.kv_transfer.kv_connector.v1.base import ( KVConnectorMetadata, KVConnectorWorkerMetadata, ) -from vllm.v1.kv_offload.worker.worker import TransferSpec +from vllm.v1.kv_offload.base import LoadStoreSpec ReqId = str @@ -60,7 +60,8 @@ class TransferJob: """ req_id: ReqId - transfer_spec: TransferSpec + src_spec: LoadStoreSpec + dst_spec: LoadStoreSpec @dataclass diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/offloading/scheduler.py b/vllm/distributed/kv_transfer/kv_connector/v1/offloading/scheduler.py index 5884186cc9..4965d9db87 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/offloading/scheduler.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/offloading/scheduler.py @@ -762,7 +762,8 @@ class OffloadingConnectorScheduler: load_job_id = self._generate_job_id() self._current_batch_load_jobs[load_job_id] = TransferJob( req_id=request.request_id, - transfer_spec=(src_spec, dst_spec), + src_spec=src_spec, + dst_spec=dst_spec, ) # a load can only be issued when no other jobs are pending. assert not req_status.transfer_jobs @@ -998,7 +999,7 @@ class OffloadingConnectorScheduler: ) store_jobs[job_id] = TransferJob( - req_id=req_id, transfer_spec=(src_spec, dst_spec) + req_id=req_id, src_spec=src_spec, dst_spec=dst_spec ) logger.debug( diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/offloading/worker.py b/vllm/distributed/kv_transfer/kv_connector/v1/offloading/worker.py index f22d6738b4..1e0435d371 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/offloading/worker.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/offloading/worker.py @@ -21,11 +21,10 @@ from vllm.v1.kv_offload.base import ( CanonicalKVCacheRef, CanonicalKVCaches, CanonicalKVCacheTensor, + GPULoadStoreSpec, + LoadStoreSpec, OffloadingSpec, -) -from vllm.v1.kv_offload.worker.worker import ( OffloadingWorker, - TransferSpec, ) logger = init_logger(__name__) @@ -36,16 +35,17 @@ class OffloadingConnectorWorker: def __init__(self, spec: OffloadingSpec): self.spec = spec - self.worker = OffloadingWorker() + self.worker: OffloadingWorker | None = None # job_id -> req_id for in-flight loads. self._load_jobs: dict[int, ReqId] = {} - self._unsubmitted_store_jobs: list[tuple[int, TransferSpec]] = [] + self._unsubmitted_store_jobs: list[ + tuple[int, GPULoadStoreSpec, LoadStoreSpec] + ] = [] self._connector_worker_meta = OffloadingWorkerMetadata() - def _register_handlers(self, kv_caches: CanonicalKVCaches): - for src_cls, dst_cls, handler in self.spec.get_handlers(kv_caches): - self.worker.register_handler(src_cls, dst_cls, handler) + def _init_worker(self, kv_caches: CanonicalKVCaches) -> None: + self.worker = self.spec.get_worker(kv_caches) def register_kv_caches( self, kv_caches: dict[str, torch.Tensor | list[torch.Tensor]] @@ -138,7 +138,7 @@ class OffloadingConnectorWorker: (block_stride, 1), storage_offset=0, ) - self._register_handlers( + self._init_worker( CanonicalKVCaches( [CanonicalKVCacheTensor(packed_tensor, block_stride)], [ @@ -204,7 +204,7 @@ class OffloadingConnectorWorker: group_data_refs=group_data_refs, ) - self._register_handlers(canonical_kv_caches) + self._init_worker(canonical_kv_caches) def register_cross_layers_kv_cache( self, kv_cache: torch.Tensor, attn_backend: type[AttentionBackend] @@ -251,11 +251,12 @@ class OffloadingConnectorWorker: tensors=[kv_cache_tensor], group_data_refs=[[kv_cache_data_ref]] ) - self._register_handlers(canonical_kv_caches) + self._init_worker(canonical_kv_caches) def handle_preemptions(self, kv_connector_metadata: OffloadingConnectorMetadata): - for job_id, transfer_spec in self._unsubmitted_store_jobs: - success = self.worker.transfer_async(job_id, transfer_spec) + assert self.worker is not None + for job_id, src_spec, dst_spec in self._unsubmitted_store_jobs: + success = self.worker.submit_store(job_id, src_spec, dst_spec) assert success self._unsubmitted_store_jobs.clear() @@ -263,14 +264,16 @@ class OffloadingConnectorWorker: self.worker.wait(kv_connector_metadata.jobs_to_flush) def start_kv_transfers(self, metadata: OffloadingConnectorMetadata): - for job_id, transfer_spec in self._unsubmitted_store_jobs: - success = self.worker.transfer_async(job_id, transfer_spec) + assert self.worker is not None + for job_id, src_spec, dst_spec in self._unsubmitted_store_jobs: + success = self.worker.submit_store(job_id, src_spec, dst_spec) assert success self._unsubmitted_store_jobs.clear() for job_id, entry in metadata.load_jobs.items(): self._load_jobs[job_id] = entry.req_id - success = self.worker.transfer_async(job_id, entry.transfer_spec) + assert isinstance(entry.dst_spec, GPULoadStoreSpec) + success = self.worker.submit_load(job_id, entry.src_spec, entry.dst_spec) assert success def prepare_store_kv(self, metadata: OffloadingConnectorMetadata): @@ -278,7 +281,10 @@ class OffloadingConnectorWorker: # NOTE(orozery): defer the store to the beginning of the next # engine step, so that offloading starts AFTER transfers related # to token sampling, thereby avoiding delays to token generation. - self._unsubmitted_store_jobs.append((job_id, entry.transfer_spec)) + assert isinstance(entry.src_spec, GPULoadStoreSpec) + self._unsubmitted_store_jobs.append( + (job_id, entry.src_spec, entry.dst_spec) + ) def get_finished(self, finished_req_ids: set[str]) -> tuple[set[str], set[str]]: """ @@ -290,6 +296,7 @@ class OffloadingConnectorWorker: finished_recving so the base scheduler can resume requests blocked on remote KV (and free aborted-during-load reqs). """ + assert self.worker is not None finished_recving: set[str] = set() for transfer_result in self.worker.get_finished(): # we currently do not support job failures @@ -328,4 +335,5 @@ class OffloadingConnectorWorker: self._unsubmitted_store_jobs.clear() self._load_jobs.clear() self._connector_worker_meta = OffloadingWorkerMetadata() - self.worker.shutdown() + if self.worker is not None: + self.worker.shutdown() diff --git a/vllm/v1/kv_offload/base.py b/vllm/v1/kv_offload/base.py index d410f42701..70a53b072f 100644 --- a/vllm/v1/kv_offload/base.py +++ b/vllm/v1/kv_offload/base.py @@ -5,7 +5,7 @@ Core abstractions for KV cache offloading in vLLM v1. """ from abc import ABC, abstractmethod -from collections.abc import Collection, Iterable, Iterator, Sequence +from collections.abc import Collection, Iterable, Sequence from dataclasses import dataclass from enum import Enum from typing import TYPE_CHECKING, Any, NewType @@ -23,7 +23,6 @@ if TYPE_CHECKING: OffloadingConnectorStats, ) from vllm.v1.kv_cache_interface import KVCacheConfig - from vllm.v1.kv_offload.worker.worker import OffloadingHandler # `OffloadKey` identifies an offloaded block. It combines a block hash with # its KV cache group index, encoded as raw bytes to avoid tuple GC overhead. @@ -432,6 +431,41 @@ class CanonicalKVCaches: group_data_refs: list[list[CanonicalKVCacheRef]] +@dataclass +class TransferResult: + job_id: int + success: bool + transfer_size: int | None = None + transfer_time: float | None = None + + +class OffloadingWorker(ABC): + """Runs in the worker process. Performs async KV transfers for ONE + offloaded medium (e.g. CPU). Direction is explicit via submit_store / + submit_load, so there is no (src_medium, dst_medium) routing.""" + + @abstractmethod + def submit_store( + self, job_id: int, src_spec: GPULoadStoreSpec, dst_spec: LoadStoreSpec + ) -> bool: + """Async GPU -> offloaded medium.""" + + @abstractmethod + def submit_load( + self, job_id: int, src_spec: LoadStoreSpec, dst_spec: GPULoadStoreSpec + ) -> bool: + """Async offloaded medium -> GPU.""" + + @abstractmethod + def get_finished(self) -> list[TransferResult]: ... + + @abstractmethod + def wait(self, job_ids: set[int]) -> None: ... + + def shutdown(self) -> None: + return + + class OffloadingSpec(ABC): """Spec for an offloading connector""" @@ -524,16 +558,14 @@ class OffloadingSpec(ABC): pass @abstractmethod - def get_handlers( - self, kv_caches: CanonicalKVCaches - ) -> Iterator[tuple[type[LoadStoreSpec], type[LoadStoreSpec], "OffloadingHandler"]]: + def get_worker(self, kv_caches: CanonicalKVCaches) -> OffloadingWorker: """ - Get offloading handlers along with their respective src and dst types. + Get an OffloadingWorker that handles async KV transfers for this spec. Args: kv_caches: Canonicalized KV caches. - Yields: - Tuples of (src_type, dst_type, offloading_handler). + Returns: + An OffloadingWorker instance for this medium. """ pass diff --git a/vllm/v1/kv_offload/cpu/gpu_worker.py b/vllm/v1/kv_offload/cpu/gpu_worker.py index f4d3869dc1..c8b9915a1e 100644 --- a/vllm/v1/kv_offload/cpu/gpu_worker.py +++ b/vllm/v1/kv_offload/cpu/gpu_worker.py @@ -7,7 +7,6 @@ from dataclasses import dataclass import numpy as np import torch -from typing_extensions import override from vllm import _custom_ops as ops from vllm.logger import init_logger @@ -20,17 +19,15 @@ from vllm.v1.kv_offload.base import ( CanonicalKVCacheRef, CanonicalKVCaches, GPULoadStoreSpec, + LoadStoreSpec, + OffloadingWorker, + TransferResult, ) from vllm.v1.kv_offload.cpu.shared_offload_region import SharedOffloadRegion from vllm.v1.kv_offload.cpu.swap_blocks_triton import ( THRESHOLD_BYTES, swap_blocks_batch, ) -from vllm.v1.kv_offload.worker.worker import ( - OffloadingHandler, - TransferResult, - TransferSpec, -) logger = init_logger(__name__) @@ -166,10 +163,9 @@ def _new_descriptor_buffers( ) -class SingleDirectionOffloadingHandler(OffloadingHandler): +class SingleDirectionOffloadingHandler: """ - SingleDirectionOffloadingHandler handles transfers for a single direction, - either CPU->GPU or GPU->CPU. + Handles transfers for a single direction, either CPU->GPU or GPU->CPU. Transfers are guaranteed to be executed in order of their submission. Each transfer uses a unique CUDA stream, and its stream will start executing only after the streams of previous transfers have finished. @@ -228,7 +224,6 @@ class SingleDirectionOffloadingHandler(OffloadingHandler): self.src_block_size_factor = 1 if self.gpu_to_cpu else block_size_factor self.dst_block_size_factor = block_size_factor if self.gpu_to_cpu else 1 - self.transfer_type = ("GPU", "CPU") if self.gpu_to_cpu else ("CPU", "GPU") # mmap_region to clean up on shutdown (gpu_to_cpu handler owns it) self._mmap_region = mmap_region # job_id -> event @@ -242,9 +237,9 @@ class SingleDirectionOffloadingHandler(OffloadingHandler): # list of pinned descriptor buffer sets available for re-use self._buffer_pool: list[tuple[torch.Tensor, torch.Tensor, torch.Tensor]] = [] - @override - def transfer_async(self, job_id: int, transfer_spec: TransferSpec) -> bool: - src_spec, dst_spec = transfer_spec + def transfer_async( + self, job_id: int, src_spec: LoadStoreSpec, dst_spec: LoadStoreSpec + ) -> bool: assert isinstance(src_spec, BlockIDsLoadStoreSpec) assert isinstance(dst_spec, BlockIDsLoadStoreSpec) @@ -425,7 +420,6 @@ class SingleDirectionOffloadingHandler(OffloadingHandler): # success return True - @override def get_finished(self) -> list[TransferResult]: results: list[TransferResult] = [] while self._transfers and self._transfers[0].end_event.query(): @@ -438,7 +432,6 @@ class SingleDirectionOffloadingHandler(OffloadingHandler): success=True, transfer_size=transfer.num_bytes, transfer_time=transfer_time, - transfer_type=self.transfer_type, ) results.append(result) @@ -451,14 +444,12 @@ class SingleDirectionOffloadingHandler(OffloadingHandler): del self._transfer_events[transfer.job_id] return results - @override def wait(self, job_ids: set[int]): for job_id in job_ids: event = self._transfer_events.get(job_id) if event is not None: event.synchronize() - @override def shutdown(self) -> None: while self._transfers: transfer = self._transfers.popleft() @@ -474,7 +465,14 @@ class SingleDirectionOffloadingHandler(OffloadingHandler): self._mmap_region = None -class CpuGpuOffloadingHandlers: +class CPUOffloadingWorker(OffloadingWorker): + """OffloadingWorker for CPU offloading. + + Composes two SingleDirectionOffloadingHandler instances (one for each + direction) and exposes them through the explicit submit_store / + submit_load API. + """ + def __init__( self, kv_caches: CanonicalKVCaches, @@ -484,7 +482,6 @@ class CpuGpuOffloadingHandlers: ): pin_memory = PIN_MEMORY logger.info("Allocating %d CPU tensors...", len(kv_caches.tensors)) - self._mmap_region = mmap_region if mmap_region is not None and pin_memory: pin_mmap_region(mmap_region) @@ -518,7 +515,7 @@ class CpuGpuOffloadingHandlers: gpu_tensors.append(gpu_tensor) cpu_tensors.append(cpu_tensor) - self.gpu_to_cpu_handler = SingleDirectionOffloadingHandler( + self._store_handler = SingleDirectionOffloadingHandler( gpu_tensors=gpu_tensors, cpu_tensors=cpu_tensors, block_size_factor=block_size_factor, @@ -527,10 +524,33 @@ class CpuGpuOffloadingHandlers: mmap_region=mmap_region, ) - self.cpu_to_gpu_handler = SingleDirectionOffloadingHandler( + self._load_handler = SingleDirectionOffloadingHandler( gpu_tensors=gpu_tensors, cpu_tensors=cpu_tensors, block_size_factor=block_size_factor, kv_cache_groups_data_refs=kv_caches.group_data_refs, gpu_to_cpu=False, ) + + def submit_store( + self, job_id: int, src_spec: GPULoadStoreSpec, dst_spec: LoadStoreSpec + ) -> bool: + """Async GPU -> CPU.""" + return self._store_handler.transfer_async(job_id, src_spec, dst_spec) + + def submit_load( + self, job_id: int, src_spec: LoadStoreSpec, dst_spec: GPULoadStoreSpec + ) -> bool: + """Async CPU -> GPU.""" + return self._load_handler.transfer_async(job_id, src_spec, dst_spec) + + def get_finished(self) -> list[TransferResult]: + return self._store_handler.get_finished() + self._load_handler.get_finished() + + def wait(self, job_ids: set[int]) -> None: + self._store_handler.wait(job_ids) + self._load_handler.wait(job_ids) + + def shutdown(self) -> None: + self._store_handler.shutdown() + self._load_handler.shutdown() diff --git a/vllm/v1/kv_offload/cpu/spec.py b/vllm/v1/kv_offload/cpu/spec.py index 7d3ba9c753..16729a9dbb 100644 --- a/vllm/v1/kv_offload/cpu/spec.py +++ b/vllm/v1/kv_offload/cpu/spec.py @@ -1,6 +1,5 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -from collections.abc import Iterator from typing import Any from typing_extensions import override @@ -11,21 +10,16 @@ from vllm.utils.math_utils import round_up from vllm.v1.kv_cache_interface import KVCacheConfig from vllm.v1.kv_offload.base import ( CanonicalKVCaches, - GPULoadStoreSpec, - LoadStoreSpec, OffloadingCounterMetadata, OffloadingGaugeMetadata, OffloadingManager, OffloadingMetricMetadata, OffloadingSpec, + OffloadingWorker, ) -from vllm.v1.kv_offload.cpu.common import ( - CPULoadStoreSpec, - CPUOffloadingMetrics, -) -from vllm.v1.kv_offload.cpu.gpu_worker import CpuGpuOffloadingHandlers +from vllm.v1.kv_offload.cpu.common import CPUOffloadingMetrics +from vllm.v1.kv_offload.cpu.gpu_worker import CPUOffloadingWorker from vllm.v1.kv_offload.cpu.manager import CPUOffloadingManager -from vllm.v1.kv_offload.worker.worker import OffloadingHandler class CPUOffloadingSpec(OffloadingSpec): @@ -107,7 +101,7 @@ class CPUOffloadingSpec(OffloadingSpec): self._manager: OffloadingManager | None = None # worker-side - self._handlers: CpuGpuOffloadingHandlers | None = None + self._worker: CPUOffloadingWorker | None = None self.eviction_policy: str = self.extra_config.get("eviction_policy", "lru") @@ -131,25 +125,22 @@ class CPUOffloadingSpec(OffloadingSpec): ) return self._manager - def create_handlers(self, kv_caches: CanonicalKVCaches) -> CpuGpuOffloadingHandlers: - return CpuGpuOffloadingHandlers( + def create_worker(self, kv_caches: CanonicalKVCaches) -> CPUOffloadingWorker: + return CPUOffloadingWorker( kv_caches=kv_caches, block_size_factor=self.block_size_factor, num_cpu_blocks=self.num_blocks, ) @override - def get_handlers( - self, kv_caches: CanonicalKVCaches - ) -> Iterator[tuple[type[LoadStoreSpec], type[LoadStoreSpec], OffloadingHandler]]: - if not self._handlers: + def get_worker(self, kv_caches: CanonicalKVCaches) -> OffloadingWorker: + if not self._worker: if not (current_platform.is_cuda_alike() or current_platform.is_xpu()): raise Exception( "CPU Offloading is currently only supported on CUDA-alike " "and XPU GPUs" ) - self._handlers = self.create_handlers(kv_caches) + self._worker = self.create_worker(kv_caches) - assert self._handlers is not None - yield GPULoadStoreSpec, CPULoadStoreSpec, self._handlers.gpu_to_cpu_handler - yield CPULoadStoreSpec, GPULoadStoreSpec, self._handlers.cpu_to_gpu_handler + assert self._worker is not None + return self._worker diff --git a/vllm/v1/kv_offload/tiering/spec.py b/vllm/v1/kv_offload/tiering/spec.py index f4a44a4a8a..406c94e3d7 100644 --- a/vllm/v1/kv_offload/tiering/spec.py +++ b/vllm/v1/kv_offload/tiering/spec.py @@ -44,7 +44,7 @@ from vllm.v1.kv_offload.base import ( OffloadingManager, OffloadingMetricMetadata, ) -from vllm.v1.kv_offload.cpu.gpu_worker import CpuGpuOffloadingHandlers +from vllm.v1.kv_offload.cpu.gpu_worker import CPUOffloadingWorker from vllm.v1.kv_offload.cpu.shared_offload_region import SharedOffloadRegion from vllm.v1.kv_offload.cpu.spec import CPUOffloadingSpec from vllm.v1.kv_offload.tiering.factory import SecondaryTierFactory @@ -163,8 +163,8 @@ class TieringOffloadingSpec(CPUOffloadingSpec): raise # Create TieringOffloadingManager. GPU↔CPU transfers use the inherited - # get_handlers(); secondary tier transfers are handled by the - # secondary tier managers and need no additional handlers here. + # get_worker(). Secondary tier transfers are handled by the + # secondary tier managers and need no additional workers here. tiering_manager = TieringOffloadingManager( primary_tier=primary_tier, secondary_tiers=secondary_tiers, @@ -187,7 +187,7 @@ class TieringOffloadingSpec(CPUOffloadingSpec): return self._manager @override - def create_handlers(self, kv_caches: CanonicalKVCaches) -> CpuGpuOffloadingHandlers: + def create_worker(self, kv_caches: CanonicalKVCaches) -> CPUOffloadingWorker: rank = torch.accelerator.current_device_index() worker_mmap = SharedOffloadRegion( instance_id=self.vllm_config.instance_id, @@ -196,7 +196,7 @@ class TieringOffloadingSpec(CPUOffloadingSpec): kv_bytes_per_block=self.kv_bytes_per_offloaded_block, cpu_page_size=self.cpu_page_size_per_worker, ) - return CpuGpuOffloadingHandlers( + return CPUOffloadingWorker( kv_caches=kv_caches, block_size_factor=self.block_size_factor, num_cpu_blocks=self.num_blocks, diff --git a/vllm/v1/kv_offload/worker/__init__.py b/vllm/v1/kv_offload/worker/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/vllm/v1/kv_offload/worker/worker.py b/vllm/v1/kv_offload/worker/worker.py deleted file mode 100644 index 2f0dd24716..0000000000 --- a/vllm/v1/kv_offload/worker/worker.py +++ /dev/null @@ -1,176 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -from abc import ABC, abstractmethod -from dataclasses import dataclass - -from vllm.logger import init_logger -from vllm.v1.kv_offload.base import LoadStoreSpec - -# a single transfer spec (src_blocks_spec, dst_blocks_spec) -TransferSpec = tuple[LoadStoreSpec, LoadStoreSpec] -# transfers are forwarded to workers by (src_medium, dst_medium) -TransferType = tuple[str, str] - -logger = init_logger(__name__) - - -@dataclass -class TransferResult: - job_id: int - success: bool - transfer_size: int | None = None # Size in bytes - transfer_time: float | None = None - transfer_type: TransferType | None = None - - -class OffloadingHandler(ABC): - """ - OffloadingHandler class for managing asynchronous KV data transfers - - This class runs in the worker. - It kicks off async KV data transfer requests, and allows - collecting back completion statuses. - - The class provides the following primitives: - transfer_async() - kicks off a new transfer job - get_finished() - returns a list of newly finished job IDs. - """ - - @abstractmethod - def transfer_async(self, job_id: int, spec: TransferSpec) -> bool: - """ - Initiates an asynchronous transfer of KV data. - - Args: - job_id: a unique ID that will be used when notifying back on - transfer completion. - spec: the (src, dst) spec of the KV data transfer. - - Returns: - True if transfer was submitted successfully. - """ - pass - - @abstractmethod - def get_finished(self) -> list[TransferResult]: - """ - Get transfers finished since last call. - - Returns: - A list of (job_id, success) of transfers. - """ - pass - - @abstractmethod - def wait(self, job_ids: set[int]) -> None: - """ - Wait for jobs to finish (blocking). - Args: - job_ids: The set of job IDs to wait for. - """ - - def shutdown(self) -> None: - """Shutdown the handler and release any resources.""" - return - - -class OffloadingWorker: - """ - OffloadingWorker class for managing asynchronous KV data transfers - using multiple OffloadingHandlers - - This class runs in the worker. - It kicks off async KV data transfer requests, by delegating - to one of its registered OffloadingHandlers, based on the transfer type. - - The class provides the following primitives: - register_handler() - registers a new handler to handle - a specific transfer type - transfer_async() - kicks off a new transfer job - using one of the registered handlers. - get_finished() - returns a list of newly finished job IDs - from all handlers. - """ - - def __init__(self): - self.handlers: set[OffloadingHandler] = set() - self.transfer_type_to_handler: dict[TransferType, OffloadingHandler] = {} - - def register_handler( - self, - src_cls: type[LoadStoreSpec], - dst_cls: type[LoadStoreSpec], - handler: OffloadingHandler, - ) -> None: - """ - Registers a new handler. - - Args: - src_cls: the source type of transfers handled by this handler. - dst_cls: the destination type of transfers handled by this handler. - handler: the handler that will handle transfers. - """ - transfer_type = (src_cls.medium(), dst_cls.medium()) - assert transfer_type not in self.transfer_type_to_handler - self.handlers.add(handler) - self.transfer_type_to_handler[transfer_type] = handler - - def transfer_async(self, job_id: int, spec: TransferSpec) -> bool: - """ - Initiates an asynchronous transfer of KV data. - - Args: - job_id: a unique ID that will be used when notifying back on - transfer completion. - spec: the (src, dst) spec of the KV data transfer. - - Returns: - True if transfer was submitted successfully. - """ - src, dst = spec - transfer_type = (src.medium(), dst.medium()) - handler = self.transfer_type_to_handler.get(transfer_type) - assert handler is not None - try: - success = handler.transfer_async(job_id, spec) - except Exception as e: - logger.warning( - "Exception in %r transfer %d: %r", - transfer_type, - job_id, - e, - exc_info=True, - ) - return False - - if not success: - logger.warning("Failed to submit %r transfer %d", transfer_type, job_id) - else: - logger.debug("Submitted %r transfer %d: %r", transfer_type, job_id, spec) - return success - - def get_finished(self) -> list[TransferResult]: - """ - Get transfers finished since last call. - - Returns: - A list of TransferResults - """ - finished = [] - for handler in self.handlers: - finished.extend(handler.get_finished()) - return finished - - def wait(self, job_ids: set[int]) -> None: - """ - Wait for jobs to finish (blocking). - - Args: - job_ids: The set of job IDs to wait for. - """ - for handler in self.handlers: - handler.wait(job_ids) - - def shutdown(self) -> None: - for handler in self.handlers: - handler.shutdown()