[KV Offload] Replace OffloadingHandler with OffloadingWorker (#45053)

Signed-off-by: Martin Hickey <martin.hickey@ie.ibm.com>
This commit is contained in:
Martin Hickey
2026-06-24 14:44:24 +03:00
committed by GitHub
parent 70749fdcca
commit f237e16b41
13 changed files with 176 additions and 465 deletions
@@ -98,7 +98,7 @@ def _make_worker(kv_cache_config: KVCacheConfig):
spec = MagicMock(spec=OffloadingSpec)
spec.kv_cache_config = kv_cache_config
spec.vllm_config = MagicMock()
spec.get_handlers.return_value = iter([])
spec.get_worker.return_value = MagicMock()
worker = OffloadingConnectorWorker(spec=spec)
worker.worker = MagicMock()
@@ -279,7 +279,7 @@ def test_register_kv_caches(backend):
worker, spec = _make_worker(kv_cache_config)
worker.register_kv_caches(kv_caches)
canonical = spec.get_handlers.call_args[0][0]
canonical = spec.get_worker.call_args[0][0]
assert isinstance(canonical, CanonicalKVCaches)
# -- Expected block tensors ----------------------------------------------
@@ -422,7 +422,7 @@ def test_register_kv_caches_uniform_type(backend):
worker, spec = _make_worker(kv_cache_config)
worker.register_kv_caches(kv_caches)
canonical = spec.get_handlers.call_args[0][0]
canonical = spec.get_worker.call_args[0][0]
assert isinstance(canonical, CanonicalKVCaches)
for block_tensor in canonical.tensors:
@@ -1,6 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from collections.abc import Callable, Iterable, Iterator
from collections.abc import Callable, Iterable
from dataclasses import dataclass
from typing import Any
from unittest.mock import MagicMock
@@ -43,19 +43,17 @@ from vllm.v1.kv_cache_interface import (
KVCacheGroupSpec,
)
from vllm.v1.kv_offload.base import (
CanonicalKVCaches,
GPULoadStoreSpec,
LoadStoreSpec,
OffloadingManager,
OffloadingSpec,
OffloadingWorker,
OffloadKey,
PrepareStoreOutput,
RequestOffloadingContext,
make_offload_key,
)
from vllm.v1.kv_offload.worker.worker import (
OffloadingHandler,
TransferResult,
TransferSpec,
make_offload_key,
)
from vllm.v1.request import Request
from vllm.v1.structured_output import StructuredOutputManager
@@ -81,9 +79,9 @@ class MockLoadStoreSpec(LoadStoreSpec):
return repr(self.offload_keys)
class MockOffloadingHandler(OffloadingHandler):
class MockOffloadingWorker(OffloadingWorker):
def __init__(self):
self.transfer_specs: dict[int, TransferSpec] = {}
self.transfer_specs: dict[int, tuple[LoadStoreSpec, LoadStoreSpec]] = {}
self.completed_transfers: list[TransferResult] = []
self.waiting_jobs: set[int] = set()
self.completed_jobs: list[int] = []
@@ -94,8 +92,17 @@ class MockOffloadingHandler(OffloadingHandler):
self.completed_transfers = []
return finished
def transfer_async(self, job_id: int, spec: TransferSpec) -> bool:
self.transfer_specs[job_id] = spec
def submit_store(
self, job_id: int, src_spec: LoadStoreSpec, dst_spec: LoadStoreSpec
) -> bool: # type: ignore[override]
self.transfer_specs[job_id] = (src_spec, dst_spec)
self.waiting_jobs.add(job_id)
return True
def submit_load(
self, job_id: int, src_spec: LoadStoreSpec, dst_spec: LoadStoreSpec
) -> bool: # type: ignore[override]
self.transfer_specs[job_id] = (src_spec, dst_spec)
self.waiting_jobs.add(job_id)
return True
@@ -109,7 +116,6 @@ class MockOffloadingHandler(OffloadingHandler):
success=True,
transfer_size=None,
transfer_time=None,
transfer_type=None,
)
self.completed_transfers.append(result)
@@ -127,21 +133,18 @@ class MockOffloadingSpec(OffloadingSpec):
self.manager.prepare_load = lambda keys, req_context: MockLoadStoreSpec(keys)
self.manager.lookup.return_value = False
self.manager.on_new_request.return_value = RequestOffloadingContext()
self.handler = MockOffloadingHandler()
self.handler = MockOffloadingWorker()
def get_manager(self) -> OffloadingManager:
return self.manager
def get_handlers(
self, _
) -> Iterator[tuple[type[LoadStoreSpec], type[LoadStoreSpec], OffloadingHandler]]:
yield GPULoadStoreSpec, MockLoadStoreSpec, self.handler
yield MockLoadStoreSpec, GPULoadStoreSpec, self.handler
def get_worker(self, _: CanonicalKVCaches) -> OffloadingWorker:
return self.handler
def complete_transfers(self):
self.handler.complete_jobs(self.handler.waiting_jobs.copy())
def get_completed_transfers(self) -> list[TransferSpec]:
def get_completed_transfers(self) -> list[tuple[LoadStoreSpec, LoadStoreSpec]]:
specs = [
self.handler.transfer_specs[job_id]
for job_id in self.handler.completed_jobs
@@ -149,7 +152,7 @@ class MockOffloadingSpec(OffloadingSpec):
self.handler.completed_jobs.clear()
return specs
def get_flushed_transfers(self):
def get_flushed_transfers(self) -> list[tuple[LoadStoreSpec, LoadStoreSpec]]:
specs = [
self.handler.transfer_specs[job_id] for job_id in self.handler.flushed_jobs
]
@@ -362,8 +365,7 @@ class RequestRunner:
self.scheduler.add_request(req)
def _parse_transfers(self):
for transfer_spec in self.offloading_spec.get_flushed_transfers():
src_spec, dst_spec = transfer_spec
for src_spec, dst_spec in self.offloading_spec.get_flushed_transfers():
if isinstance(src_spec, GPULoadStoreSpec):
# store flush
for block_id in src_spec.block_ids:
@@ -375,9 +377,7 @@ class RequestRunner:
block_size_factor = self.block_size_factor
for transfer_spec in self.offloading_spec.get_completed_transfers():
src_spec, dst_spec = transfer_spec
for src_spec, dst_spec in self.offloading_spec.get_completed_transfers():
if isinstance(src_spec, GPULoadStoreSpec):
store = True
gpu_spec = src_spec
+20 -21
View File
@@ -17,7 +17,7 @@ from vllm.v1.kv_offload.base import (
GPULoadStoreSpec,
)
from vllm.v1.kv_offload.cpu.common import CPULoadStoreSpec
from vllm.v1.kv_offload.cpu.gpu_worker import CpuGpuOffloadingHandlers
from vllm.v1.kv_offload.cpu.gpu_worker import CPUOffloadingWorker
from vllm.v1.kv_offload.cpu.shared_offload_region import SharedOffloadRegion
NUM_GPU_BLOCKS = [64]
@@ -103,7 +103,7 @@ def test_transfer(
cpu_page_size=cpu_page_size,
)
handlers = CpuGpuOffloadingHandlers(
worker = CPUOffloadingWorker(
kv_caches=kv_caches,
block_size_factor=block_size_factor,
num_cpu_blocks=num_cpu_blocks,
@@ -130,7 +130,7 @@ def test_transfer(
# set transfer direction
if gpu_to_cpu:
handler = handlers.gpu_to_cpu_handler
handler = worker._store_handler
src_spec = GPULoadStoreSpec(
gpu_blocks, group_sizes=(len(gpu_blocks),), block_indices=(blocks_to_skip,)
)
@@ -138,7 +138,7 @@ def test_transfer(
dst_to_src = dict(zip(cpu_blocks_expanded, gpu_blocks))
num_dst_sub_blocks = num_gpu_blocks
else:
handler = handlers.cpu_to_gpu_handler
handler = worker._load_handler
src_spec = CPULoadStoreSpec(cpu_blocks)
dst_spec = GPULoadStoreSpec(
gpu_blocks, group_sizes=(len(gpu_blocks),), block_indices=(blocks_to_skip,)
@@ -156,23 +156,21 @@ def test_transfer(
orig_src_tensors = [x.clone() for x in handler.src_tensors]
orig_dst_tensors = [x.clone() for x in handler.dst_tensors]
# call transfer function
# call transfer function via public API
start_time = time.time()
assert handler.transfer_async(1, (src_spec, dst_spec))
if gpu_to_cpu:
assert worker.submit_store(1, src_spec, dst_spec)
else:
assert worker.submit_load(1, src_spec, dst_spec)
assert {x.job_id for x in handler._transfers} == {1}
# wait for transfer to complete
end_time = time.time() + 10
while time.time() < end_time:
finished = handler.get_finished()
finished = worker.get_finished()
if finished:
assert finished[0].job_id == 1
assert finished[0].success
assert (
finished[0].transfer_type == ("GPU", "CPU")
if gpu_to_cpu
else ("CPU", "GPU")
)
assert finished[0].transfer_size == (
len(gpu_blocks)
* sum([x.page_size_bytes for x in handler.kv_cache_groups_data_refs[0]])
@@ -208,8 +206,7 @@ def test_transfer(
del orig_tensor, tensor, src_tensor, dst_tensor, orig_dst_tensor
del src_view, dst_view, orig_dst_view, expected
handlers.cpu_to_gpu_handler.shutdown()
handlers.gpu_to_cpu_handler.shutdown()
worker.shutdown()
if mmap_region:
mmap_region.cleanup()
@@ -276,7 +273,7 @@ def test_transfer_multi_group(
tensors=kv_cache_tensors, group_data_refs=kv_cache_groups_data_refs
)
handlers = CpuGpuOffloadingHandlers(
worker = CPUOffloadingWorker(
kv_caches=canonical_kv_caches,
block_size_factor=block_size_factor,
num_cpu_blocks=num_cpu_blocks,
@@ -338,7 +335,7 @@ def test_transfer_multi_group(
block_indices: list[int] = [0, 0, sub_blocks_to_skip]
if gpu_to_cpu:
handler = handlers.gpu_to_cpu_handler
handler = worker._store_handler
src_spec = GPULoadStoreSpec(
gpu_blocks, group_sizes=group_sizes, block_indices=block_indices
)
@@ -352,7 +349,7 @@ def test_transfer_multi_group(
]
num_dst_sub_blocks = num_cpu_blocks * block_size_factor
else:
handler = handlers.cpu_to_gpu_handler
handler = worker._load_handler
src_spec = CPULoadStoreSpec(cpu_blocks)
dst_spec = GPULoadStoreSpec(
gpu_blocks, group_sizes=group_sizes, block_indices=block_indices
@@ -375,12 +372,15 @@ def test_transfer_multi_group(
orig_src_tensors = [x.clone() for x in handler.src_tensors]
orig_dst_tensors = [x.clone() for x in handler.dst_tensors]
assert handler.transfer_async(1, (src_spec, dst_spec))
if gpu_to_cpu:
assert worker.submit_store(1, src_spec, dst_spec)
else:
assert worker.submit_load(1, src_spec, dst_spec)
assert {x.job_id for x in handler._transfers} == {1}
end_time = time.time() + 10
while time.time() < end_time:
finished = handler.get_finished()
finished = worker.get_finished()
if finished:
assert finished[0].job_id == 1
assert finished[0].success
@@ -418,5 +418,4 @@ def test_transfer_multi_group(
dst_view[dst_sub_block].cpu(), expected.cpu()
)
handlers.cpu_to_gpu_handler.shutdown()
handlers.gpu_to_cpu_handler.shutdown()
worker.shutdown()
-165
View File
@@ -1,165 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from vllm.v1.kv_offload.base import LoadStoreSpec
from vllm.v1.kv_offload.worker.worker import (
OffloadingHandler,
OffloadingWorker,
TransferResult,
TransferSpec,
)
class LoadStoreSpec1(LoadStoreSpec):
def __init__(
self,
submit_success: bool = True,
async_success: bool = True,
exception: bool = False,
):
self.finished = False
self.submit_success = submit_success
self.async_success = async_success
self.exception = exception
@staticmethod
def medium() -> str:
return "1"
def __repr__(self):
return f"{self.medium()}: {id(self)}"
class LoadStoreSpec2(LoadStoreSpec):
@staticmethod
def medium() -> str:
return "2"
def __repr__(self):
return f"{self.medium()}: {id(self)}"
class OffloadingHandler1To2(OffloadingHandler):
def __init__(self):
self.transfers: dict[int, LoadStoreSpec1] = {}
def transfer_async(self, job_id: int, spec: TransferSpec) -> bool:
src, dst = spec
assert isinstance(src, LoadStoreSpec1)
assert isinstance(dst, LoadStoreSpec2)
if src.exception:
raise Exception("An expected exception. Don't worry!")
if not src.submit_success:
return False
self.transfers[job_id] = src
return True
def get_finished(self) -> list[TransferResult]:
finished = []
for job_id, spec in list(self.transfers.items()):
if spec.finished:
finished.append((job_id, spec.async_success))
del self.transfers[job_id]
return finished
def wait(self, job_ids: set[int]) -> None:
for job_id in job_ids:
spec = self.transfers.get(job_id)
if spec:
assert spec.finished
class OffloadingHandler2To1(OffloadingHandler):
def __init__(self):
self.transfers: dict[int, LoadStoreSpec1] = {}
def transfer_async(self, job_id: int, spec: TransferSpec) -> bool:
src, dst = spec
assert isinstance(src, LoadStoreSpec2)
assert isinstance(dst, LoadStoreSpec1)
self.transfers[job_id] = dst
return True
def get_finished(self) -> list[TransferResult]:
finished = []
for job_id, spec in list(self.transfers.items()):
if spec.finished:
finished.append((job_id, spec.async_success))
del self.transfers[job_id]
return finished
def wait(self, job_ids: set[int]) -> None:
for job_id in job_ids:
spec = self.transfers.get(job_id)
if spec:
assert spec.finished
def test_offloading_worker():
"""
Tests OffloadingWorker with 2 handlers.
One handler performs 1->2 transfers, and the other handles 2->1.
"""
worker = OffloadingWorker()
handler1to2 = OffloadingHandler1To2()
handler2to1 = OffloadingHandler2To1()
worker.register_handler(LoadStoreSpec1, LoadStoreSpec2, handler1to2)
worker.register_handler(LoadStoreSpec2, LoadStoreSpec1, handler2to1)
# 1st transfer 1->2 (exception)
src1 = LoadStoreSpec1(exception=True)
dst1 = LoadStoreSpec2()
assert not worker.transfer_async(1, (src1, dst1))
# 2ed transfer 1->2 (failure to submit)
src2 = LoadStoreSpec1(submit_success=False)
dst2 = LoadStoreSpec2()
assert not worker.transfer_async(2, (src2, dst2))
# 3rd transfer 1->2 (failure)
src3 = LoadStoreSpec1(async_success=False)
dst3 = LoadStoreSpec2()
assert worker.transfer_async(3, (src3, dst3))
# 4th transfer 1->2 (success)
src4 = LoadStoreSpec1()
dst4 = LoadStoreSpec2()
worker.transfer_async(4, (src4, dst4))
assert set(handler1to2.transfers.keys()) == {3, 4}
# 5th transfer 2->1
src5 = LoadStoreSpec2()
dst5 = LoadStoreSpec1()
worker.transfer_async(5, (src5, dst5))
assert set(handler2to1.transfers.keys()) == {5}
# no transfer completed yet
assert worker.get_finished() == []
# complete 3rd, 4th
src3.finished = True
src4.finished = True
# 6th transfer 1->2
src6 = LoadStoreSpec1()
dst6 = LoadStoreSpec2()
worker.transfer_async(6, (src6, dst6))
# 7th transfer 2->1
src7 = LoadStoreSpec2()
dst7 = LoadStoreSpec1()
worker.transfer_async(7, (src7, dst7))
# 6th and 7th transfers started
assert 6 in handler1to2.transfers
assert 7 in handler2to1.transfers
# verify result of 3rd and 4th transfers
assert sorted(worker.get_finished()) == [(3, False), (4, True)]
# complete 6th and 7th transfers
src6.finished = True
dst7.finished = True
assert sorted(worker.get_finished()) == [(6, True), (7, True)]