[KV Offload] Replace OffloadingHandler with OffloadingWorker (#45053)
Signed-off-by: Martin Hickey <martin.hickey@ie.ibm.com>
This commit is contained in:
@@ -98,7 +98,7 @@ def _make_worker(kv_cache_config: KVCacheConfig):
|
||||
spec = MagicMock(spec=OffloadingSpec)
|
||||
spec.kv_cache_config = kv_cache_config
|
||||
spec.vllm_config = MagicMock()
|
||||
spec.get_handlers.return_value = iter([])
|
||||
spec.get_worker.return_value = MagicMock()
|
||||
|
||||
worker = OffloadingConnectorWorker(spec=spec)
|
||||
worker.worker = MagicMock()
|
||||
@@ -279,7 +279,7 @@ def test_register_kv_caches(backend):
|
||||
worker, spec = _make_worker(kv_cache_config)
|
||||
worker.register_kv_caches(kv_caches)
|
||||
|
||||
canonical = spec.get_handlers.call_args[0][0]
|
||||
canonical = spec.get_worker.call_args[0][0]
|
||||
assert isinstance(canonical, CanonicalKVCaches)
|
||||
|
||||
# -- Expected block tensors ----------------------------------------------
|
||||
@@ -422,7 +422,7 @@ def test_register_kv_caches_uniform_type(backend):
|
||||
worker, spec = _make_worker(kv_cache_config)
|
||||
worker.register_kv_caches(kv_caches)
|
||||
|
||||
canonical = spec.get_handlers.call_args[0][0]
|
||||
canonical = spec.get_worker.call_args[0][0]
|
||||
assert isinstance(canonical, CanonicalKVCaches)
|
||||
|
||||
for block_tensor in canonical.tensors:
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from collections.abc import Callable, Iterable, Iterator
|
||||
from collections.abc import Callable, Iterable
|
||||
from dataclasses import dataclass
|
||||
from typing import Any
|
||||
from unittest.mock import MagicMock
|
||||
@@ -43,19 +43,17 @@ from vllm.v1.kv_cache_interface import (
|
||||
KVCacheGroupSpec,
|
||||
)
|
||||
from vllm.v1.kv_offload.base import (
|
||||
CanonicalKVCaches,
|
||||
GPULoadStoreSpec,
|
||||
LoadStoreSpec,
|
||||
OffloadingManager,
|
||||
OffloadingSpec,
|
||||
OffloadingWorker,
|
||||
OffloadKey,
|
||||
PrepareStoreOutput,
|
||||
RequestOffloadingContext,
|
||||
make_offload_key,
|
||||
)
|
||||
from vllm.v1.kv_offload.worker.worker import (
|
||||
OffloadingHandler,
|
||||
TransferResult,
|
||||
TransferSpec,
|
||||
make_offload_key,
|
||||
)
|
||||
from vllm.v1.request import Request
|
||||
from vllm.v1.structured_output import StructuredOutputManager
|
||||
@@ -81,9 +79,9 @@ class MockLoadStoreSpec(LoadStoreSpec):
|
||||
return repr(self.offload_keys)
|
||||
|
||||
|
||||
class MockOffloadingHandler(OffloadingHandler):
|
||||
class MockOffloadingWorker(OffloadingWorker):
|
||||
def __init__(self):
|
||||
self.transfer_specs: dict[int, TransferSpec] = {}
|
||||
self.transfer_specs: dict[int, tuple[LoadStoreSpec, LoadStoreSpec]] = {}
|
||||
self.completed_transfers: list[TransferResult] = []
|
||||
self.waiting_jobs: set[int] = set()
|
||||
self.completed_jobs: list[int] = []
|
||||
@@ -94,8 +92,17 @@ class MockOffloadingHandler(OffloadingHandler):
|
||||
self.completed_transfers = []
|
||||
return finished
|
||||
|
||||
def transfer_async(self, job_id: int, spec: TransferSpec) -> bool:
|
||||
self.transfer_specs[job_id] = spec
|
||||
def submit_store(
|
||||
self, job_id: int, src_spec: LoadStoreSpec, dst_spec: LoadStoreSpec
|
||||
) -> bool: # type: ignore[override]
|
||||
self.transfer_specs[job_id] = (src_spec, dst_spec)
|
||||
self.waiting_jobs.add(job_id)
|
||||
return True
|
||||
|
||||
def submit_load(
|
||||
self, job_id: int, src_spec: LoadStoreSpec, dst_spec: LoadStoreSpec
|
||||
) -> bool: # type: ignore[override]
|
||||
self.transfer_specs[job_id] = (src_spec, dst_spec)
|
||||
self.waiting_jobs.add(job_id)
|
||||
return True
|
||||
|
||||
@@ -109,7 +116,6 @@ class MockOffloadingHandler(OffloadingHandler):
|
||||
success=True,
|
||||
transfer_size=None,
|
||||
transfer_time=None,
|
||||
transfer_type=None,
|
||||
)
|
||||
self.completed_transfers.append(result)
|
||||
|
||||
@@ -127,21 +133,18 @@ class MockOffloadingSpec(OffloadingSpec):
|
||||
self.manager.prepare_load = lambda keys, req_context: MockLoadStoreSpec(keys)
|
||||
self.manager.lookup.return_value = False
|
||||
self.manager.on_new_request.return_value = RequestOffloadingContext()
|
||||
self.handler = MockOffloadingHandler()
|
||||
self.handler = MockOffloadingWorker()
|
||||
|
||||
def get_manager(self) -> OffloadingManager:
|
||||
return self.manager
|
||||
|
||||
def get_handlers(
|
||||
self, _
|
||||
) -> Iterator[tuple[type[LoadStoreSpec], type[LoadStoreSpec], OffloadingHandler]]:
|
||||
yield GPULoadStoreSpec, MockLoadStoreSpec, self.handler
|
||||
yield MockLoadStoreSpec, GPULoadStoreSpec, self.handler
|
||||
def get_worker(self, _: CanonicalKVCaches) -> OffloadingWorker:
|
||||
return self.handler
|
||||
|
||||
def complete_transfers(self):
|
||||
self.handler.complete_jobs(self.handler.waiting_jobs.copy())
|
||||
|
||||
def get_completed_transfers(self) -> list[TransferSpec]:
|
||||
def get_completed_transfers(self) -> list[tuple[LoadStoreSpec, LoadStoreSpec]]:
|
||||
specs = [
|
||||
self.handler.transfer_specs[job_id]
|
||||
for job_id in self.handler.completed_jobs
|
||||
@@ -149,7 +152,7 @@ class MockOffloadingSpec(OffloadingSpec):
|
||||
self.handler.completed_jobs.clear()
|
||||
return specs
|
||||
|
||||
def get_flushed_transfers(self):
|
||||
def get_flushed_transfers(self) -> list[tuple[LoadStoreSpec, LoadStoreSpec]]:
|
||||
specs = [
|
||||
self.handler.transfer_specs[job_id] for job_id in self.handler.flushed_jobs
|
||||
]
|
||||
@@ -362,8 +365,7 @@ class RequestRunner:
|
||||
self.scheduler.add_request(req)
|
||||
|
||||
def _parse_transfers(self):
|
||||
for transfer_spec in self.offloading_spec.get_flushed_transfers():
|
||||
src_spec, dst_spec = transfer_spec
|
||||
for src_spec, dst_spec in self.offloading_spec.get_flushed_transfers():
|
||||
if isinstance(src_spec, GPULoadStoreSpec):
|
||||
# store flush
|
||||
for block_id in src_spec.block_ids:
|
||||
@@ -375,9 +377,7 @@ class RequestRunner:
|
||||
|
||||
block_size_factor = self.block_size_factor
|
||||
|
||||
for transfer_spec in self.offloading_spec.get_completed_transfers():
|
||||
src_spec, dst_spec = transfer_spec
|
||||
|
||||
for src_spec, dst_spec in self.offloading_spec.get_completed_transfers():
|
||||
if isinstance(src_spec, GPULoadStoreSpec):
|
||||
store = True
|
||||
gpu_spec = src_spec
|
||||
|
||||
@@ -17,7 +17,7 @@ from vllm.v1.kv_offload.base import (
|
||||
GPULoadStoreSpec,
|
||||
)
|
||||
from vllm.v1.kv_offload.cpu.common import CPULoadStoreSpec
|
||||
from vllm.v1.kv_offload.cpu.gpu_worker import CpuGpuOffloadingHandlers
|
||||
from vllm.v1.kv_offload.cpu.gpu_worker import CPUOffloadingWorker
|
||||
from vllm.v1.kv_offload.cpu.shared_offload_region import SharedOffloadRegion
|
||||
|
||||
NUM_GPU_BLOCKS = [64]
|
||||
@@ -103,7 +103,7 @@ def test_transfer(
|
||||
cpu_page_size=cpu_page_size,
|
||||
)
|
||||
|
||||
handlers = CpuGpuOffloadingHandlers(
|
||||
worker = CPUOffloadingWorker(
|
||||
kv_caches=kv_caches,
|
||||
block_size_factor=block_size_factor,
|
||||
num_cpu_blocks=num_cpu_blocks,
|
||||
@@ -130,7 +130,7 @@ def test_transfer(
|
||||
|
||||
# set transfer direction
|
||||
if gpu_to_cpu:
|
||||
handler = handlers.gpu_to_cpu_handler
|
||||
handler = worker._store_handler
|
||||
src_spec = GPULoadStoreSpec(
|
||||
gpu_blocks, group_sizes=(len(gpu_blocks),), block_indices=(blocks_to_skip,)
|
||||
)
|
||||
@@ -138,7 +138,7 @@ def test_transfer(
|
||||
dst_to_src = dict(zip(cpu_blocks_expanded, gpu_blocks))
|
||||
num_dst_sub_blocks = num_gpu_blocks
|
||||
else:
|
||||
handler = handlers.cpu_to_gpu_handler
|
||||
handler = worker._load_handler
|
||||
src_spec = CPULoadStoreSpec(cpu_blocks)
|
||||
dst_spec = GPULoadStoreSpec(
|
||||
gpu_blocks, group_sizes=(len(gpu_blocks),), block_indices=(blocks_to_skip,)
|
||||
@@ -156,23 +156,21 @@ def test_transfer(
|
||||
orig_src_tensors = [x.clone() for x in handler.src_tensors]
|
||||
orig_dst_tensors = [x.clone() for x in handler.dst_tensors]
|
||||
|
||||
# call transfer function
|
||||
# call transfer function via public API
|
||||
start_time = time.time()
|
||||
assert handler.transfer_async(1, (src_spec, dst_spec))
|
||||
if gpu_to_cpu:
|
||||
assert worker.submit_store(1, src_spec, dst_spec)
|
||||
else:
|
||||
assert worker.submit_load(1, src_spec, dst_spec)
|
||||
assert {x.job_id for x in handler._transfers} == {1}
|
||||
|
||||
# wait for transfer to complete
|
||||
end_time = time.time() + 10
|
||||
while time.time() < end_time:
|
||||
finished = handler.get_finished()
|
||||
finished = worker.get_finished()
|
||||
if finished:
|
||||
assert finished[0].job_id == 1
|
||||
assert finished[0].success
|
||||
assert (
|
||||
finished[0].transfer_type == ("GPU", "CPU")
|
||||
if gpu_to_cpu
|
||||
else ("CPU", "GPU")
|
||||
)
|
||||
assert finished[0].transfer_size == (
|
||||
len(gpu_blocks)
|
||||
* sum([x.page_size_bytes for x in handler.kv_cache_groups_data_refs[0]])
|
||||
@@ -208,8 +206,7 @@ def test_transfer(
|
||||
del orig_tensor, tensor, src_tensor, dst_tensor, orig_dst_tensor
|
||||
del src_view, dst_view, orig_dst_view, expected
|
||||
|
||||
handlers.cpu_to_gpu_handler.shutdown()
|
||||
handlers.gpu_to_cpu_handler.shutdown()
|
||||
worker.shutdown()
|
||||
if mmap_region:
|
||||
mmap_region.cleanup()
|
||||
|
||||
@@ -276,7 +273,7 @@ def test_transfer_multi_group(
|
||||
tensors=kv_cache_tensors, group_data_refs=kv_cache_groups_data_refs
|
||||
)
|
||||
|
||||
handlers = CpuGpuOffloadingHandlers(
|
||||
worker = CPUOffloadingWorker(
|
||||
kv_caches=canonical_kv_caches,
|
||||
block_size_factor=block_size_factor,
|
||||
num_cpu_blocks=num_cpu_blocks,
|
||||
@@ -338,7 +335,7 @@ def test_transfer_multi_group(
|
||||
block_indices: list[int] = [0, 0, sub_blocks_to_skip]
|
||||
|
||||
if gpu_to_cpu:
|
||||
handler = handlers.gpu_to_cpu_handler
|
||||
handler = worker._store_handler
|
||||
src_spec = GPULoadStoreSpec(
|
||||
gpu_blocks, group_sizes=group_sizes, block_indices=block_indices
|
||||
)
|
||||
@@ -352,7 +349,7 @@ def test_transfer_multi_group(
|
||||
]
|
||||
num_dst_sub_blocks = num_cpu_blocks * block_size_factor
|
||||
else:
|
||||
handler = handlers.cpu_to_gpu_handler
|
||||
handler = worker._load_handler
|
||||
src_spec = CPULoadStoreSpec(cpu_blocks)
|
||||
dst_spec = GPULoadStoreSpec(
|
||||
gpu_blocks, group_sizes=group_sizes, block_indices=block_indices
|
||||
@@ -375,12 +372,15 @@ def test_transfer_multi_group(
|
||||
orig_src_tensors = [x.clone() for x in handler.src_tensors]
|
||||
orig_dst_tensors = [x.clone() for x in handler.dst_tensors]
|
||||
|
||||
assert handler.transfer_async(1, (src_spec, dst_spec))
|
||||
if gpu_to_cpu:
|
||||
assert worker.submit_store(1, src_spec, dst_spec)
|
||||
else:
|
||||
assert worker.submit_load(1, src_spec, dst_spec)
|
||||
assert {x.job_id for x in handler._transfers} == {1}
|
||||
|
||||
end_time = time.time() + 10
|
||||
while time.time() < end_time:
|
||||
finished = handler.get_finished()
|
||||
finished = worker.get_finished()
|
||||
if finished:
|
||||
assert finished[0].job_id == 1
|
||||
assert finished[0].success
|
||||
@@ -418,5 +418,4 @@ def test_transfer_multi_group(
|
||||
dst_view[dst_sub_block].cpu(), expected.cpu()
|
||||
)
|
||||
|
||||
handlers.cpu_to_gpu_handler.shutdown()
|
||||
handlers.gpu_to_cpu_handler.shutdown()
|
||||
worker.shutdown()
|
||||
|
||||
@@ -1,165 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from vllm.v1.kv_offload.base import LoadStoreSpec
|
||||
from vllm.v1.kv_offload.worker.worker import (
|
||||
OffloadingHandler,
|
||||
OffloadingWorker,
|
||||
TransferResult,
|
||||
TransferSpec,
|
||||
)
|
||||
|
||||
|
||||
class LoadStoreSpec1(LoadStoreSpec):
|
||||
def __init__(
|
||||
self,
|
||||
submit_success: bool = True,
|
||||
async_success: bool = True,
|
||||
exception: bool = False,
|
||||
):
|
||||
self.finished = False
|
||||
self.submit_success = submit_success
|
||||
self.async_success = async_success
|
||||
self.exception = exception
|
||||
|
||||
@staticmethod
|
||||
def medium() -> str:
|
||||
return "1"
|
||||
|
||||
def __repr__(self):
|
||||
return f"{self.medium()}: {id(self)}"
|
||||
|
||||
|
||||
class LoadStoreSpec2(LoadStoreSpec):
|
||||
@staticmethod
|
||||
def medium() -> str:
|
||||
return "2"
|
||||
|
||||
def __repr__(self):
|
||||
return f"{self.medium()}: {id(self)}"
|
||||
|
||||
|
||||
class OffloadingHandler1To2(OffloadingHandler):
|
||||
def __init__(self):
|
||||
self.transfers: dict[int, LoadStoreSpec1] = {}
|
||||
|
||||
def transfer_async(self, job_id: int, spec: TransferSpec) -> bool:
|
||||
src, dst = spec
|
||||
assert isinstance(src, LoadStoreSpec1)
|
||||
assert isinstance(dst, LoadStoreSpec2)
|
||||
|
||||
if src.exception:
|
||||
raise Exception("An expected exception. Don't worry!")
|
||||
if not src.submit_success:
|
||||
return False
|
||||
|
||||
self.transfers[job_id] = src
|
||||
return True
|
||||
|
||||
def get_finished(self) -> list[TransferResult]:
|
||||
finished = []
|
||||
for job_id, spec in list(self.transfers.items()):
|
||||
if spec.finished:
|
||||
finished.append((job_id, spec.async_success))
|
||||
del self.transfers[job_id]
|
||||
return finished
|
||||
|
||||
def wait(self, job_ids: set[int]) -> None:
|
||||
for job_id in job_ids:
|
||||
spec = self.transfers.get(job_id)
|
||||
if spec:
|
||||
assert spec.finished
|
||||
|
||||
|
||||
class OffloadingHandler2To1(OffloadingHandler):
|
||||
def __init__(self):
|
||||
self.transfers: dict[int, LoadStoreSpec1] = {}
|
||||
|
||||
def transfer_async(self, job_id: int, spec: TransferSpec) -> bool:
|
||||
src, dst = spec
|
||||
assert isinstance(src, LoadStoreSpec2)
|
||||
assert isinstance(dst, LoadStoreSpec1)
|
||||
|
||||
self.transfers[job_id] = dst
|
||||
return True
|
||||
|
||||
def get_finished(self) -> list[TransferResult]:
|
||||
finished = []
|
||||
for job_id, spec in list(self.transfers.items()):
|
||||
if spec.finished:
|
||||
finished.append((job_id, spec.async_success))
|
||||
del self.transfers[job_id]
|
||||
return finished
|
||||
|
||||
def wait(self, job_ids: set[int]) -> None:
|
||||
for job_id in job_ids:
|
||||
spec = self.transfers.get(job_id)
|
||||
if spec:
|
||||
assert spec.finished
|
||||
|
||||
|
||||
def test_offloading_worker():
|
||||
"""
|
||||
Tests OffloadingWorker with 2 handlers.
|
||||
One handler performs 1->2 transfers, and the other handles 2->1.
|
||||
"""
|
||||
worker = OffloadingWorker()
|
||||
handler1to2 = OffloadingHandler1To2()
|
||||
handler2to1 = OffloadingHandler2To1()
|
||||
worker.register_handler(LoadStoreSpec1, LoadStoreSpec2, handler1to2)
|
||||
worker.register_handler(LoadStoreSpec2, LoadStoreSpec1, handler2to1)
|
||||
|
||||
# 1st transfer 1->2 (exception)
|
||||
src1 = LoadStoreSpec1(exception=True)
|
||||
dst1 = LoadStoreSpec2()
|
||||
assert not worker.transfer_async(1, (src1, dst1))
|
||||
|
||||
# 2ed transfer 1->2 (failure to submit)
|
||||
src2 = LoadStoreSpec1(submit_success=False)
|
||||
dst2 = LoadStoreSpec2()
|
||||
assert not worker.transfer_async(2, (src2, dst2))
|
||||
|
||||
# 3rd transfer 1->2 (failure)
|
||||
src3 = LoadStoreSpec1(async_success=False)
|
||||
dst3 = LoadStoreSpec2()
|
||||
assert worker.transfer_async(3, (src3, dst3))
|
||||
|
||||
# 4th transfer 1->2 (success)
|
||||
src4 = LoadStoreSpec1()
|
||||
dst4 = LoadStoreSpec2()
|
||||
worker.transfer_async(4, (src4, dst4))
|
||||
assert set(handler1to2.transfers.keys()) == {3, 4}
|
||||
|
||||
# 5th transfer 2->1
|
||||
src5 = LoadStoreSpec2()
|
||||
dst5 = LoadStoreSpec1()
|
||||
worker.transfer_async(5, (src5, dst5))
|
||||
assert set(handler2to1.transfers.keys()) == {5}
|
||||
|
||||
# no transfer completed yet
|
||||
assert worker.get_finished() == []
|
||||
|
||||
# complete 3rd, 4th
|
||||
src3.finished = True
|
||||
src4.finished = True
|
||||
|
||||
# 6th transfer 1->2
|
||||
src6 = LoadStoreSpec1()
|
||||
dst6 = LoadStoreSpec2()
|
||||
worker.transfer_async(6, (src6, dst6))
|
||||
|
||||
# 7th transfer 2->1
|
||||
src7 = LoadStoreSpec2()
|
||||
dst7 = LoadStoreSpec1()
|
||||
worker.transfer_async(7, (src7, dst7))
|
||||
|
||||
# 6th and 7th transfers started
|
||||
assert 6 in handler1to2.transfers
|
||||
assert 7 in handler2to1.transfers
|
||||
|
||||
# verify result of 3rd and 4th transfers
|
||||
assert sorted(worker.get_finished()) == [(3, False), (4, True)]
|
||||
|
||||
# complete 6th and 7th transfers
|
||||
src6.finished = True
|
||||
dst7.finished = True
|
||||
assert sorted(worker.get_finished()) == [(6, True), (7, True)]
|
||||
Reference in New Issue
Block a user