diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/offloading/scheduler.py b/vllm/distributed/kv_transfer/kv_connector/v1/offloading/scheduler.py index 1831bd5770..52e3ce09eb 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/offloading/scheduler.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/offloading/scheduler.py @@ -428,3 +428,6 @@ class OffloadingConnectorScheduler: medium=event.medium, lora_name=None, ) + + def shutdown(self) -> None: + self.manager.shutdown() diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/offloading/worker.py b/vllm/distributed/kv_transfer/kv_connector/v1/offloading/worker.py index 23c62b6eca..cc6d8262c7 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/offloading/worker.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/offloading/worker.py @@ -394,3 +394,13 @@ class OffloadingConnectorWorker: kv_connector_stats = self.kv_connector_stats self.kv_connector_stats = OffloadingConnectorStats() return kv_connector_stats + + def shutdown(self) -> None: + # Drop deferred store jobs: there is no point in submitting + # them during shutdown. + self._unsubmitted_store_jobs.clear() + self._jobs.clear() + self._load_job.clear() + self._store_jobs.clear() + self._finished_reqs_waiting_for_store.clear() + self.worker.shutdown() diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/offloading_connector.py b/vllm/distributed/kv_transfer/kv_connector/v1/offloading_connector.py index 547ee2578a..f11281dcf1 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/offloading_connector.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/offloading_connector.py @@ -64,6 +64,12 @@ class OffloadingConnector(KVConnectorBase_V1): elif role == KVConnectorRole.WORKER: self.connector_worker = OffloadingConnectorWorker(spec) + def shutdown(self) -> None: + if self.connector_worker is not None: + self.connector_worker.shutdown() + if self.connector_scheduler is not None: + self.connector_scheduler.shutdown() + def register_kv_caches(self, kv_caches: dict[str, torch.Tensor]): assert self.connector_worker is not None self.connector_worker.register_kv_caches(kv_caches) diff --git a/vllm/v1/kv_offload/abstract.py b/vllm/v1/kv_offload/abstract.py index e169c5bb4d..5b3278403a 100644 --- a/vllm/v1/kv_offload/abstract.py +++ b/vllm/v1/kv_offload/abstract.py @@ -178,3 +178,7 @@ class OffloadingManager(ABC): New OffloadingEvents collected since the last call. """ return () + + def shutdown(self) -> None: + """Shutdown the manager and release any resources.""" + return diff --git a/vllm/v1/kv_offload/worker/cpu_gpu.py b/vllm/v1/kv_offload/worker/cpu_gpu.py index cd0136d488..e14c85549b 100644 --- a/vllm/v1/kv_offload/worker/cpu_gpu.py +++ b/vllm/v1/kv_offload/worker/cpu_gpu.py @@ -274,6 +274,14 @@ class SingleDirectionOffloadingHandler(OffloadingHandler): if event is not None: event.synchronize() + def shutdown(self) -> None: + while self._transfers: + transfer = self._transfers.popleft() + transfer.end_event.synchronize() + self._transfer_events.clear() + self._stream_pool.clear() + self._event_pool.clear() + class CpuGpuOffloadingHandlers: def __init__( diff --git a/vllm/v1/kv_offload/worker/worker.py b/vllm/v1/kv_offload/worker/worker.py index efb31c2a0e..2a47cc8e9d 100644 --- a/vllm/v1/kv_offload/worker/worker.py +++ b/vllm/v1/kv_offload/worker/worker.py @@ -69,6 +69,10 @@ class OffloadingHandler(ABC): job_ids: The set of job IDs to wait for. """ + def shutdown(self) -> None: + """Shutdown the handler and release any resources.""" + return + class OffloadingWorker: """ @@ -166,3 +170,7 @@ class OffloadingWorker: """ for handler in self.handlers: handler.wait(job_ids) + + def shutdown(self) -> None: + for handler in self.handlers: + handler.shutdown()