[KV Offload] Implement shutdown() in OffloadingConnector and related classes (#39182)
Signed-off-by: Ronen Schaffer <ronen.schaffer@ibm.com>
This commit is contained in:
@@ -428,3 +428,6 @@ class OffloadingConnectorScheduler:
|
||||
medium=event.medium,
|
||||
lora_name=None,
|
||||
)
|
||||
|
||||
def shutdown(self) -> None:
|
||||
self.manager.shutdown()
|
||||
|
||||
@@ -394,3 +394,13 @@ class OffloadingConnectorWorker:
|
||||
kv_connector_stats = self.kv_connector_stats
|
||||
self.kv_connector_stats = OffloadingConnectorStats()
|
||||
return kv_connector_stats
|
||||
|
||||
def shutdown(self) -> None:
|
||||
# Drop deferred store jobs: there is no point in submitting
|
||||
# them during shutdown.
|
||||
self._unsubmitted_store_jobs.clear()
|
||||
self._jobs.clear()
|
||||
self._load_job.clear()
|
||||
self._store_jobs.clear()
|
||||
self._finished_reqs_waiting_for_store.clear()
|
||||
self.worker.shutdown()
|
||||
|
||||
@@ -64,6 +64,12 @@ class OffloadingConnector(KVConnectorBase_V1):
|
||||
elif role == KVConnectorRole.WORKER:
|
||||
self.connector_worker = OffloadingConnectorWorker(spec)
|
||||
|
||||
def shutdown(self) -> None:
|
||||
if self.connector_worker is not None:
|
||||
self.connector_worker.shutdown()
|
||||
if self.connector_scheduler is not None:
|
||||
self.connector_scheduler.shutdown()
|
||||
|
||||
def register_kv_caches(self, kv_caches: dict[str, torch.Tensor]):
|
||||
assert self.connector_worker is not None
|
||||
self.connector_worker.register_kv_caches(kv_caches)
|
||||
|
||||
Reference in New Issue
Block a user