[KV Offload] Implement shutdown() in OffloadingConnector and related classes (#39182)

Signed-off-by: Ronen Schaffer <ronen.schaffer@ibm.com>
This commit is contained in:
Ronen Schaffer
2026-04-10 08:06:22 +03:00
committed by GitHub
parent c48b2b83bd
commit a5b17fba8f
6 changed files with 39 additions and 0 deletions
@@ -428,3 +428,6 @@ class OffloadingConnectorScheduler:
medium=event.medium,
lora_name=None,
)
def shutdown(self) -> None:
self.manager.shutdown()
@@ -394,3 +394,13 @@ class OffloadingConnectorWorker:
kv_connector_stats = self.kv_connector_stats
self.kv_connector_stats = OffloadingConnectorStats()
return kv_connector_stats
def shutdown(self) -> None:
# Drop deferred store jobs: there is no point in submitting
# them during shutdown.
self._unsubmitted_store_jobs.clear()
self._jobs.clear()
self._load_job.clear()
self._store_jobs.clear()
self._finished_reqs_waiting_for_store.clear()
self.worker.shutdown()
@@ -64,6 +64,12 @@ class OffloadingConnector(KVConnectorBase_V1):
elif role == KVConnectorRole.WORKER:
self.connector_worker = OffloadingConnectorWorker(spec)
def shutdown(self) -> None:
if self.connector_worker is not None:
self.connector_worker.shutdown()
if self.connector_scheduler is not None:
self.connector_scheduler.shutdown()
def register_kv_caches(self, kv_caches: dict[str, torch.Tensor]):
assert self.connector_worker is not None
self.connector_worker.register_kv_caches(kv_caches)