[P/D] Add a shutdown method to the Connector API (#22699)

Signed-off-by: chaunceyjiang <chaunceyjiang@gmail.com>
This commit is contained in:
Chauncey
2025-09-07 23:07:00 -07:00
committed by GitHub
parent 8c892b1831
commit 61aa4b2901
10 changed files with 52 additions and 12 deletions
+3
View File
@@ -601,6 +601,9 @@ class Worker(WorkerBase):
self.model_runner.save_tensorized_model(
tensorizer_config=tensorizer_config, )
def shutdown(self) -> None:
self.model_runner.ensure_kv_transfer_shutdown()
def init_worker_distributed_environment(
vllm_config: VllmConfig,
@@ -9,7 +9,8 @@ from typing import Generator # noqa: UP035
from typing import TYPE_CHECKING, Optional
from vllm.config import VllmConfig
from vllm.distributed.kv_transfer import (get_kv_transfer_group,
from vllm.distributed.kv_transfer import (ensure_kv_transfer_shutdown,
get_kv_transfer_group,
has_kv_transfer_group)
from vllm.distributed.kv_transfer.kv_connector.base import KVConnectorBase
from vllm.forward_context import get_forward_context, set_forward_context
@@ -42,6 +43,11 @@ class KVConnectorModelRunnerMixin:
# Do this here to save a collective_rpc.
kv_connector.start_load_kv(get_forward_context())
@staticmethod
def ensure_kv_transfer_shutdown() -> None:
if has_kv_transfer_group():
ensure_kv_transfer_shutdown()
@staticmethod
def maybe_wait_for_kv_save() -> None:
if has_kv_transfer_group():
+3
View File
@@ -330,6 +330,9 @@ class TPUWorker:
ensure_kv_transfer_initialized(vllm_config)
def shutdown(self) -> None:
self.model_runner.ensure_kv_transfer_shutdown()
if USE_TPU_COMMONS:
from tpu_commons.worker import TPUWorker as TPUCommonsWorker