[RL] Release weight-transfer communicators without stopping inference (#61060)
Signed-off-by: aoshen02 <aoshen@inferact.ai> Signed-off-by: fh3 <a@b>
This commit is contained in:
@@ -514,9 +514,7 @@ class WeightTransferEngine(ABC, Generic[TInitInfo, TUpdateInfo]):
|
||||
|
||||
@abstractmethod
|
||||
def shutdown(self) -> None:
|
||||
"""Shutdown the weight transfer engine.
|
||||
This should be called when the worker is shutting down.
|
||||
"""
|
||||
"""Release transfer resources without shutting down the inference worker."""
|
||||
raise NotImplementedError
|
||||
|
||||
|
||||
|
||||
@@ -60,7 +60,7 @@ class HTTPVLLMWeightSyncClient:
|
||||
|
||||
Mirrors `vllm/entrypoints/rl/online/api_router.py`:
|
||||
`/init_weight_transfer_engine`, `/start_weight_update`, `/update_weights`,
|
||||
`/finish_weight_update`.
|
||||
`/finish_weight_update`, `/shutdown_weight_transfer_engine`.
|
||||
"""
|
||||
|
||||
def __init__(self, base_url: str, timeout: float = 300) -> None:
|
||||
@@ -78,6 +78,9 @@ class HTTPVLLMWeightSyncClient:
|
||||
def init_weight_transfer_engine(self, init_info: dict[str, Any]) -> None:
|
||||
self._post("init_weight_transfer_engine", {"init_info": init_info})
|
||||
|
||||
def shutdown_weight_transfer_engine(self) -> None:
|
||||
self._post("shutdown_weight_transfer_engine")
|
||||
|
||||
def start_weight_update(self) -> None:
|
||||
self._post("start_weight_update")
|
||||
|
||||
@@ -109,6 +112,11 @@ class RayVLLMWeightSyncClient:
|
||||
request = WeightTransferInitRequest(init_info=init_info)
|
||||
ray.get([h.init_weight_transfer_engine.remote(request) for h in self.handles])
|
||||
|
||||
def shutdown_weight_transfer_engine(self) -> None:
|
||||
import ray
|
||||
|
||||
ray.get([h.shutdown_weight_transfer_engine.remote() for h in self.handles])
|
||||
|
||||
def start_weight_update(self) -> None:
|
||||
import ray
|
||||
|
||||
|
||||
@@ -449,5 +449,7 @@ class M2NWeightTransferEngine(
|
||||
torch.accelerator.synchronize()
|
||||
self._handle.destroy()
|
||||
self._handle = None
|
||||
self.model_update_group = None
|
||||
if self.model_update_group is not None:
|
||||
self.model_update_group.destroy()
|
||||
self.model_update_group = None
|
||||
self._parameter_destinations = []
|
||||
|
||||
@@ -410,4 +410,6 @@ class M2NTrainerWeightTransferEngine(TrainerWeightTransferEngine[M2NTrainerInitI
|
||||
if self._executor is not None:
|
||||
self._executor.shutdown()
|
||||
self._executor = None
|
||||
self.group = None
|
||||
if self.group is not None:
|
||||
self.group.destroy()
|
||||
self.group = None
|
||||
|
||||
@@ -229,7 +229,7 @@ class NCCLWeightTransferEngine(
|
||||
|
||||
def shutdown(self) -> None:
|
||||
if self.model_update_group is not None:
|
||||
# Clean up the communicator by removing the reference
|
||||
self.model_update_group.destroy()
|
||||
self.model_update_group = None
|
||||
|
||||
|
||||
@@ -456,4 +456,6 @@ class NCCLTrainerWeightTransferEngine(TrainerWeightTransferEngine[NCCLTrainerIni
|
||||
torch.cuda.current_stream().synchronize()
|
||||
|
||||
def shutdown(self) -> None:
|
||||
self.model_update_group = None
|
||||
if self.model_update_group is not None:
|
||||
self.model_update_group.destroy()
|
||||
self.model_update_group = None
|
||||
|
||||
@@ -186,6 +186,7 @@ class SparseNCCLWeightTransferEngine(
|
||||
|
||||
def shutdown(self) -> None:
|
||||
if self.model_update_group is not None:
|
||||
self.model_update_group.destroy()
|
||||
self.model_update_group = None
|
||||
|
||||
|
||||
@@ -371,4 +372,6 @@ class SparseNCCLTrainerWeightTransferEngine(
|
||||
torch.cuda.current_stream().synchronize()
|
||||
|
||||
def shutdown(self) -> None:
|
||||
self.model_update_group = None
|
||||
if self.model_update_group is not None:
|
||||
self.model_update_group.destroy()
|
||||
self.model_update_group = None
|
||||
|
||||
Reference in New Issue
Block a user