[RL] Release weight-transfer communicators without stopping inference (#61060)
Signed-off-by: aoshen02 <aoshen@inferact.ai> Signed-off-by: fh3 <a@b>
This commit is contained in:
@@ -26,6 +26,10 @@ class RLOfflineMixin(OfflineInferenceMixin):
|
||||
"init_weight_transfer_engine", kwargs={"init_info": init_info_dict}
|
||||
)
|
||||
|
||||
def shutdown_weight_transfer_engine(self) -> None:
|
||||
"""Release the trainer connection without shutting down inference."""
|
||||
self.llm_engine.collective_rpc("shutdown_weight_transfer_engine")
|
||||
|
||||
def start_weight_update(self) -> None:
|
||||
"""Start a new weight update."""
|
||||
self.llm_engine.collective_rpc("start_weight_update")
|
||||
|
||||
@@ -168,6 +168,13 @@ async def init_weight_transfer_engine(
|
||||
return JSONResponse(content={"message": "Weight transfer initialized"})
|
||||
|
||||
|
||||
@router.post("/shutdown_weight_transfer_engine")
|
||||
async def shutdown_weight_transfer_engine(raw_request: Request):
|
||||
with weight_operation_metrics().record("shutdown"):
|
||||
await engine_client(raw_request).shutdown_weight_transfer_engine()
|
||||
return JSONResponse(content={"message": "Weight transfer shut down"})
|
||||
|
||||
|
||||
@router.post("/start_weight_update")
|
||||
async def start_weight_update(raw_request: Request):
|
||||
with weight_operation_metrics().record("start"):
|
||||
|
||||
@@ -10,7 +10,7 @@ from typing import Literal
|
||||
|
||||
from prometheus_client import REGISTRY, CollectorRegistry, Gauge, Histogram
|
||||
|
||||
Operation = Literal["init", "start", "start_draft", "update", "finish"]
|
||||
Operation = Literal["init", "shutdown", "start", "start_draft", "update", "finish"]
|
||||
|
||||
|
||||
class WeightOperationMetrics:
|
||||
|
||||
Reference in New Issue
Block a user