[RL] Release weight-transfer communicators without stopping inference (#61060)

Signed-off-by: aoshen02 <aoshen@inferact.ai>
Signed-off-by: fh3 <a@b>
This commit is contained in:
aoshen02
2026-10-11 21:15:14 +08:00
committed by GitHub
parent c85616efc6
commit bd5bbce6e4
22 changed files with 181 additions and 14 deletions
+12
View File
@@ -828,6 +828,18 @@ impl EngineCoreClient {
Ok(()) Ok(())
} }
/// Release the trainer connection without shutting down inference.
pub async fn shutdown_weight_transfer_engine(&self) -> Result<()> {
self.collective_rpc(
"shutdown_weight_transfer_engine",
None,
Vec::<JsonValue>::new(),
BTreeMap::<String, JsonValue>::new(),
)
.await?;
Ok(())
}
/// Start a weight update for the base model. /// Start a weight update for the base model.
pub async fn start_weight_update(&self) -> Result<()> { pub async fn start_weight_update(&self) -> Result<()> {
self.collective_rpc( self.collective_rpc(
+4
View File
@@ -145,6 +145,10 @@ fn build_router_with_options(
"/init_weight_transfer_engine", "/init_weight_transfer_engine",
post(weight_transfer::init_weight_transfer_engine), post(weight_transfer::init_weight_transfer_engine),
) )
.route(
"/shutdown_weight_transfer_engine",
post(weight_transfer::shutdown_weight_transfer_engine),
)
.route( .route(
"/start_weight_update", "/start_weight_update",
post(weight_transfer::start_weight_update), post(weight_transfer::start_weight_update),
+7 -3
View File
@@ -6708,6 +6708,7 @@ async fn admin_routes_are_hidden_when_dev_mode_is_disabled() {
("POST", "/reset_mm_cache"), ("POST", "/reset_mm_cache"),
("POST", "/reset_encoder_cache"), ("POST", "/reset_encoder_cache"),
("POST", "/init_weight_transfer_engine"), ("POST", "/init_weight_transfer_engine"),
("POST", "/shutdown_weight_transfer_engine"),
("POST", "/start_weight_update"), ("POST", "/start_weight_update"),
("POST", "/start_draft_weight_update"), ("POST", "/start_draft_weight_update"),
("POST", "/update_weights"), ("POST", "/update_weights"),
@@ -6739,7 +6740,7 @@ async fn weight_transfer_routes_support_the_http_training_lifecycle() {
let (mut app, engine_task) = test_admin_app_with_engine_script(|dealer, push| { let (mut app, engine_task) = test_admin_app_with_engine_script(|dealer, push| {
boxed_test_future(async move { boxed_test_future(async move {
let mut calls = Vec::new(); let mut calls = Vec::new();
for _ in 0..10 { for _ in 0..11 {
let utility = recv_engine_message(dealer).await; let utility = recv_engine_message(dealer).await;
assert_eq!(utility[0].as_ref(), &[0x03]); assert_eq!(utility[0].as_ref(), &[0x03]);
let payload = decode_value(&utility[1]).expect("decode utility"); let payload = decode_value(&utility[1]).expect("decode utility");
@@ -6769,6 +6770,7 @@ async fn weight_transfer_routes_support_the_http_training_lifecycle() {
"collective_rpc [\"finish_weight_update\",null,[],{}]", "collective_rpc [\"finish_weight_update\",null,[],{}]",
"set_weight_version [\"step-2\"]", "set_weight_version [\"step-2\"]",
"get_weight_version []", "get_weight_version []",
"collective_rpc [\"shutdown_weight_transfer_engine\",null,[],{}]",
] ]
"#]].assert_debug_eq(&calls); "#]].assert_debug_eq(&calls);
}) })
@@ -6817,6 +6819,7 @@ async fn weight_transfer_routes_support_the_http_training_lifecycle() {
Some(json!({"new_version": "step-2"})), Some(json!({"new_version": "step-2"})),
), ),
("GET", "/weight_info", None), ("GET", "/weight_info", None),
("POST", "/shutdown_weight_transfer_engine", None),
] { ] {
let mut request = Request::builder().method(method).uri(path); let mut request = Request::builder().method(method).uri(path);
let body = match body { let body = match body {
@@ -6838,7 +6841,7 @@ async fn weight_transfer_routes_support_the_http_training_lifecycle() {
} }
engine_task.await.expect("mock engine task"); engine_task.await.expect("mock engine task");
let metrics_after = METRICS.render().unwrap(); let metrics_after = METRICS.render().unwrap();
for operation in ["update", "finish"] { for (operation, count) in [("update", 2.0), ("finish", 2.0), ("shutdown", 1.0)] {
assert_eq!( assert_eq!(
metric_delta( metric_delta(
&metrics_before, &metrics_before,
@@ -6846,7 +6849,7 @@ async fn weight_transfer_routes_support_the_http_training_lifecycle() {
"vllm:rl_weight_update_operation_duration_seconds_count", "vllm:rl_weight_update_operation_duration_seconds_count",
Some(&format!("operation=\"{operation}\"")), Some(&format!("operation=\"{operation}\"")),
), ),
2.0 count
); );
assert_eq!( assert_eq!(
metric_value( metric_value(
@@ -6868,6 +6871,7 @@ async fn weight_transfer_routes_support_the_http_training_lifecycle() {
"{\"message\":\"Weight update finished\"}", "{\"message\":\"Weight update finished\"}",
"{\"success\":true,\"new_version\":\"step-2\"}", "{\"success\":true,\"new_version\":\"step-2\"}",
"{\"weight_version\":\"step-2\"}", "{\"weight_version\":\"step-2\"}",
"{\"message\":\"Weight transfer shut down\"}",
] ]
"#]] "#]]
.assert_debug_eq(&responses); .assert_debug_eq(&responses);
@@ -75,6 +75,22 @@ pub async fn init_weight_transfer_engine(
})) }))
} }
/// Release the trainer connection without shutting down inference.
pub async fn shutdown_weight_transfer_engine(
State(state): State<Arc<AppState>>,
) -> Result<Json<MessageResponse>, ApiError> {
let _recorder = METRICS.api_server.record_weight_operation("shutdown");
state
.engine_core_client()
.shutdown_weight_transfer_engine()
.await
.map_err(|error| utility_call_error("shutdown_weight_transfer_engine", error))?;
Ok(Json(MessageResponse {
message: "Weight transfer shut down",
}))
}
/// Start a weight update transaction. /// Start a weight update transaction.
pub async fn start_weight_update( pub async fn start_weight_update(
State(state): State<Arc<AppState>>, State(state): State<Arc<AppState>>,
+28
View File
@@ -218,6 +218,27 @@ class TestSparseNCCLWeightTransferUpdateInfoValidation:
# --- Unit Tests: Engine Parsing --- # --- Unit Tests: Engine Parsing ---
@pytest.mark.parametrize(
"engine_type",
[
NCCLWeightTransferEngine,
NCCLTrainerWeightTransferEngine,
SparseNCCLWeightTransferEngine,
SparseNCCLTrainerWeightTransferEngine,
],
)
def test_nccl_shutdown_destroys_communicator_once(engine_type):
engine = object.__new__(engine_type)
group = MagicMock()
engine.model_update_group = group
engine.shutdown()
engine.shutdown()
group.destroy.assert_called_once_with()
assert engine.model_update_group is None
class TestNCCLEngineParsing: class TestNCCLEngineParsing:
"""Test NCCLWeightTransferEngine parsing methods.""" """Test NCCLWeightTransferEngine parsing methods."""
@@ -1292,6 +1313,9 @@ class TestTrainerClients:
handle.finish_weight_update.remote.assert_called_once_with() handle.finish_weight_update.remote.assert_called_once_with()
handle.update_weight_version.remote.assert_called_once_with("step-42") handle.update_weight_version.remote.assert_called_once_with("step-42")
client.shutdown_weight_transfer_engine()
handle.shutdown_weight_transfer_engine.remote.assert_called_once_with()
def test_http_client_pickles_ipc_handles_for_json(self, monkeypatch): def test_http_client_pickles_ipc_handles_for_json(self, monkeypatch):
"""HTTP update_weights must encode raw ipc_handles as a base64 pickle.""" """HTTP update_weights must encode raw ipc_handles as a base64 pickle."""
captured = {} captured = {}
@@ -1322,6 +1346,7 @@ class TestTrainerClients:
captured = {} captured = {}
def fake_post(self, path, json=None): def fake_post(self, path, json=None):
captured["path"] = path
captured["json"] = json captured["json"] = json
monkeypatch.setattr(HTTPVLLMWeightSyncClient, "_post", fake_post) monkeypatch.setattr(HTTPVLLMWeightSyncClient, "_post", fake_post)
@@ -1333,6 +1358,9 @@ class TestTrainerClients:
client.finish_weight_update("step-42") client.finish_weight_update("step-42")
assert captured["json"] == {"weight_version": "step-42"} assert captured["json"] == {"weight_version": "step-42"}
client.shutdown_weight_transfer_engine()
assert captured == {"path": "shutdown_weight_transfer_engine", "json": None}
class TestModuleSource: class TestModuleSource:
"""`ModuleSource` metadata vs. materialized iteration (dense, no GPU).""" """`ModuleSource` metadata vs. materialized iteration (dense, no GPU)."""
+25 -1
View File
@@ -46,7 +46,10 @@ from vllm.distributed.weight_transfer.m2n_source import (
mesh_from_tensor, mesh_from_tensor,
placements_from_tensor, placements_from_tensor,
) )
from vllm.distributed.weight_transfer.m2n_trainer import M2NTrainerInitInfo from vllm.distributed.weight_transfer.m2n_trainer import (
M2NTrainerInitInfo,
M2NTrainerWeightTransferEngine,
)
from vllm.model_executor.layers.linear import ( from vllm.model_executor.layers.linear import (
ColumnParallelLinear, ColumnParallelLinear,
RowParallelLinear, RowParallelLinear,
@@ -57,6 +60,27 @@ from vllm.platforms import current_platform
VALID_UID_B64 = base64.b64encode(b"\x00" * 128).decode() VALID_UID_B64 = base64.b64encode(b"\x00" * 128).decode()
@pytest.mark.parametrize(
"engine_type,group_attribute",
[
(M2NWeightTransferEngine, "model_update_group"),
(M2NTrainerWeightTransferEngine, "group"),
],
)
def test_shutdown_destroys_communicator_once(engine_type, group_attribute):
engine = object.__new__(engine_type)
engine._handle = None
engine._executor = None
group = Mock()
setattr(engine, group_attribute, group)
engine.shutdown()
engine.shutdown()
group.destroy.assert_called_once_with()
assert getattr(engine, group_attribute) is None
class TestLayout: class TestLayout:
def test_replicated_splits_nothing(self): def test_replicated_splits_nothing(self):
"""A replicated tensor imposes no divisibility constraint. The shape """A replicated tensor imposes no divisibility constraint. The shape
@@ -89,6 +89,11 @@ def test_invalid_field_is_rejected_before_the_engine(client, path, body):
("/update_weights", {"update_info": {"names": []}}, "update_weights"), ("/update_weights", {"update_info": {"names": []}}, "update_weights"),
("/update_weights", {"update_info": [{}, {}]}, "update_weights"), ("/update_weights", {"update_info": [{}, {}]}, "update_weights"),
("/collective_rpc", {"method": "m"}, "collective_rpc"), ("/collective_rpc", {"method": "m"}, "collective_rpc"),
(
"/shutdown_weight_transfer_engine",
{},
"shutdown_weight_transfer_engine",
),
], ],
) )
def test_valid_body_reaches_the_engine(client, path, body, engine_call): def test_valid_body_reaches_the_engine(client, path, body, engine_call):
@@ -169,6 +169,13 @@ def test_init_weight_transfer_engine_calls_engine(vllm_runner):
assert called, "init_transfer_engine should have been called" assert called, "init_transfer_engine should have been called"
assert param == "hello", f"Expected 'hello', got {param}" assert param == "hello", f"Expected 'hello', got {param}"
llm.shutdown_weight_transfer_engine()
assert all(
llm.collective_rpc(
lambda worker: worker.weight_transfer_engine.shutdown_called
)
)
@create_new_process_for_each_test() @create_new_process_for_each_test()
def test_update_weights_calls_engine(vllm_runner): def test_update_weights_calls_engine(vllm_runner):
@@ -75,6 +75,7 @@ def test_record_releases_in_flight_and_observes_duration(registry, fail):
"operation,route,body", "operation,route,body",
[ [
("init", api_router.init_weight_transfer_engine, {"init_info": {}}), ("init", api_router.init_weight_transfer_engine, {"init_info": {}}),
("shutdown", api_router.shutdown_weight_transfer_engine, None),
("start", api_router.start_weight_update, None), ("start", api_router.start_weight_update, None),
("start_draft", api_router.start_draft_weight_update, None), ("start_draft", api_router.start_draft_weight_update, None),
("update", api_router.update_weights, {"update_info": {}}), ("update", api_router.update_weights, {"update_info": {}}),
@@ -24,6 +24,7 @@ class _RecordingEngine:
self.raise_on_update = raise_on_update self.raise_on_update = raise_on_update
self.started = False self.started = False
self.finished = False self.finished = False
self.closed = False
self.reset_count = 0 self.reset_count = 0
self.supports_draft_weight_update = False self.supports_draft_weight_update = False
self.update_calls: list[dict] = [] self.update_calls: list[dict] = []
@@ -49,6 +50,9 @@ class _RecordingEngine:
def reset_weight_update_target(self) -> None: def reset_weight_update_target(self) -> None:
self.reset_count += 1 self.reset_count += 1
def shutdown(self) -> None:
self.closed = True
class _RecordingModelRunner: class _RecordingModelRunner:
def __init__(self) -> None: def __init__(self) -> None:
@@ -82,6 +86,25 @@ def test_reload_weights_sets_current_config():
assert model_runner.seen_config is worker.vllm_config assert model_runner.seen_config is worker.vllm_config
@pytest.mark.parametrize(
"enabled,active", [(True, False), (True, True), (False, False)]
)
def test_shutdown_weight_transfer_engine_preserves_inference(enabled, active):
engine = _RecordingEngine() if enabled else None
worker = _make_worker(engine)
worker._weight_update_active = active
model_runner = worker.model_runner
if active:
with pytest.raises(RuntimeError, match="during a weight update"):
worker.shutdown_weight_transfer_engine()
else:
worker.shutdown_weight_transfer_engine()
if engine is not None:
assert engine.closed is not active
assert worker.weight_transfer_engine is engine
assert worker.model_runner is model_runner
def test_reload_parameter_lookup_preserves_lora_module_names(): def test_reload_parameter_lookup_preserves_lora_module_names():
base_layer = nn.Module() base_layer = nn.Module()
qweight = nn.Parameter(torch.ones(1)) qweight = nn.Parameter(torch.ones(1))
+1 -3
View File
@@ -514,9 +514,7 @@ class WeightTransferEngine(ABC, Generic[TInitInfo, TUpdateInfo]):
@abstractmethod @abstractmethod
def shutdown(self) -> None: def shutdown(self) -> None:
"""Shutdown the weight transfer engine. """Release transfer resources without shutting down the inference worker."""
This should be called when the worker is shutting down.
"""
raise NotImplementedError raise NotImplementedError
+9 -1
View File
@@ -60,7 +60,7 @@ class HTTPVLLMWeightSyncClient:
Mirrors `vllm/entrypoints/rl/online/api_router.py`: Mirrors `vllm/entrypoints/rl/online/api_router.py`:
`/init_weight_transfer_engine`, `/start_weight_update`, `/update_weights`, `/init_weight_transfer_engine`, `/start_weight_update`, `/update_weights`,
`/finish_weight_update`. `/finish_weight_update`, `/shutdown_weight_transfer_engine`.
""" """
def __init__(self, base_url: str, timeout: float = 300) -> None: def __init__(self, base_url: str, timeout: float = 300) -> None:
@@ -78,6 +78,9 @@ class HTTPVLLMWeightSyncClient:
def init_weight_transfer_engine(self, init_info: dict[str, Any]) -> None: def init_weight_transfer_engine(self, init_info: dict[str, Any]) -> None:
self._post("init_weight_transfer_engine", {"init_info": init_info}) self._post("init_weight_transfer_engine", {"init_info": init_info})
def shutdown_weight_transfer_engine(self) -> None:
self._post("shutdown_weight_transfer_engine")
def start_weight_update(self) -> None: def start_weight_update(self) -> None:
self._post("start_weight_update") self._post("start_weight_update")
@@ -109,6 +112,11 @@ class RayVLLMWeightSyncClient:
request = WeightTransferInitRequest(init_info=init_info) request = WeightTransferInitRequest(init_info=init_info)
ray.get([h.init_weight_transfer_engine.remote(request) for h in self.handles]) ray.get([h.init_weight_transfer_engine.remote(request) for h in self.handles])
def shutdown_weight_transfer_engine(self) -> None:
import ray
ray.get([h.shutdown_weight_transfer_engine.remote() for h in self.handles])
def start_weight_update(self) -> None: def start_weight_update(self) -> None:
import ray import ray
@@ -449,5 +449,7 @@ class M2NWeightTransferEngine(
torch.accelerator.synchronize() torch.accelerator.synchronize()
self._handle.destroy() self._handle.destroy()
self._handle = None self._handle = None
self.model_update_group = None if self.model_update_group is not None:
self.model_update_group.destroy()
self.model_update_group = None
self._parameter_destinations = [] self._parameter_destinations = []
@@ -410,4 +410,6 @@ class M2NTrainerWeightTransferEngine(TrainerWeightTransferEngine[M2NTrainerInitI
if self._executor is not None: if self._executor is not None:
self._executor.shutdown() self._executor.shutdown()
self._executor = None self._executor = None
self.group = None if self.group is not None:
self.group.destroy()
self.group = None
@@ -229,7 +229,7 @@ class NCCLWeightTransferEngine(
def shutdown(self) -> None: def shutdown(self) -> None:
if self.model_update_group is not None: if self.model_update_group is not None:
# Clean up the communicator by removing the reference self.model_update_group.destroy()
self.model_update_group = None self.model_update_group = None
@@ -456,4 +456,6 @@ class NCCLTrainerWeightTransferEngine(TrainerWeightTransferEngine[NCCLTrainerIni
torch.cuda.current_stream().synchronize() torch.cuda.current_stream().synchronize()
def shutdown(self) -> None: def shutdown(self) -> None:
self.model_update_group = None if self.model_update_group is not None:
self.model_update_group.destroy()
self.model_update_group = None
@@ -186,6 +186,7 @@ class SparseNCCLWeightTransferEngine(
def shutdown(self) -> None: def shutdown(self) -> None:
if self.model_update_group is not None: if self.model_update_group is not None:
self.model_update_group.destroy()
self.model_update_group = None self.model_update_group = None
@@ -371,4 +372,6 @@ class SparseNCCLTrainerWeightTransferEngine(
torch.cuda.current_stream().synchronize() torch.cuda.current_stream().synchronize()
def shutdown(self) -> None: def shutdown(self) -> None:
self.model_update_group = None if self.model_update_group is not None:
self.model_update_group.destroy()
self.model_update_group = None
+4
View File
@@ -296,6 +296,10 @@ class EngineClient(ABC):
"""Initialize weight transfer for RL training.""" """Initialize weight transfer for RL training."""
raise NotImplementedError raise NotImplementedError
async def shutdown_weight_transfer_engine(self) -> None:
"""Release the trainer connection without shutting down inference."""
raise NotImplementedError
async def start_weight_update(self) -> None: async def start_weight_update(self) -> None:
"""Start a new weight update.""" """Start a new weight update."""
raise NotImplementedError raise NotImplementedError
+4
View File
@@ -26,6 +26,10 @@ class RLOfflineMixin(OfflineInferenceMixin):
"init_weight_transfer_engine", kwargs={"init_info": init_info_dict} "init_weight_transfer_engine", kwargs={"init_info": init_info_dict}
) )
def shutdown_weight_transfer_engine(self) -> None:
"""Release the trainer connection without shutting down inference."""
self.llm_engine.collective_rpc("shutdown_weight_transfer_engine")
def start_weight_update(self) -> None: def start_weight_update(self) -> None:
"""Start a new weight update.""" """Start a new weight update."""
self.llm_engine.collective_rpc("start_weight_update") self.llm_engine.collective_rpc("start_weight_update")
+7
View File
@@ -168,6 +168,13 @@ async def init_weight_transfer_engine(
return JSONResponse(content={"message": "Weight transfer initialized"}) return JSONResponse(content={"message": "Weight transfer initialized"})
@router.post("/shutdown_weight_transfer_engine")
async def shutdown_weight_transfer_engine(raw_request: Request):
with weight_operation_metrics().record("shutdown"):
await engine_client(raw_request).shutdown_weight_transfer_engine()
return JSONResponse(content={"message": "Weight transfer shut down"})
@router.post("/start_weight_update") @router.post("/start_weight_update")
async def start_weight_update(raw_request: Request): async def start_weight_update(raw_request: Request):
with weight_operation_metrics().record("start"): with weight_operation_metrics().record("start"):
+1 -1
View File
@@ -10,7 +10,7 @@ from typing import Literal
from prometheus_client import REGISTRY, CollectorRegistry, Gauge, Histogram from prometheus_client import REGISTRY, CollectorRegistry, Gauge, Histogram
Operation = Literal["init", "start", "start_draft", "update", "finish"] Operation = Literal["init", "shutdown", "start", "start_draft", "update", "finish"]
class WeightOperationMetrics: class WeightOperationMetrics:
+4
View File
@@ -1318,6 +1318,10 @@ class AsyncLLM(EngineClient):
"init_weight_transfer_engine", kwargs={"init_info": request.init_info} "init_weight_transfer_engine", kwargs={"init_info": request.init_info}
) )
async def shutdown_weight_transfer_engine(self) -> None:
"""Release the trainer connection without shutting down inference."""
await self.collective_rpc("shutdown_weight_transfer_engine")
async def start_weight_update(self) -> None: async def start_weight_update(self) -> None:
"""Start a new weight update.""" """Start a new weight update."""
await self.collective_rpc("start_weight_update") await self.collective_rpc("start_weight_update")
+9
View File
@@ -1482,6 +1482,15 @@ class Worker(WorkerBase):
typed_init_info = self.weight_transfer_engine.parse_init_info(init_info) typed_init_info = self.weight_transfer_engine.parse_init_info(init_info)
self.weight_transfer_engine.init_transfer_engine(typed_init_info) self.weight_transfer_engine.init_transfer_engine(typed_init_info)
def shutdown_weight_transfer_engine(self) -> None:
"""Release a trainer connection without shutting down inference."""
if self._weight_update_active:
raise RuntimeError(
"Cannot release a trainer connection during a weight update"
)
if self.weight_transfer_engine is not None:
self.weight_transfer_engine.shutdown()
def start_weight_update(self) -> None: def start_weight_update(self) -> None:
"""Start a new weight update session. """Start a new weight update session.