[RL] Release weight-transfer communicators without stopping inference (#61060)

Signed-off-by: aoshen02 <aoshen@inferact.ai>
Signed-off-by: fh3 <a@b>
This commit is contained in:
aoshen02
2026-10-11 21:15:14 +08:00
committed by GitHub
parent c85616efc6
commit bd5bbce6e4
22 changed files with 181 additions and 14 deletions
+12
View File
@@ -828,6 +828,18 @@ impl EngineCoreClient {
Ok(())
}
/// Release the trainer connection without shutting down inference.
pub async fn shutdown_weight_transfer_engine(&self) -> Result<()> {
self.collective_rpc(
"shutdown_weight_transfer_engine",
None,
Vec::<JsonValue>::new(),
BTreeMap::<String, JsonValue>::new(),
)
.await?;
Ok(())
}
/// Start a weight update for the base model.
pub async fn start_weight_update(&self) -> Result<()> {
self.collective_rpc(
+4
View File
@@ -145,6 +145,10 @@ fn build_router_with_options(
"/init_weight_transfer_engine",
post(weight_transfer::init_weight_transfer_engine),
)
.route(
"/shutdown_weight_transfer_engine",
post(weight_transfer::shutdown_weight_transfer_engine),
)
.route(
"/start_weight_update",
post(weight_transfer::start_weight_update),
+7 -3
View File
@@ -6708,6 +6708,7 @@ async fn admin_routes_are_hidden_when_dev_mode_is_disabled() {
("POST", "/reset_mm_cache"),
("POST", "/reset_encoder_cache"),
("POST", "/init_weight_transfer_engine"),
("POST", "/shutdown_weight_transfer_engine"),
("POST", "/start_weight_update"),
("POST", "/start_draft_weight_update"),
("POST", "/update_weights"),
@@ -6739,7 +6740,7 @@ async fn weight_transfer_routes_support_the_http_training_lifecycle() {
let (mut app, engine_task) = test_admin_app_with_engine_script(|dealer, push| {
boxed_test_future(async move {
let mut calls = Vec::new();
for _ in 0..10 {
for _ in 0..11 {
let utility = recv_engine_message(dealer).await;
assert_eq!(utility[0].as_ref(), &[0x03]);
let payload = decode_value(&utility[1]).expect("decode utility");
@@ -6769,6 +6770,7 @@ async fn weight_transfer_routes_support_the_http_training_lifecycle() {
"collective_rpc [\"finish_weight_update\",null,[],{}]",
"set_weight_version [\"step-2\"]",
"get_weight_version []",
"collective_rpc [\"shutdown_weight_transfer_engine\",null,[],{}]",
]
"#]].assert_debug_eq(&calls);
})
@@ -6817,6 +6819,7 @@ async fn weight_transfer_routes_support_the_http_training_lifecycle() {
Some(json!({"new_version": "step-2"})),
),
("GET", "/weight_info", None),
("POST", "/shutdown_weight_transfer_engine", None),
] {
let mut request = Request::builder().method(method).uri(path);
let body = match body {
@@ -6838,7 +6841,7 @@ async fn weight_transfer_routes_support_the_http_training_lifecycle() {
}
engine_task.await.expect("mock engine task");
let metrics_after = METRICS.render().unwrap();
for operation in ["update", "finish"] {
for (operation, count) in [("update", 2.0), ("finish", 2.0), ("shutdown", 1.0)] {
assert_eq!(
metric_delta(
&metrics_before,
@@ -6846,7 +6849,7 @@ async fn weight_transfer_routes_support_the_http_training_lifecycle() {
"vllm:rl_weight_update_operation_duration_seconds_count",
Some(&format!("operation=\"{operation}\"")),
),
2.0
count
);
assert_eq!(
metric_value(
@@ -6868,6 +6871,7 @@ async fn weight_transfer_routes_support_the_http_training_lifecycle() {
"{\"message\":\"Weight update finished\"}",
"{\"success\":true,\"new_version\":\"step-2\"}",
"{\"weight_version\":\"step-2\"}",
"{\"message\":\"Weight transfer shut down\"}",
]
"#]]
.assert_debug_eq(&responses);
@@ -75,6 +75,22 @@ pub async fn init_weight_transfer_engine(
}))
}
/// Release the trainer connection without shutting down inference.
pub async fn shutdown_weight_transfer_engine(
State(state): State<Arc<AppState>>,
) -> Result<Json<MessageResponse>, ApiError> {
let _recorder = METRICS.api_server.record_weight_operation("shutdown");
state
.engine_core_client()
.shutdown_weight_transfer_engine()
.await
.map_err(|error| utility_call_error("shutdown_weight_transfer_engine", error))?;
Ok(Json(MessageResponse {
message: "Weight transfer shut down",
}))
}
/// Start a weight update transaction.
pub async fn start_weight_update(
State(state): State<Arc<AppState>>,