[RL] Release weight-transfer communicators without stopping inference (#61060)
Signed-off-by: aoshen02 <aoshen@inferact.ai> Signed-off-by: fh3 <a@b>
This commit is contained in:
@@ -145,6 +145,10 @@ fn build_router_with_options(
|
||||
"/init_weight_transfer_engine",
|
||||
post(weight_transfer::init_weight_transfer_engine),
|
||||
)
|
||||
.route(
|
||||
"/shutdown_weight_transfer_engine",
|
||||
post(weight_transfer::shutdown_weight_transfer_engine),
|
||||
)
|
||||
.route(
|
||||
"/start_weight_update",
|
||||
post(weight_transfer::start_weight_update),
|
||||
|
||||
@@ -6708,6 +6708,7 @@ async fn admin_routes_are_hidden_when_dev_mode_is_disabled() {
|
||||
("POST", "/reset_mm_cache"),
|
||||
("POST", "/reset_encoder_cache"),
|
||||
("POST", "/init_weight_transfer_engine"),
|
||||
("POST", "/shutdown_weight_transfer_engine"),
|
||||
("POST", "/start_weight_update"),
|
||||
("POST", "/start_draft_weight_update"),
|
||||
("POST", "/update_weights"),
|
||||
@@ -6739,7 +6740,7 @@ async fn weight_transfer_routes_support_the_http_training_lifecycle() {
|
||||
let (mut app, engine_task) = test_admin_app_with_engine_script(|dealer, push| {
|
||||
boxed_test_future(async move {
|
||||
let mut calls = Vec::new();
|
||||
for _ in 0..10 {
|
||||
for _ in 0..11 {
|
||||
let utility = recv_engine_message(dealer).await;
|
||||
assert_eq!(utility[0].as_ref(), &[0x03]);
|
||||
let payload = decode_value(&utility[1]).expect("decode utility");
|
||||
@@ -6769,6 +6770,7 @@ async fn weight_transfer_routes_support_the_http_training_lifecycle() {
|
||||
"collective_rpc [\"finish_weight_update\",null,[],{}]",
|
||||
"set_weight_version [\"step-2\"]",
|
||||
"get_weight_version []",
|
||||
"collective_rpc [\"shutdown_weight_transfer_engine\",null,[],{}]",
|
||||
]
|
||||
"#]].assert_debug_eq(&calls);
|
||||
})
|
||||
@@ -6817,6 +6819,7 @@ async fn weight_transfer_routes_support_the_http_training_lifecycle() {
|
||||
Some(json!({"new_version": "step-2"})),
|
||||
),
|
||||
("GET", "/weight_info", None),
|
||||
("POST", "/shutdown_weight_transfer_engine", None),
|
||||
] {
|
||||
let mut request = Request::builder().method(method).uri(path);
|
||||
let body = match body {
|
||||
@@ -6838,7 +6841,7 @@ async fn weight_transfer_routes_support_the_http_training_lifecycle() {
|
||||
}
|
||||
engine_task.await.expect("mock engine task");
|
||||
let metrics_after = METRICS.render().unwrap();
|
||||
for operation in ["update", "finish"] {
|
||||
for (operation, count) in [("update", 2.0), ("finish", 2.0), ("shutdown", 1.0)] {
|
||||
assert_eq!(
|
||||
metric_delta(
|
||||
&metrics_before,
|
||||
@@ -6846,7 +6849,7 @@ async fn weight_transfer_routes_support_the_http_training_lifecycle() {
|
||||
"vllm:rl_weight_update_operation_duration_seconds_count",
|
||||
Some(&format!("operation=\"{operation}\"")),
|
||||
),
|
||||
2.0
|
||||
count
|
||||
);
|
||||
assert_eq!(
|
||||
metric_value(
|
||||
@@ -6868,6 +6871,7 @@ async fn weight_transfer_routes_support_the_http_training_lifecycle() {
|
||||
"{\"message\":\"Weight update finished\"}",
|
||||
"{\"success\":true,\"new_version\":\"step-2\"}",
|
||||
"{\"weight_version\":\"step-2\"}",
|
||||
"{\"message\":\"Weight transfer shut down\"}",
|
||||
]
|
||||
"#]]
|
||||
.assert_debug_eq(&responses);
|
||||
|
||||
@@ -75,6 +75,22 @@ pub async fn init_weight_transfer_engine(
|
||||
}))
|
||||
}
|
||||
|
||||
/// Release the trainer connection without shutting down inference.
|
||||
pub async fn shutdown_weight_transfer_engine(
|
||||
State(state): State<Arc<AppState>>,
|
||||
) -> Result<Json<MessageResponse>, ApiError> {
|
||||
let _recorder = METRICS.api_server.record_weight_operation("shutdown");
|
||||
state
|
||||
.engine_core_client()
|
||||
.shutdown_weight_transfer_engine()
|
||||
.await
|
||||
.map_err(|error| utility_call_error("shutdown_weight_transfer_engine", error))?;
|
||||
|
||||
Ok(Json(MessageResponse {
|
||||
message: "Weight transfer shut down",
|
||||
}))
|
||||
}
|
||||
|
||||
/// Start a weight update transaction.
|
||||
pub async fn start_weight_update(
|
||||
State(state): State<Arc<AppState>>,
|
||||
|
||||
Reference in New Issue
Block a user