From 7632f767f0d896a902566acab29775816b39a92d Mon Sep 17 00:00:00 2001 From: Zupeng Wang Date: Mon, 14 Sep 2026 11:37:04 +0800 Subject: [PATCH] [Rust Frontend] Support HTTP RL weight synchronization (#56567) Co-authored-by: Bugen Zhao Co-authored-by: Codex Signed-off-by: Wang Zupeng Signed-off-by: Bugen Zhao --- rust/README.md | 11 + rust/src/server/src/error.rs | 7 + rust/src/server/src/routes.rs | 23 ++ rust/src/server/src/routes/abort_requests.rs | 2 +- rust/src/server/src/routes/collective_rpc.rs | 2 +- .../src/routes/openai/utils/validated_json.rs | 5 +- rust/src/server/src/routes/tests.rs | 249 ++++++++++++++++++ rust/src/server/src/routes/weight_transfer.rs | 190 +++++++++++++ 8 files changed, 483 insertions(+), 6 deletions(-) create mode 100644 rust/src/server/src/routes/weight_transfer.rs diff --git a/rust/README.md b/rust/README.md index 06b50dbbfe..0cc33d64f8 100644 --- a/rust/README.md +++ b/rust/README.md @@ -43,6 +43,17 @@ For example: VLLM_USE_RUST_FRONTEND=1 vllm serve Qwen/Qwen3-0.6B ``` +### RL weight synchronization + +With `VLLM_SERVER_DEV_MODE=1`, the Rust frontend supports the HTTP weight-transfer +lifecycle used by `HTTPVLLMWeightSyncClient`: initialization, starting an update, +transferring weights, and finishing the update. It also supports draft-model +updates and `/update_weight_version` and `/weight_info` for version tracking. + +For trainer-side usage, see the [IPC](../examples/rl/rlhf_http_ipc.py) and +[NCCL](../examples/rl/rlhf_http_nccl.py) examples. Configure the weight-transfer +backend on the server and pause generation during updates, as shown there. + ### External Engine `vllm-rs serve` can be run standalone with `--data-parallel-size-local 0` when the Python engines diff --git a/rust/src/server/src/error.rs b/rust/src/server/src/error.rs index b658388cbd..9f083a1a61 100644 --- a/rust/src/server/src/error.rs +++ b/rust/src/server/src/error.rs @@ -2,6 +2,7 @@ // SPDX-FileCopyrightText: Copyright contributors to the vLLM project use axum::Json; +use axum::extract::rejection::JsonRejection; use axum::http::StatusCode; use axum::response::{IntoResponse, Response}; use thiserror_ext::{AsReport as _, Construct, Macro}; @@ -70,6 +71,12 @@ impl ApiError { } } +impl From for ApiError { + fn from(error: JsonRejection) -> Self { + Self::json_parse_error(error.body_text()) + } +} + impl IntoResponse for ApiError { fn into_response(self) -> Response { (self.status_code(), Json(self.to_error_response())).into_response() diff --git a/rust/src/server/src/routes.rs b/rust/src/server/src/routes.rs index 393a08450f..e7e91938bb 100644 --- a/rust/src/server/src/routes.rs +++ b/rust/src/server/src/routes.rs @@ -17,6 +17,7 @@ mod server_info; mod sleep; mod tokenize; mod version; +mod weight_transfer; mod world_size; use std::sync::Arc; @@ -139,6 +140,28 @@ fn build_router_with_options( .route("/reset_mm_cache", post(cache::reset_mm_cache)) .route("/reset_encoder_cache", post(cache::reset_encoder_cache)) .route("/collective_rpc", post(collective_rpc::collective_rpc)) + .route( + "/init_weight_transfer_engine", + post(weight_transfer::init_weight_transfer_engine), + ) + .route( + "/start_weight_update", + post(weight_transfer::start_weight_update), + ) + .route( + "/start_draft_weight_update", + post(weight_transfer::start_draft_weight_update), + ) + .route("/update_weights", post(weight_transfer::update_weights)) + .route( + "/finish_weight_update", + post(weight_transfer::finish_weight_update), + ) + .route( + "/update_weight_version", + post(weight_transfer::update_weight_version), + ) + .route("/weight_info", get(weight_transfer::weight_info)) .route("/abort_requests", post(abort_requests::abort_requests)) .route("/sleep", post(sleep::sleep)) .route("/wake_up", post(sleep::wake_up)) diff --git a/rust/src/server/src/routes/abort_requests.rs b/rust/src/server/src/routes/abort_requests.rs index 2987703160..d08e5af3fa 100644 --- a/rust/src/server/src/routes/abort_requests.rs +++ b/rust/src/server/src/routes/abort_requests.rs @@ -22,7 +22,7 @@ pub async fn abort_requests( State(state): State>, body: Result, JsonRejection>, ) -> Result { - let Json(body) = body.map_err(|error| ApiError::json_parse_error(error.body_text()))?; + let Json(body) = body?; // Empty/missing `request_ids` aborts all in-flight requests. let request_ids = body.request_ids.unwrap_or_default(); diff --git a/rust/src/server/src/routes/collective_rpc.rs b/rust/src/server/src/routes/collective_rpc.rs index bd769bee43..fa3782cbe9 100644 --- a/rust/src/server/src/routes/collective_rpc.rs +++ b/rust/src/server/src/routes/collective_rpc.rs @@ -36,7 +36,7 @@ pub async fn collective_rpc( State(state): State>, body: Result, JsonRejection>, ) -> Result, ApiError> { - let Json(body) = body.map_err(|error| ApiError::json_parse_error(error.body_text()))?; + let Json(body) = body?; let method = body.method.ok_or_else(|| { ApiError::invalid_request( "Missing 'method' in request body".to_string(), diff --git a/rust/src/server/src/routes/openai/utils/validated_json.rs b/rust/src/server/src/routes/openai/utils/validated_json.rs index 1ddec3a9f5..a9b68bc3b4 100644 --- a/rust/src/server/src/routes/openai/utils/validated_json.rs +++ b/rust/src/server/src/routes/openai/utils/validated_json.rs @@ -5,7 +5,6 @@ //! Variation of https://github.com/lightseekorg/smg/blob/main/crates/protocols/src/validated.rs use axum::Json; -use axum::extract::rejection::JsonRejection; use axum::extract::{FromRequest, Request}; use serde::de::DeserializeOwned; use validator::Validate; @@ -30,9 +29,7 @@ where type Rejection = ApiError; async fn from_request(req: Request, state: &S) -> Result { - let Json(mut data) = Json::::from_request(req, state) - .await - .map_err(|err: JsonRejection| ApiError::json_parse_error(err.body_text()))?; + let Json(mut data) = Json::::from_request(req, state).await?; data.normalize(); diff --git a/rust/src/server/src/routes/tests.rs b/rust/src/server/src/routes/tests.rs index 210ae044a0..2851ec1e88 100644 --- a/rust/src/server/src/routes/tests.rs +++ b/rust/src/server/src/routes/tests.rs @@ -6113,6 +6113,13 @@ async fn admin_routes_are_hidden_when_dev_mode_is_disabled() { ("POST", "/reset_prefix_cache"), ("POST", "/reset_mm_cache"), ("POST", "/reset_encoder_cache"), + ("POST", "/init_weight_transfer_engine"), + ("POST", "/start_weight_update"), + ("POST", "/start_draft_weight_update"), + ("POST", "/update_weights"), + ("POST", "/finish_weight_update"), + ("POST", "/update_weight_version"), + ("GET", "/weight_info"), ] { let response = app .clone() @@ -6132,6 +6139,248 @@ async fn admin_routes_are_hidden_when_dev_mode_is_disabled() { engine_task.abort_and_join().await; } +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[serial] +async fn weight_transfer_routes_support_the_http_training_lifecycle() { + let (mut app, engine_task) = test_admin_app_with_engine_script(|dealer, push| { + boxed_test_future(async move { + let mut calls = Vec::new(); + for _ in 0..10 { + let utility = recv_engine_message(dealer).await; + assert_eq!(utility[0].as_ref(), &[0x03]); + let payload = decode_value(&utility[1]).expect("decode utility"); + let array = payload.as_array().expect("utility array"); + let call_id = array[1].as_u64().expect("call id"); + let args: serde_json::Value = + rmpv::ext::from_value(array[3].clone()).expect("decode args"); + calls.push(format!("{} {args}", array[2].as_str().expect("method"))); + let result = if array[2] == Value::from("get_weight_version") { + utility_result_value("step-2") + } else if array[2] == Value::from("collective_rpc") { + utility_result_value(vec![(), ()]) + } else { + utility_none_result() + }; + send_outputs(push, utility_outputs(call_id, result)).await; + } + expect_test::expect![[r#" + [ + "collective_rpc [\"init_weight_transfer_engine\",null,[],{\"init_info\":{\"master_address\":\"127.0.0.1\",\"master_port\":29500,\"rank_offset\":1,\"world_size\":3}}]", + "collective_rpc [\"start_weight_update\",null,[],{}]", + "collective_rpc [\"update_weights\",null,[],{\"update_info\":{\"names\":[\"model.embed_tokens.weight\"],\"dtype_names\":[\"bfloat16\"],\"shapes\":[[32,16]],\"is_checkpoint_format\":true}}]", + "collective_rpc [\"update_weights\",null,[],{\"update_info\":[{\"rank\":0,\"ipc_handles_pickled\":\"opaque-handle-0\"},{\"rank\":1,\"ipc_handles_pickled\":\"opaque-handle-1\"}]}]", + "collective_rpc [\"finish_weight_update\",null,[],{}]", + "set_weight_version [\"step-1\"]", + "collective_rpc [\"start_draft_weight_update\",null,[],{}]", + "collective_rpc [\"finish_weight_update\",null,[],{}]", + "set_weight_version [\"step-2\"]", + "get_weight_version []", + ] + "#]].assert_debug_eq(&calls); + }) + }) + .await; + + let mut responses = Vec::new(); + for (method, path, body) in [ + ( + "POST", + "/init_weight_transfer_engine", + Some(json!({ + "init_info": {"master_address": "127.0.0.1", "master_port": 29500, + "rank_offset": 1, "world_size": 3} + })), + ), + ("POST", "/start_weight_update", None), + ( + "POST", + "/update_weights", + Some(json!({ + "update_info": {"names": ["model.embed_tokens.weight"], + "dtype_names": ["bfloat16"], "shapes": [[32, 16]], + "is_checkpoint_format": true} + })), + ), + ( + "POST", + "/update_weights", + Some(json!({ + "update_info": [{"rank": 0, "ipc_handles_pickled": "opaque-handle-0"}, + {"rank": 1, "ipc_handles_pickled": "opaque-handle-1"}] + })), + ), + ( + "POST", + "/finish_weight_update", + Some(json!({"weight_version": "step-1"})), + ), + ("POST", "/start_draft_weight_update", None), + ("POST", "/finish_weight_update", None), + ( + "POST", + "/update_weight_version", + Some(json!({"new_version": "step-2"})), + ), + ("GET", "/weight_info", None), + ] { + let mut request = Request::builder().method(method).uri(path); + let body = match body { + Some(body) => { + request = request.header("content-type", "application/json"); + Body::from(body.to_string()) + } + None => Body::empty(), + }; + let response = + app.call(request.body(body).expect("build request")).await.expect("call app"); + assert_eq!(response.status(), StatusCode::OK, "{path}"); + let body = to_bytes(response.into_body(), usize::MAX).await.expect("read body"); + responses.push( + serde_json::from_slice::(&body) + .expect("decode json") + .to_string(), + ); + } + engine_task.await.expect("mock engine task"); + expect_test::expect![[r#" + [ + "{\"message\":\"Weight transfer initialized\"}", + "{\"message\":\"Weight update started\"}", + "{\"message\":\"Weights updated\"}", + "{\"message\":\"Weights updated\"}", + "{\"message\":\"Weight update finished\"}", + "{\"message\":\"Draft weight update started\"}", + "{\"message\":\"Weight update finished\"}", + "{\"success\":true,\"new_version\":\"step-2\"}", + "{\"weight_version\":\"step-2\"}", + ] + "#]] + .assert_debug_eq(&responses); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[serial] +async fn weight_transfer_routes_reject_invalid_payloads_before_engine_calls() { + let (mut app, engine_task) = test_admin_app_with_engine_script(|dealer, _push| { + boxed_test_future(async move { + let message = recv_engine_message(dealer).await; + panic!("invalid payload reached engine: {message:?}"); + }) + }) + .await; + + for (path, body) in [ + ("/init_weight_transfer_engine", "{"), + ("/init_weight_transfer_engine", "{}"), + ("/init_weight_transfer_engine", r#"{"init_info":null}"#), + ("/init_weight_transfer_engine", r#"{"init_info":[]}"#), + ("/update_weights", "{"), + ("/update_weights", "{}"), + ("/update_weights", r#"{"update_info":null}"#), + ("/update_weights", r#"{"update_info":1}"#), + ("/update_weights", r#"{"update_info":[{},null]}"#), + ("/finish_weight_update", "{"), + ("/finish_weight_update", r#"{"weight_version":1}"#), + ("/update_weight_version", "{}"), + ("/update_weight_version", r#"{"new_version":null}"#), + ] { + let response = app + .call( + Request::builder() + .method("POST") + .uri(path) + .header("content-type", "application/json") + .body(Body::from(body)) + .expect("build request"), + ) + .await + .expect("call app"); + assert_eq!(response.status(), StatusCode::BAD_REQUEST, "{path}: {body}"); + } + engine_task.abort_and_join().await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[serial] +async fn weight_transfer_routes_do_not_commit_version_when_finish_fails() { + let (mut app, engine_task) = test_admin_app_with_engine_script(|dealer, push| { + boxed_test_future(async move { + let utility = recv_engine_message(dealer).await; + let payload = decode_value(&utility[1]).expect("decode utility"); + let array = payload.as_array().expect("utility array"); + assert_eq!(array[2], Value::from("collective_rpc")); + assert_eq!( + array[3].as_array().expect("args")[0], + Value::from("finish_weight_update") + ); + send_outputs( + push, + UtilityCallOutput { + output: UtilityOutput { + call_id: array[1].as_u64().expect("call id").into(), + failure_message: Some("weight reload failed".to_string()), + result: None, + }, + ..Default::default() + } + .into(), + ) + .await; + let utility = recv_engine_message(dealer).await; + let payload = decode_value(&utility[1]).expect("decode utility"); + let array = payload.as_array().expect("utility array"); + assert_eq!(array[2], Value::from("get_weight_version")); + send_outputs( + push, + utility_outputs( + array[1].as_u64().expect("call id"), + utility_result_value("step-0"), + ), + ) + .await; + }) + }) + .await; + + let response = app + .call( + Request::builder() + .method("POST") + .uri("/finish_weight_update") + .header("content-type", "application/json") + .body(Body::from(r#"{"weight_version":"step-1"}"#)) + .expect("build request"), + ) + .await + .expect("call app"); + assert_eq!(response.status(), StatusCode::INTERNAL_SERVER_ERROR); + let body = to_bytes(response.into_body(), usize::MAX).await.expect("read body"); + let error: serde_json::Value = serde_json::from_slice(&body).expect("decode json"); + assert!( + error["error"]["message"] + .as_str() + .expect("error message") + .contains("weight reload failed") + ); + + let response = app + .call( + Request::builder() + .uri("/weight_info") + .body(Body::empty()) + .expect("build request"), + ) + .await + .expect("call app"); + assert_eq!(response.status(), StatusCode::OK); + let body = to_bytes(response.into_body(), usize::MAX).await.expect("read body"); + assert_eq!( + serde_json::from_slice::(&body).expect("decode json"), + json!({"weight_version": "step-0"}) + ); + engine_task.await.expect("mock engine task"); +} + // ========================= Stop string tests ========================= #[tokio::test(flavor = "multi_thread", worker_threads = 2)] diff --git a/rust/src/server/src/routes/weight_transfer.rs b/rust/src/server/src/routes/weight_transfer.rs new file mode 100644 index 0000000000..df55e25593 --- /dev/null +++ b/rust/src/server/src/routes/weight_transfer.rs @@ -0,0 +1,190 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +use std::sync::Arc; + +use axum::Json; +use axum::extract::State; +use axum::extract::rejection::JsonRejection; +use serde::{Deserialize, Serialize}; +use serde_json::Value as JsonValue; + +use crate::error::{ApiError, invalid_request}; +use crate::state::AppState; +use crate::utils::utility_call_error; + +#[derive(Deserialize)] +pub(crate) struct InitWeightTransferRequest { + init_info: Option, +} + +#[derive(Deserialize)] +pub(crate) struct UpdateWeightsRequest { + update_info: Option, +} + +#[derive(Default, Deserialize)] +pub(crate) struct FinishWeightUpdateRequest { + weight_version: Option, +} + +#[derive(Deserialize)] +pub(crate) struct UpdateWeightVersionRequest { + new_version: String, +} + +#[derive(Serialize)] +pub(crate) struct MessageResponse { + message: &'static str, +} + +#[derive(Serialize)] +pub(crate) struct UpdateWeightVersionResponse { + success: bool, + new_version: String, +} + +#[derive(Serialize)] +pub(crate) struct WeightInfoResponse { + weight_version: String, +} + +/// Initialize weight transfer for RL training. +pub async fn init_weight_transfer_engine( + State(state): State>, + body: Result, JsonRejection>, +) -> Result, ApiError> { + let Json(body) = body?; + let init_info = body.init_info.filter(JsonValue::is_object).ok_or_else(|| { + invalid_request!( + param = Some("init_info"), + "'init_info' must be a JSON object" + ) + })?; + + state + .engine_core_client() + .init_weight_transfer_engine(init_info) + .await + .map_err(|error| utility_call_error("init_weight_transfer_engine", error))?; + + Ok(Json(MessageResponse { + message: "Weight transfer initialized", + })) +} + +/// Start a weight update transaction. +pub async fn start_weight_update( + State(state): State>, +) -> Result, ApiError> { + state + .engine_core_client() + .start_weight_update() + .await + .map_err(|error| utility_call_error("start_weight_update", error))?; + + Ok(Json(MessageResponse { + message: "Weight update started", + })) +} + +/// Start a draft-model weight update transaction. +pub async fn start_draft_weight_update( + State(state): State>, +) -> Result, ApiError> { + state + .engine_core_client() + .start_draft_weight_update() + .await + .map_err(|error| utility_call_error("start_draft_weight_update", error))?; + + Ok(Json(MessageResponse { + message: "Draft weight update started", + })) +} + +/// Update model weights with backend-specific transfer metadata. +pub async fn update_weights( + State(state): State>, + body: Result, JsonRejection>, +) -> Result, ApiError> { + let Json(body) = body?; + let update_info = body + .update_info + .filter(|info| { + info.is_object() + || info.as_array().is_some_and(|items| items.iter().all(JsonValue::is_object)) + }) + .ok_or_else(|| { + invalid_request!( + param = Some("update_info"), + "'update_info' must be a JSON object or a list of per-worker JSON objects", + ) + })?; + + state + .engine_core_client() + .update_weights(update_info) + .await + .map_err(|error| utility_call_error("update_weights", error))?; + + Ok(Json(MessageResponse { + message: "Weights updated", + })) +} + +/// Finish a weight update transaction and optionally set the weight version. +pub async fn finish_weight_update( + State(state): State>, + body: Result>, JsonRejection>, +) -> Result, ApiError> { + // HTTPVLLMWeightSyncClient omits the body when no version is supplied. + let request = body?.map(|Json(request)| request).unwrap_or_default(); + + let client = state.engine_core_client(); + client + .finish_weight_update() + .await + .map_err(|error| utility_call_error("finish_weight_update", error))?; + if let Some(version) = request.weight_version { + client + .set_weight_version(&version) + .await + .map_err(|error| utility_call_error("set_weight_version", error))?; + } + + Ok(Json(MessageResponse { + message: "Weight update finished", + })) +} + +/// Update the weight version identifier. +pub async fn update_weight_version( + State(state): State>, + body: Result, JsonRejection>, +) -> Result, ApiError> { + let Json(body) = body?; + state + .engine_core_client() + .set_weight_version(&body.new_version) + .await + .map_err(|error| utility_call_error("set_weight_version", error))?; + + Ok(Json(UpdateWeightVersionResponse { + success: true, + new_version: body.new_version, + })) +} + +/// Get current weight metadata. +pub async fn weight_info( + State(state): State>, +) -> Result, ApiError> { + let weight_version = state + .engine_core_client() + .get_weight_version() + .await + .map_err(|error| utility_call_error("get_weight_version", error))?; + + Ok(Json(WeightInfoResponse { weight_version })) +}