[Rust Frontend] Support HTTP RL weight synchronization (#56567)

Co-authored-by: Bugen Zhao <i@bugenzhao.com>
Co-authored-by: Codex <noreply@openai.com>
Signed-off-by: Wang Zupeng <zupenwang@gmail.com>
Signed-off-by: Bugen Zhao <i@bugenzhao.com>
This commit is contained in:
Zupeng Wang
2026-09-14 03:37:04 +00:00
committed by GitHub
co-authored by Bugen Zhao Codex
parent 2d9c30a7b5
commit 7632f767f0
8 changed files with 483 additions and 6 deletions
+11
View File
@@ -43,6 +43,17 @@ For example:
VLLM_USE_RUST_FRONTEND=1 vllm serve Qwen/Qwen3-0.6B
```
### RL weight synchronization
With `VLLM_SERVER_DEV_MODE=1`, the Rust frontend supports the HTTP weight-transfer
lifecycle used by `HTTPVLLMWeightSyncClient`: initialization, starting an update,
transferring weights, and finishing the update. It also supports draft-model
updates and `/update_weight_version` and `/weight_info` for version tracking.
For trainer-side usage, see the [IPC](../examples/rl/rlhf_http_ipc.py) and
[NCCL](../examples/rl/rlhf_http_nccl.py) examples. Configure the weight-transfer
backend on the server and pause generation during updates, as shown there.
### External Engine
`vllm-rs serve` can be run standalone with `--data-parallel-size-local 0` when the Python engines
+7
View File
@@ -2,6 +2,7 @@
// SPDX-FileCopyrightText: Copyright contributors to the vLLM project
use axum::Json;
use axum::extract::rejection::JsonRejection;
use axum::http::StatusCode;
use axum::response::{IntoResponse, Response};
use thiserror_ext::{AsReport as _, Construct, Macro};
@@ -70,6 +71,12 @@ impl ApiError {
}
}
impl From<JsonRejection> for ApiError {
fn from(error: JsonRejection) -> Self {
Self::json_parse_error(error.body_text())
}
}
impl IntoResponse for ApiError {
fn into_response(self) -> Response {
(self.status_code(), Json(self.to_error_response())).into_response()
+23
View File
@@ -17,6 +17,7 @@ mod server_info;
mod sleep;
mod tokenize;
mod version;
mod weight_transfer;
mod world_size;
use std::sync::Arc;
@@ -139,6 +140,28 @@ fn build_router_with_options(
.route("/reset_mm_cache", post(cache::reset_mm_cache))
.route("/reset_encoder_cache", post(cache::reset_encoder_cache))
.route("/collective_rpc", post(collective_rpc::collective_rpc))
.route(
"/init_weight_transfer_engine",
post(weight_transfer::init_weight_transfer_engine),
)
.route(
"/start_weight_update",
post(weight_transfer::start_weight_update),
)
.route(
"/start_draft_weight_update",
post(weight_transfer::start_draft_weight_update),
)
.route("/update_weights", post(weight_transfer::update_weights))
.route(
"/finish_weight_update",
post(weight_transfer::finish_weight_update),
)
.route(
"/update_weight_version",
post(weight_transfer::update_weight_version),
)
.route("/weight_info", get(weight_transfer::weight_info))
.route("/abort_requests", post(abort_requests::abort_requests))
.route("/sleep", post(sleep::sleep))
.route("/wake_up", post(sleep::wake_up))
+1 -1
View File
@@ -22,7 +22,7 @@ pub async fn abort_requests(
State(state): State<Arc<AppState>>,
body: Result<Json<AbortRequestsRequest>, JsonRejection>,
) -> Result<StatusCode, ApiError> {
let Json(body) = body.map_err(|error| ApiError::json_parse_error(error.body_text()))?;
let Json(body) = body?;
// Empty/missing `request_ids` aborts all in-flight requests.
let request_ids = body.request_ids.unwrap_or_default();
+1 -1
View File
@@ -36,7 +36,7 @@ pub async fn collective_rpc(
State(state): State<Arc<AppState>>,
body: Result<Json<CollectiveRpcRequest>, JsonRejection>,
) -> Result<Json<CollectiveRpcResponse>, ApiError> {
let Json(body) = body.map_err(|error| ApiError::json_parse_error(error.body_text()))?;
let Json(body) = body?;
let method = body.method.ok_or_else(|| {
ApiError::invalid_request(
"Missing 'method' in request body".to_string(),
@@ -5,7 +5,6 @@
//! Variation of https://github.com/lightseekorg/smg/blob/main/crates/protocols/src/validated.rs
use axum::Json;
use axum::extract::rejection::JsonRejection;
use axum::extract::{FromRequest, Request};
use serde::de::DeserializeOwned;
use validator::Validate;
@@ -30,9 +29,7 @@ where
type Rejection = ApiError;
async fn from_request(req: Request, state: &S) -> Result<Self, Self::Rejection> {
let Json(mut data) = Json::<T>::from_request(req, state)
.await
.map_err(|err: JsonRejection| ApiError::json_parse_error(err.body_text()))?;
let Json(mut data) = Json::<T>::from_request(req, state).await?;
data.normalize();
+249
View File
@@ -6113,6 +6113,13 @@ async fn admin_routes_are_hidden_when_dev_mode_is_disabled() {
("POST", "/reset_prefix_cache"),
("POST", "/reset_mm_cache"),
("POST", "/reset_encoder_cache"),
("POST", "/init_weight_transfer_engine"),
("POST", "/start_weight_update"),
("POST", "/start_draft_weight_update"),
("POST", "/update_weights"),
("POST", "/finish_weight_update"),
("POST", "/update_weight_version"),
("GET", "/weight_info"),
] {
let response = app
.clone()
@@ -6132,6 +6139,248 @@ async fn admin_routes_are_hidden_when_dev_mode_is_disabled() {
engine_task.abort_and_join().await;
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
#[serial]
async fn weight_transfer_routes_support_the_http_training_lifecycle() {
let (mut app, engine_task) = test_admin_app_with_engine_script(|dealer, push| {
boxed_test_future(async move {
let mut calls = Vec::new();
for _ in 0..10 {
let utility = recv_engine_message(dealer).await;
assert_eq!(utility[0].as_ref(), &[0x03]);
let payload = decode_value(&utility[1]).expect("decode utility");
let array = payload.as_array().expect("utility array");
let call_id = array[1].as_u64().expect("call id");
let args: serde_json::Value =
rmpv::ext::from_value(array[3].clone()).expect("decode args");
calls.push(format!("{} {args}", array[2].as_str().expect("method")));
let result = if array[2] == Value::from("get_weight_version") {
utility_result_value("step-2")
} else if array[2] == Value::from("collective_rpc") {
utility_result_value(vec![(), ()])
} else {
utility_none_result()
};
send_outputs(push, utility_outputs(call_id, result)).await;
}
expect_test::expect![[r#"
[
"collective_rpc [\"init_weight_transfer_engine\",null,[],{\"init_info\":{\"master_address\":\"127.0.0.1\",\"master_port\":29500,\"rank_offset\":1,\"world_size\":3}}]",
"collective_rpc [\"start_weight_update\",null,[],{}]",
"collective_rpc [\"update_weights\",null,[],{\"update_info\":{\"names\":[\"model.embed_tokens.weight\"],\"dtype_names\":[\"bfloat16\"],\"shapes\":[[32,16]],\"is_checkpoint_format\":true}}]",
"collective_rpc [\"update_weights\",null,[],{\"update_info\":[{\"rank\":0,\"ipc_handles_pickled\":\"opaque-handle-0\"},{\"rank\":1,\"ipc_handles_pickled\":\"opaque-handle-1\"}]}]",
"collective_rpc [\"finish_weight_update\",null,[],{}]",
"set_weight_version [\"step-1\"]",
"collective_rpc [\"start_draft_weight_update\",null,[],{}]",
"collective_rpc [\"finish_weight_update\",null,[],{}]",
"set_weight_version [\"step-2\"]",
"get_weight_version []",
]
"#]].assert_debug_eq(&calls);
})
})
.await;
let mut responses = Vec::new();
for (method, path, body) in [
(
"POST",
"/init_weight_transfer_engine",
Some(json!({
"init_info": {"master_address": "127.0.0.1", "master_port": 29500,
"rank_offset": 1, "world_size": 3}
})),
),
("POST", "/start_weight_update", None),
(
"POST",
"/update_weights",
Some(json!({
"update_info": {"names": ["model.embed_tokens.weight"],
"dtype_names": ["bfloat16"], "shapes": [[32, 16]],
"is_checkpoint_format": true}
})),
),
(
"POST",
"/update_weights",
Some(json!({
"update_info": [{"rank": 0, "ipc_handles_pickled": "opaque-handle-0"},
{"rank": 1, "ipc_handles_pickled": "opaque-handle-1"}]
})),
),
(
"POST",
"/finish_weight_update",
Some(json!({"weight_version": "step-1"})),
),
("POST", "/start_draft_weight_update", None),
("POST", "/finish_weight_update", None),
(
"POST",
"/update_weight_version",
Some(json!({"new_version": "step-2"})),
),
("GET", "/weight_info", None),
] {
let mut request = Request::builder().method(method).uri(path);
let body = match body {
Some(body) => {
request = request.header("content-type", "application/json");
Body::from(body.to_string())
}
None => Body::empty(),
};
let response =
app.call(request.body(body).expect("build request")).await.expect("call app");
assert_eq!(response.status(), StatusCode::OK, "{path}");
let body = to_bytes(response.into_body(), usize::MAX).await.expect("read body");
responses.push(
serde_json::from_slice::<serde_json::Value>(&body)
.expect("decode json")
.to_string(),
);
}
engine_task.await.expect("mock engine task");
expect_test::expect![[r#"
[
"{\"message\":\"Weight transfer initialized\"}",
"{\"message\":\"Weight update started\"}",
"{\"message\":\"Weights updated\"}",
"{\"message\":\"Weights updated\"}",
"{\"message\":\"Weight update finished\"}",
"{\"message\":\"Draft weight update started\"}",
"{\"message\":\"Weight update finished\"}",
"{\"success\":true,\"new_version\":\"step-2\"}",
"{\"weight_version\":\"step-2\"}",
]
"#]]
.assert_debug_eq(&responses);
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
#[serial]
async fn weight_transfer_routes_reject_invalid_payloads_before_engine_calls() {
let (mut app, engine_task) = test_admin_app_with_engine_script(|dealer, _push| {
boxed_test_future(async move {
let message = recv_engine_message(dealer).await;
panic!("invalid payload reached engine: {message:?}");
})
})
.await;
for (path, body) in [
("/init_weight_transfer_engine", "{"),
("/init_weight_transfer_engine", "{}"),
("/init_weight_transfer_engine", r#"{"init_info":null}"#),
("/init_weight_transfer_engine", r#"{"init_info":[]}"#),
("/update_weights", "{"),
("/update_weights", "{}"),
("/update_weights", r#"{"update_info":null}"#),
("/update_weights", r#"{"update_info":1}"#),
("/update_weights", r#"{"update_info":[{},null]}"#),
("/finish_weight_update", "{"),
("/finish_weight_update", r#"{"weight_version":1}"#),
("/update_weight_version", "{}"),
("/update_weight_version", r#"{"new_version":null}"#),
] {
let response = app
.call(
Request::builder()
.method("POST")
.uri(path)
.header("content-type", "application/json")
.body(Body::from(body))
.expect("build request"),
)
.await
.expect("call app");
assert_eq!(response.status(), StatusCode::BAD_REQUEST, "{path}: {body}");
}
engine_task.abort_and_join().await;
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
#[serial]
async fn weight_transfer_routes_do_not_commit_version_when_finish_fails() {
let (mut app, engine_task) = test_admin_app_with_engine_script(|dealer, push| {
boxed_test_future(async move {
let utility = recv_engine_message(dealer).await;
let payload = decode_value(&utility[1]).expect("decode utility");
let array = payload.as_array().expect("utility array");
assert_eq!(array[2], Value::from("collective_rpc"));
assert_eq!(
array[3].as_array().expect("args")[0],
Value::from("finish_weight_update")
);
send_outputs(
push,
UtilityCallOutput {
output: UtilityOutput {
call_id: array[1].as_u64().expect("call id").into(),
failure_message: Some("weight reload failed".to_string()),
result: None,
},
..Default::default()
}
.into(),
)
.await;
let utility = recv_engine_message(dealer).await;
let payload = decode_value(&utility[1]).expect("decode utility");
let array = payload.as_array().expect("utility array");
assert_eq!(array[2], Value::from("get_weight_version"));
send_outputs(
push,
utility_outputs(
array[1].as_u64().expect("call id"),
utility_result_value("step-0"),
),
)
.await;
})
})
.await;
let response = app
.call(
Request::builder()
.method("POST")
.uri("/finish_weight_update")
.header("content-type", "application/json")
.body(Body::from(r#"{"weight_version":"step-1"}"#))
.expect("build request"),
)
.await
.expect("call app");
assert_eq!(response.status(), StatusCode::INTERNAL_SERVER_ERROR);
let body = to_bytes(response.into_body(), usize::MAX).await.expect("read body");
let error: serde_json::Value = serde_json::from_slice(&body).expect("decode json");
assert!(
error["error"]["message"]
.as_str()
.expect("error message")
.contains("weight reload failed")
);
let response = app
.call(
Request::builder()
.uri("/weight_info")
.body(Body::empty())
.expect("build request"),
)
.await
.expect("call app");
assert_eq!(response.status(), StatusCode::OK);
let body = to_bytes(response.into_body(), usize::MAX).await.expect("read body");
assert_eq!(
serde_json::from_slice::<serde_json::Value>(&body).expect("decode json"),
json!({"weight_version": "step-0"})
);
engine_task.await.expect("mock engine task");
}
// ========================= Stop string tests =========================
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
@@ -0,0 +1,190 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright contributors to the vLLM project
use std::sync::Arc;
use axum::Json;
use axum::extract::State;
use axum::extract::rejection::JsonRejection;
use serde::{Deserialize, Serialize};
use serde_json::Value as JsonValue;
use crate::error::{ApiError, invalid_request};
use crate::state::AppState;
use crate::utils::utility_call_error;
#[derive(Deserialize)]
pub(crate) struct InitWeightTransferRequest {
init_info: Option<JsonValue>,
}
#[derive(Deserialize)]
pub(crate) struct UpdateWeightsRequest {
update_info: Option<JsonValue>,
}
#[derive(Default, Deserialize)]
pub(crate) struct FinishWeightUpdateRequest {
weight_version: Option<String>,
}
#[derive(Deserialize)]
pub(crate) struct UpdateWeightVersionRequest {
new_version: String,
}
#[derive(Serialize)]
pub(crate) struct MessageResponse {
message: &'static str,
}
#[derive(Serialize)]
pub(crate) struct UpdateWeightVersionResponse {
success: bool,
new_version: String,
}
#[derive(Serialize)]
pub(crate) struct WeightInfoResponse {
weight_version: String,
}
/// Initialize weight transfer for RL training.
pub async fn init_weight_transfer_engine(
State(state): State<Arc<AppState>>,
body: Result<Json<InitWeightTransferRequest>, JsonRejection>,
) -> Result<Json<MessageResponse>, ApiError> {
let Json(body) = body?;
let init_info = body.init_info.filter(JsonValue::is_object).ok_or_else(|| {
invalid_request!(
param = Some("init_info"),
"'init_info' must be a JSON object"
)
})?;
state
.engine_core_client()
.init_weight_transfer_engine(init_info)
.await
.map_err(|error| utility_call_error("init_weight_transfer_engine", error))?;
Ok(Json(MessageResponse {
message: "Weight transfer initialized",
}))
}
/// Start a weight update transaction.
pub async fn start_weight_update(
State(state): State<Arc<AppState>>,
) -> Result<Json<MessageResponse>, ApiError> {
state
.engine_core_client()
.start_weight_update()
.await
.map_err(|error| utility_call_error("start_weight_update", error))?;
Ok(Json(MessageResponse {
message: "Weight update started",
}))
}
/// Start a draft-model weight update transaction.
pub async fn start_draft_weight_update(
State(state): State<Arc<AppState>>,
) -> Result<Json<MessageResponse>, ApiError> {
state
.engine_core_client()
.start_draft_weight_update()
.await
.map_err(|error| utility_call_error("start_draft_weight_update", error))?;
Ok(Json(MessageResponse {
message: "Draft weight update started",
}))
}
/// Update model weights with backend-specific transfer metadata.
pub async fn update_weights(
State(state): State<Arc<AppState>>,
body: Result<Json<UpdateWeightsRequest>, JsonRejection>,
) -> Result<Json<MessageResponse>, ApiError> {
let Json(body) = body?;
let update_info = body
.update_info
.filter(|info| {
info.is_object()
|| info.as_array().is_some_and(|items| items.iter().all(JsonValue::is_object))
})
.ok_or_else(|| {
invalid_request!(
param = Some("update_info"),
"'update_info' must be a JSON object or a list of per-worker JSON objects",
)
})?;
state
.engine_core_client()
.update_weights(update_info)
.await
.map_err(|error| utility_call_error("update_weights", error))?;
Ok(Json(MessageResponse {
message: "Weights updated",
}))
}
/// Finish a weight update transaction and optionally set the weight version.
pub async fn finish_weight_update(
State(state): State<Arc<AppState>>,
body: Result<Option<Json<FinishWeightUpdateRequest>>, JsonRejection>,
) -> Result<Json<MessageResponse>, ApiError> {
// HTTPVLLMWeightSyncClient omits the body when no version is supplied.
let request = body?.map(|Json(request)| request).unwrap_or_default();
let client = state.engine_core_client();
client
.finish_weight_update()
.await
.map_err(|error| utility_call_error("finish_weight_update", error))?;
if let Some(version) = request.weight_version {
client
.set_weight_version(&version)
.await
.map_err(|error| utility_call_error("set_weight_version", error))?;
}
Ok(Json(MessageResponse {
message: "Weight update finished",
}))
}
/// Update the weight version identifier.
pub async fn update_weight_version(
State(state): State<Arc<AppState>>,
body: Result<Json<UpdateWeightVersionRequest>, JsonRejection>,
) -> Result<Json<UpdateWeightVersionResponse>, ApiError> {
let Json(body) = body?;
state
.engine_core_client()
.set_weight_version(&body.new_version)
.await
.map_err(|error| utility_call_error("set_weight_version", error))?;
Ok(Json(UpdateWeightVersionResponse {
success: true,
new_version: body.new_version,
}))
}
/// Get current weight metadata.
pub async fn weight_info(
State(state): State<Arc<AppState>>,
) -> Result<Json<WeightInfoResponse>, ApiError> {
let weight_version = state
.engine_core_client()
.get_weight_version()
.await
.map_err(|error| utility_call_error("get_weight_version", error))?;
Ok(Json(WeightInfoResponse { weight_version }))
}