support rl feature : weight checker (#51350)
Signed-off-by: shiyuan680 <yangcheng104@huawei.com> Signed-off-by: Ronald1995 <ronaldautomobile@163.com> Signed-off-by: aoshen02 <aoshen02@users.noreply.github.com> Co-authored-by: shiyuan680 <yangcheng104@huawei.com> Co-authored-by: Ronald1995 <ronaldautomobile@163.com> Co-authored-by: DeepSeek Harness <noreply@deepseek.com> Co-authored-by: aoshen02 <aoshen02@users.noreply.github.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com> Co-authored-by: aoshen02 <aoshen@inferact.ai>
This commit is contained in:
co-authored by
shiyuan680
Ronald1995
DeepSeek Harness
aoshen02
Claude Opus 5.5
aoshen02
parent
a8e069f81b
commit
42d9f8ccdb
@@ -18,6 +18,7 @@ mod server_info;
|
||||
mod sleep;
|
||||
mod tokenize;
|
||||
mod version;
|
||||
mod weight_checker;
|
||||
mod weight_transfer;
|
||||
mod world_size;
|
||||
|
||||
@@ -142,6 +143,7 @@ fn build_router_with_options(
|
||||
post(weight_transfer::update_weight_version),
|
||||
)
|
||||
.route("/weight_info", get(weight_transfer::weight_info))
|
||||
.route("/weight_checker", post(weight_checker::weight_checker))
|
||||
.route("/abort_requests", post(abort_requests::abort_requests))
|
||||
.route("/sleep", post(sleep::sleep))
|
||||
.route(
|
||||
|
||||
@@ -5813,6 +5813,82 @@ async fn collective_rpc_route_sends_expected_utility_call_and_returns_results()
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
#[serial]
|
||||
async fn weight_checker_route_merges_worker_checksums_and_compares() {
|
||||
let (mut app, engine_task) = test_admin_app_with_engine_script(|dealer, push| {
|
||||
boxed_test_future(async move {
|
||||
let workers = Value::Array(vec![
|
||||
Value::Map(vec![(Value::from("tp0:a"), Value::from("1"))]),
|
||||
Value::Map(vec![(Value::from("tp1:a"), Value::from("2"))]),
|
||||
]);
|
||||
for (method, result) in [
|
||||
("compute_weight_checksums", workers.clone()),
|
||||
("compute_weight_checksums", workers),
|
||||
("reset_weights", Value::Array(vec![Value::Nil, Value::Nil])),
|
||||
] {
|
||||
let utility = recv_engine_message(dealer).await;
|
||||
let payload = decode_value(&utility[1]).expect("decode utility payload");
|
||||
let array = payload.as_array().expect("utility payload array");
|
||||
assert_eq!(array[2], Value::from("collective_rpc"));
|
||||
assert_eq!(
|
||||
array[3].as_array().expect("rpc args")[0],
|
||||
Value::from(method)
|
||||
);
|
||||
let call_id = array[1].as_u64().expect("call id");
|
||||
let envelope = UtilityResultEnvelope::without_type_info(result);
|
||||
send_outputs(push, utility_outputs(call_id, envelope)).await;
|
||||
}
|
||||
})
|
||||
})
|
||||
.await;
|
||||
|
||||
let (status, body) =
|
||||
post_json(&mut app, "/weight_checker", json!({"action": "checksum"})).await;
|
||||
assert_eq!(status, StatusCode::OK);
|
||||
assert_eq!(body, json!({"checksums": {"tp0:a": "1", "tp1:a": "2"}}));
|
||||
|
||||
let baseline = json!({"tp0:a": "1", "tp1:a": "9", "tp2:a": "3"});
|
||||
let (status, body) = post_json(
|
||||
&mut app,
|
||||
"/weight_checker",
|
||||
json!({"action": "compare", "baseline": baseline}),
|
||||
)
|
||||
.await;
|
||||
assert_eq!(status, StatusCode::OK);
|
||||
assert_eq!(
|
||||
body,
|
||||
json!({"match": false, "mismatches": ["tp1:a", "tp2:a"]})
|
||||
);
|
||||
|
||||
let (status, body) = post_json(&mut app, "/weight_checker", json!({"action": "reset"})).await;
|
||||
assert_eq!(status, StatusCode::OK);
|
||||
assert_eq!(body, json!({"status": "reset"}));
|
||||
engine_task.abort_and_join().await;
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
#[serial]
|
||||
async fn weight_checker_route_rejects_invalid_requests_before_engine_calls() {
|
||||
let (mut app, engine_task) = test_admin_app_with_engine_script(|dealer, _push| {
|
||||
boxed_test_future(async move {
|
||||
let message = recv_engine_message(dealer).await;
|
||||
panic!("invalid request reached engine: {message:?}");
|
||||
})
|
||||
})
|
||||
.await;
|
||||
for body in [
|
||||
json!({}),
|
||||
json!({"action": "frobnicate"}),
|
||||
json!({"action": "compare"}),
|
||||
json!({"action": "compare", "baseline": {"k": 1}}),
|
||||
] {
|
||||
let (status, _) = post_json(&mut app, "/weight_checker", body.clone()).await;
|
||||
assert_eq!(status, StatusCode::BAD_REQUEST, "{body}");
|
||||
}
|
||||
engine_task.abort_and_join().await;
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
#[serial]
|
||||
async fn sleep_route_uses_python_compatible_default_query_values() {
|
||||
|
||||
@@ -0,0 +1,113 @@
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
// SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
use std::collections::{BTreeMap, BTreeSet};
|
||||
use std::sync::Arc;
|
||||
|
||||
use axum::Json;
|
||||
use axum::extract::State;
|
||||
use axum::extract::rejection::JsonRejection;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use serde_json::Value as JsonValue;
|
||||
|
||||
use crate::error::{ApiError, invalid_request};
|
||||
use crate::state::AppState;
|
||||
use crate::utils::utility_call_error;
|
||||
|
||||
#[derive(Debug, Deserialize, PartialEq)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub(crate) enum WeightCheckerAction {
|
||||
Checksum,
|
||||
Reset,
|
||||
Compare,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
pub(crate) struct WeightCheckerRequest {
|
||||
action: WeightCheckerAction,
|
||||
#[serde(default)]
|
||||
baseline: Option<BTreeMap<String, String>>,
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
#[serde(untagged)]
|
||||
pub(crate) enum WeightCheckerResponse {
|
||||
Checksums {
|
||||
checksums: BTreeMap<String, String>,
|
||||
},
|
||||
Reset {
|
||||
status: &'static str,
|
||||
},
|
||||
Compare {
|
||||
#[serde(rename = "match")]
|
||||
matched: bool,
|
||||
mismatches: Vec<String>,
|
||||
},
|
||||
}
|
||||
|
||||
/// Checksum, reset, or compare model weights against a baseline.
|
||||
pub async fn weight_checker(
|
||||
State(state): State<Arc<AppState>>,
|
||||
body: Result<Json<WeightCheckerRequest>, JsonRejection>,
|
||||
) -> Result<Json<WeightCheckerResponse>, ApiError> {
|
||||
let Json(body) = body?;
|
||||
let client = state.engine_core_client();
|
||||
let no_args = || {
|
||||
(
|
||||
Vec::<JsonValue>::new(),
|
||||
BTreeMap::<String, JsonValue>::new(),
|
||||
)
|
||||
};
|
||||
|
||||
if body.action == WeightCheckerAction::Reset {
|
||||
let (args, kwargs) = no_args();
|
||||
client
|
||||
.collective_rpc("reset_weights", None, args, kwargs)
|
||||
.await
|
||||
.map_err(|error| utility_call_error("reset_weights", error))?;
|
||||
return Ok(Json(WeightCheckerResponse::Reset { status: "reset" }));
|
||||
}
|
||||
if body.action == WeightCheckerAction::Compare && body.baseline.is_none() {
|
||||
return Err(invalid_request!(
|
||||
param = Some("baseline"),
|
||||
"action='compare' requires a 'baseline' object"
|
||||
));
|
||||
}
|
||||
|
||||
let (args, kwargs) = no_args();
|
||||
let workers = client
|
||||
.collective_rpc("compute_weight_checksums", None, args, kwargs)
|
||||
.await
|
||||
.map_err(|error| utility_call_error("compute_weight_checksums", error))?;
|
||||
let mut checksums = BTreeMap::new();
|
||||
for worker in workers {
|
||||
let worker: BTreeMap<String, String> = rmpv::ext::from_value(worker).map_err(|error| {
|
||||
ApiError::server_error(format!("invalid weight checksums: {error}"))
|
||||
})?;
|
||||
for (key, digest) in worker {
|
||||
if checksums.contains_key(&key) {
|
||||
return Err(ApiError::server_error(format!(
|
||||
"Duplicate weight checksum keys: {key}"
|
||||
)));
|
||||
}
|
||||
checksums.insert(key, digest);
|
||||
}
|
||||
}
|
||||
|
||||
let Some(baseline) = body.baseline.filter(|_| body.action == WeightCheckerAction::Compare)
|
||||
else {
|
||||
return Ok(Json(WeightCheckerResponse::Checksums { checksums }));
|
||||
};
|
||||
let mismatches: Vec<String> = checksums
|
||||
.keys()
|
||||
.chain(baseline.keys())
|
||||
.collect::<BTreeSet<_>>()
|
||||
.into_iter()
|
||||
.filter(|key| checksums.get(*key) != baseline.get(*key))
|
||||
.cloned()
|
||||
.collect();
|
||||
Ok(Json(WeightCheckerResponse::Compare {
|
||||
matched: mismatches.is_empty(),
|
||||
mismatches,
|
||||
}))
|
||||
}
|
||||
Reference in New Issue
Block a user