[Rust Frontend][gRPC] Add engine-aware health reporting (#48992)
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> Signed-off-by: Connor Carpenter <connorc@nvidia.com>
This commit is contained in:
co-authored by
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
72d16aee15
commit
1940c8441e
@@ -460,6 +460,12 @@ impl EngineCoreClient {
|
||||
self.inner.is_healthy()
|
||||
}
|
||||
|
||||
/// Subscribe to engine health changes. The current value is `true` while
|
||||
/// the client is healthy and changes permanently to `false` on failure.
|
||||
pub fn subscribe_health(&self) -> tokio::sync::watch::Receiver<bool> {
|
||||
self.inner.subscribe_health()
|
||||
}
|
||||
|
||||
/// Return the first persistent health error observed by the client, if any.
|
||||
pub fn health_error(&self) -> Option<Arc<Error>> {
|
||||
self.inner.health_error()
|
||||
|
||||
@@ -9,7 +9,7 @@ use arc_swap::ArcSwapOption;
|
||||
use parking_lot::Mutex;
|
||||
use thiserror_ext::AsReport as _;
|
||||
use tokio::runtime::Handle;
|
||||
use tokio::sync::mpsc;
|
||||
use tokio::sync::{mpsc, watch};
|
||||
use tracing::{debug, info, trace, warn};
|
||||
use vllm_metrics::METRICS;
|
||||
use zeromq::RouterSendHalf;
|
||||
@@ -36,6 +36,7 @@ pub(crate) struct ClientInner {
|
||||
request_reg: Mutex<RequestRegistry>,
|
||||
utility_reg: Mutex<UtilityRegistry>,
|
||||
health_error: ArcSwapOption<Error>,
|
||||
health_tx: watch::Sender<bool>,
|
||||
}
|
||||
|
||||
impl ClientInner {
|
||||
@@ -57,6 +58,7 @@ impl ClientInner {
|
||||
request_reg: Mutex::new(RequestRegistry::new(engines)),
|
||||
utility_reg: Mutex::new(UtilityRegistry::default()),
|
||||
health_error: ArcSwapOption::empty(),
|
||||
health_tx: watch::Sender::new(true),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -169,6 +171,7 @@ impl ClientInner {
|
||||
/// persistent health error.
|
||||
pub fn close_registries(&self, error: Arc<Error>) {
|
||||
let persistent_error = self.record_health_error(error);
|
||||
self.publish_unhealthy();
|
||||
let request_senders = self.request_reg.lock().close();
|
||||
let utility_senders = self.utility_reg.lock().close();
|
||||
|
||||
@@ -191,6 +194,12 @@ impl ClientInner {
|
||||
self.health_error.load().is_none()
|
||||
}
|
||||
|
||||
/// Subscribe to engine health changes. The current value is `true` while
|
||||
/// the client is healthy and changes permanently to `false` on failure.
|
||||
pub fn subscribe_health(&self) -> watch::Receiver<bool> {
|
||||
self.health_tx.subscribe()
|
||||
}
|
||||
|
||||
/// Resolve one utility output to the waiting caller. Returns `true` if a
|
||||
/// waiting caller existed.
|
||||
pub fn resolve_utility_output(&self, output: UtilityOutput) -> bool {
|
||||
@@ -280,6 +289,11 @@ impl ClientInner {
|
||||
.expect("health error must be recorded before registries close")
|
||||
}
|
||||
|
||||
/// Publish the sticky healthy-to-unhealthy transition.
|
||||
fn publish_unhealthy(&self) {
|
||||
self.health_tx.send_if_modified(|healthy| std::mem::replace(healthy, false));
|
||||
}
|
||||
|
||||
/// Assert there is a recorded health error and return a `Shared` variant
|
||||
/// wrapping it for error returns when the client is already closed.
|
||||
fn closed_error(&self) -> Error {
|
||||
@@ -461,13 +475,18 @@ mod tests {
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
async fn close_registries_records_first_health_error_only() {
|
||||
let inner = test_inner().await;
|
||||
let mut health = inner.subscribe_health();
|
||||
assert!(*health.borrow());
|
||||
|
||||
inner.close_registries(Arc::new(Error::EngineCoreDead));
|
||||
health.changed().await.expect("health sender remains open");
|
||||
assert!(!inner.is_healthy());
|
||||
assert!(!*health.borrow());
|
||||
assert!(matches!(
|
||||
inner.health_error().as_deref(),
|
||||
Some(Error::EngineCoreDead)
|
||||
));
|
||||
assert!(!*inner.subscribe_health().borrow());
|
||||
|
||||
inner.close_registries(Arc::new(client_closed!("shutdown")));
|
||||
assert!(matches!(
|
||||
|
||||
Reference in New Issue
Block a user