[Rust Frontend][gRPC] Add RL lifecycle control (#51316)
Signed-off-by: Connor Carpenter <connorc@nvidia.com> Co-authored-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
co-authored by
Nick Hill
parent
3e3ceb1961
commit
f473870ecf
@@ -3,9 +3,13 @@
|
||||
|
||||
use std::sync::Arc;
|
||||
|
||||
use serde_json::Value as JsonValue;
|
||||
use thiserror_ext::AsReport as _;
|
||||
use tokio::sync::Mutex;
|
||||
use tonic::{Request, Response, Status};
|
||||
use vllm_engine_core_client::EngineCoreClient;
|
||||
use vllm_engine_core_client::protocol::handshake::EngineCoreReadyResponse;
|
||||
use vllm_engine_core_client::protocol::utility::PauseMode as EnginePauseMode;
|
||||
|
||||
use super::{ControlServer, pb};
|
||||
use crate::state::AppState;
|
||||
@@ -15,17 +19,25 @@ pub(crate) type ControlGrpcService = ControlServer<ControlServiceImpl>;
|
||||
/// gRPC control service backed by the shared application state.
|
||||
pub struct ControlServiceImpl {
|
||||
state: Arc<AppState>,
|
||||
rl_lock: Mutex<()>,
|
||||
}
|
||||
|
||||
impl ControlServiceImpl {
|
||||
pub fn new(state: Arc<AppState>) -> Self {
|
||||
Self { state }
|
||||
Self {
|
||||
state,
|
||||
rl_lock: Mutex::new(()),
|
||||
}
|
||||
}
|
||||
|
||||
fn ready(&self) -> &EngineCoreReadyResponse {
|
||||
self.state.engine_core_client().ready_response()
|
||||
}
|
||||
|
||||
fn client(&self) -> &EngineCoreClient {
|
||||
self.state.engine_core_client()
|
||||
}
|
||||
|
||||
fn parallelism_info(&self) -> pb::ParallelismInfo {
|
||||
let ready = self.ready();
|
||||
pb::ParallelismInfo {
|
||||
@@ -36,10 +48,91 @@ impl ControlServiceImpl {
|
||||
decode_context_parallel_size: ready.decode_context_parallel_size,
|
||||
}
|
||||
}
|
||||
|
||||
fn weight_transfer_backend(&self) -> Option<&str> {
|
||||
let responses = self.client().ready_responses();
|
||||
let backend = responses.first()?.weight_transfer_backend.as_deref()?;
|
||||
responses
|
||||
.iter()
|
||||
.all(|ready| ready.weight_transfer_backend.as_deref() == Some(backend))
|
||||
.then_some(backend)
|
||||
}
|
||||
|
||||
fn sleep_mode_enabled(&self) -> bool {
|
||||
self.client().ready_responses().iter().all(|ready| ready.enable_sleep_mode)
|
||||
}
|
||||
|
||||
fn draft_weight_updates_enabled(&self) -> bool {
|
||||
self.client()
|
||||
.ready_responses()
|
||||
.iter()
|
||||
.all(|ready| ready.supports_draft_weight_updates)
|
||||
}
|
||||
|
||||
fn rl_capabilities(&self) -> pb::RlCapabilities {
|
||||
let backend = self.weight_transfer_backend();
|
||||
pb::RlCapabilities {
|
||||
weight_transfer_enabled: backend.is_some(),
|
||||
weight_transfer_backend: backend.unwrap_or_default().to_string(),
|
||||
sleep_mode_enabled: self.sleep_mode_enabled(),
|
||||
draft_weight_updates_enabled: self.draft_weight_updates_enabled(),
|
||||
}
|
||||
}
|
||||
|
||||
fn require_weight_transfer(&self) -> Result<(), Status> {
|
||||
self.weight_transfer_backend().map(|_| ()).ok_or_else(|| {
|
||||
Status::failed_precondition(
|
||||
"weight transfer is not configured; start vLLM with --weight-transfer-config",
|
||||
)
|
||||
})
|
||||
}
|
||||
|
||||
fn require_sleep_mode(&self) -> Result<(), Status> {
|
||||
self.sleep_mode_enabled().then_some(()).ok_or_else(|| {
|
||||
Status::failed_precondition(
|
||||
"sleep mode is not configured; start vLLM with --enable-sleep-mode",
|
||||
)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
const GRPC_API_VERSION: &str = "vllm";
|
||||
|
||||
fn utility_status(method: &'static str, error: vllm_engine_core_client::Error) -> Status {
|
||||
Status::internal(format!("{method} failed: {}", error.to_report_string()))
|
||||
}
|
||||
|
||||
fn pause_mode(mode: i32) -> Result<EnginePauseMode, Status> {
|
||||
match pb::PauseMode::try_from(mode) {
|
||||
Ok(pb::PauseMode::Unspecified | pb::PauseMode::Abort) => Ok(EnginePauseMode::Abort),
|
||||
Ok(pb::PauseMode::Wait) => Ok(EnginePauseMode::Wait),
|
||||
Ok(pb::PauseMode::Keep) => Ok(EnginePauseMode::Keep),
|
||||
Err(_) => Err(Status::invalid_argument("invalid pause mode")),
|
||||
}
|
||||
}
|
||||
|
||||
fn json_object(bytes: &[u8], field: &'static str) -> Result<JsonValue, Status> {
|
||||
let value = serde_json::from_slice::<JsonValue>(bytes).map_err(|error| {
|
||||
Status::invalid_argument(format!(
|
||||
"{field} must contain valid JSON: {}",
|
||||
error.to_report_string()
|
||||
))
|
||||
})?;
|
||||
if !value.is_object() {
|
||||
return Err(Status::invalid_argument(format!(
|
||||
"{field} must contain a JSON object"
|
||||
)));
|
||||
}
|
||||
Ok(value)
|
||||
}
|
||||
|
||||
fn weight_version(value: String) -> Result<String, Status> {
|
||||
if value.trim().is_empty() {
|
||||
return Err(Status::invalid_argument("weight_version must not be empty"));
|
||||
}
|
||||
Ok(value)
|
||||
}
|
||||
|
||||
#[tonic::async_trait]
|
||||
impl pb::control_server::Control for ControlServiceImpl {
|
||||
async fn get_server_info(
|
||||
@@ -57,6 +150,7 @@ impl pb::control_server::Control for ControlServiceImpl {
|
||||
total_kv_blocks: self.state.engine_core_client().total_num_gpu_blocks(),
|
||||
max_running_requests: ready.max_num_seqs,
|
||||
max_batched_tokens: ready.max_num_batched_tokens,
|
||||
rl_capabilities: Some(self.rl_capabilities()),
|
||||
}))
|
||||
}
|
||||
|
||||
@@ -112,6 +206,194 @@ impl pb::control_server::Control for ControlServiceImpl {
|
||||
let sources = client.ready_responses().into_iter().filter_map(kv_event_source).collect();
|
||||
Ok(Response::new(pb::GetKvEventSourcesResponse { sources }))
|
||||
}
|
||||
|
||||
async fn pause_generation(
|
||||
&self,
|
||||
request: Request<pb::PauseGenerationRequest>,
|
||||
) -> Result<Response<pb::PauseGenerationResponse>, Status> {
|
||||
let request = request.into_inner();
|
||||
let mode = pause_mode(request.mode)?;
|
||||
let clear_cache = request.clear_cache.unwrap_or(true);
|
||||
let _guard = self.rl_lock.lock().await;
|
||||
self.client()
|
||||
.pause_scheduler(mode, clear_cache)
|
||||
.await
|
||||
.map_err(|error| utility_status("pause_generation", error))?;
|
||||
Ok(Response::new(pb::PauseGenerationResponse {}))
|
||||
}
|
||||
|
||||
async fn resume_generation(
|
||||
&self,
|
||||
_request: Request<pb::ResumeGenerationRequest>,
|
||||
) -> Result<Response<pb::ResumeGenerationResponse>, Status> {
|
||||
let _guard = self.rl_lock.lock().await;
|
||||
self.client()
|
||||
.resume_scheduler()
|
||||
.await
|
||||
.map_err(|error| utility_status("resume_generation", error))?;
|
||||
Ok(Response::new(pb::ResumeGenerationResponse {}))
|
||||
}
|
||||
|
||||
async fn is_paused(
|
||||
&self,
|
||||
_request: Request<pb::IsPausedRequest>,
|
||||
) -> Result<Response<pb::IsPausedResponse>, Status> {
|
||||
let paused = self
|
||||
.client()
|
||||
.is_scheduler_paused()
|
||||
.await
|
||||
.map_err(|error| utility_status("is_paused", error))?;
|
||||
Ok(Response::new(pb::IsPausedResponse { paused }))
|
||||
}
|
||||
|
||||
async fn sleep(
|
||||
&self,
|
||||
request: Request<pb::SleepRequest>,
|
||||
) -> Result<Response<pb::SleepResponse>, Status> {
|
||||
self.require_sleep_mode()?;
|
||||
let request = request.into_inner();
|
||||
let mode = pause_mode(request.mode)?;
|
||||
let level = request.level.unwrap_or(1);
|
||||
let _guard = self.rl_lock.lock().await;
|
||||
self.client()
|
||||
.sleep(level, mode)
|
||||
.await
|
||||
.map_err(|error| utility_status("sleep", error))?;
|
||||
Ok(Response::new(pb::SleepResponse {}))
|
||||
}
|
||||
|
||||
async fn wake_up(
|
||||
&self,
|
||||
request: Request<pb::WakeUpRequest>,
|
||||
) -> Result<Response<pb::WakeUpResponse>, Status> {
|
||||
self.require_sleep_mode()?;
|
||||
let tags = request.into_inner().tags;
|
||||
let tags = (!tags.is_empty()).then_some(tags);
|
||||
let _guard = self.rl_lock.lock().await;
|
||||
self.client()
|
||||
.wake_up(tags)
|
||||
.await
|
||||
.map_err(|error| utility_status("wake_up", error))?;
|
||||
Ok(Response::new(pb::WakeUpResponse {}))
|
||||
}
|
||||
|
||||
async fn is_sleeping(
|
||||
&self,
|
||||
_request: Request<pb::IsSleepingRequest>,
|
||||
) -> Result<Response<pb::IsSleepingResponse>, Status> {
|
||||
let sleeping = self
|
||||
.client()
|
||||
.is_sleeping()
|
||||
.await
|
||||
.map_err(|error| utility_status("is_sleeping", error))?;
|
||||
Ok(Response::new(pb::IsSleepingResponse { sleeping }))
|
||||
}
|
||||
|
||||
async fn init_weight_transfer_engine(
|
||||
&self,
|
||||
request: Request<pb::InitWeightTransferEngineRequest>,
|
||||
) -> Result<Response<pb::InitWeightTransferEngineResponse>, Status> {
|
||||
self.require_weight_transfer()?;
|
||||
let init_info = json_object(&request.into_inner().init_info_json, "init_info_json")?;
|
||||
let _guard = self.rl_lock.lock().await;
|
||||
self.client()
|
||||
.init_weight_transfer_engine(init_info)
|
||||
.await
|
||||
.map_err(|error| utility_status("init_weight_transfer_engine", error))?;
|
||||
Ok(Response::new(pb::InitWeightTransferEngineResponse {}))
|
||||
}
|
||||
|
||||
async fn start_weight_update(
|
||||
&self,
|
||||
_request: Request<pb::StartWeightUpdateRequest>,
|
||||
) -> Result<Response<pb::StartWeightUpdateResponse>, Status> {
|
||||
self.require_weight_transfer()?;
|
||||
let _guard = self.rl_lock.lock().await;
|
||||
self.client()
|
||||
.start_weight_update()
|
||||
.await
|
||||
.map_err(|error| utility_status("start_weight_update", error))?;
|
||||
Ok(Response::new(pb::StartWeightUpdateResponse {}))
|
||||
}
|
||||
|
||||
async fn start_draft_weight_update(
|
||||
&self,
|
||||
_request: Request<pb::StartDraftWeightUpdateRequest>,
|
||||
) -> Result<Response<pb::StartDraftWeightUpdateResponse>, Status> {
|
||||
self.require_weight_transfer()?;
|
||||
if !self.draft_weight_updates_enabled() {
|
||||
return Err(Status::failed_precondition(
|
||||
"draft weight updates require a configured speculative draft model",
|
||||
));
|
||||
}
|
||||
let _guard = self.rl_lock.lock().await;
|
||||
self.client()
|
||||
.start_draft_weight_update()
|
||||
.await
|
||||
.map_err(|error| utility_status("start_draft_weight_update", error))?;
|
||||
Ok(Response::new(pb::StartDraftWeightUpdateResponse {}))
|
||||
}
|
||||
|
||||
async fn update_weights(
|
||||
&self,
|
||||
request: Request<pb::UpdateWeightsRequest>,
|
||||
) -> Result<Response<pb::UpdateWeightsResponse>, Status> {
|
||||
self.require_weight_transfer()?;
|
||||
let update_info = json_object(&request.into_inner().update_info_json, "update_info_json")?;
|
||||
let _guard = self.rl_lock.lock().await;
|
||||
self.client()
|
||||
.update_weights(update_info)
|
||||
.await
|
||||
.map_err(|error| utility_status("update_weights", error))?;
|
||||
Ok(Response::new(pb::UpdateWeightsResponse {}))
|
||||
}
|
||||
|
||||
async fn finish_weight_update(
|
||||
&self,
|
||||
request: Request<pb::FinishWeightUpdateRequest>,
|
||||
) -> Result<Response<pb::FinishWeightUpdateResponse>, Status> {
|
||||
self.require_weight_transfer()?;
|
||||
let version = request.into_inner().weight_version.map(weight_version).transpose()?;
|
||||
let _guard = self.rl_lock.lock().await;
|
||||
self.client()
|
||||
.finish_weight_update()
|
||||
.await
|
||||
.map_err(|error| utility_status("finish_weight_update", error))?;
|
||||
if let Some(version) = version {
|
||||
self.client()
|
||||
.set_weight_version(&version)
|
||||
.await
|
||||
.map_err(|error| utility_status("update_weight_version", error))?;
|
||||
}
|
||||
Ok(Response::new(pb::FinishWeightUpdateResponse {}))
|
||||
}
|
||||
|
||||
async fn update_weight_version(
|
||||
&self,
|
||||
request: Request<pb::UpdateWeightVersionRequest>,
|
||||
) -> Result<Response<pb::UpdateWeightVersionResponse>, Status> {
|
||||
let version = weight_version(request.into_inner().weight_version)?;
|
||||
let _guard = self.rl_lock.lock().await;
|
||||
self.client()
|
||||
.set_weight_version(&version)
|
||||
.await
|
||||
.map_err(|error| utility_status("update_weight_version", error))?;
|
||||
Ok(Response::new(pb::UpdateWeightVersionResponse {}))
|
||||
}
|
||||
|
||||
async fn get_weight_version(
|
||||
&self,
|
||||
_request: Request<pb::GetWeightVersionRequest>,
|
||||
) -> Result<Response<pb::GetWeightVersionResponse>, Status> {
|
||||
let weight_version = self
|
||||
.client()
|
||||
.get_weight_version()
|
||||
.await
|
||||
.map_err(|error| utility_status("get_weight_version", error))?;
|
||||
Ok(Response::new(pb::GetWeightVersionResponse {
|
||||
weight_version,
|
||||
}))
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn kv_event_source(response: &EngineCoreReadyResponse) -> Option<pb::KvEventSource> {
|
||||
|
||||
@@ -12,6 +12,7 @@ use std::time::Duration;
|
||||
use futures::StreamExt as _;
|
||||
use hyper_util::rt::TokioIo;
|
||||
use openssl::ssl::{SslConnector, SslFiletype, SslMethod};
|
||||
use rmpv::Value;
|
||||
use serial_test::serial;
|
||||
use tokio::io::{AsyncReadExt as _, AsyncWriteExt as _};
|
||||
use tokio::net::TcpStream;
|
||||
@@ -30,14 +31,15 @@ use vllm_engine_core_client::mock_engine::{
|
||||
DEFAULT_MOCK_BLOCK_SIZE, DEFAULT_MOCK_MAX_MODEL_LEN, DEFAULT_MOCK_NUM_GPU_BLOCKS,
|
||||
default_ready_response,
|
||||
};
|
||||
use vllm_engine_core_client::protocol::handshake::KvEventsConfig;
|
||||
use vllm_engine_core_client::protocol::decode_value;
|
||||
use vllm_engine_core_client::protocol::handshake::{EngineCoreReadyResponse, KvEventsConfig};
|
||||
use vllm_engine_core_client::protocol::output::{
|
||||
EngineCoreFinishReason, EngineCoreOutput, EngineCoreOutputs, RequestBatchOutputs,
|
||||
UtilityCallOutput,
|
||||
};
|
||||
use vllm_engine_core_client::protocol::request::EngineCoreRequest;
|
||||
use vllm_engine_core_client::test_utils::{
|
||||
IpcNamespace, spawn_mock_engine_task, spawn_mock_engine_task_with_ready,
|
||||
};
|
||||
use vllm_engine_core_client::protocol::utility::{UtilityOutput, UtilityResultEnvelope};
|
||||
use vllm_engine_core_client::test_utils::{IpcNamespace, spawn_mock_engine_task_with_ready};
|
||||
use vllm_engine_core_client::{EngineCoreClient, EngineCoreClientConfig, EngineId, TransportMode};
|
||||
use vllm_llm::Llm;
|
||||
use vllm_text::tokenizer::DynTokenizer;
|
||||
@@ -291,13 +293,10 @@ async fn setup_grpc_service_with_backend<F>(
|
||||
where
|
||||
F: FnOnce(&EngineCoreRequest) + Send + 'static,
|
||||
{
|
||||
let ipc = IpcNamespace::new().expect("create ipc namespace");
|
||||
let handshake_address = ipc.handshake_endpoint();
|
||||
let engine_id = engine_id.into();
|
||||
|
||||
let engine_task = MockEngineTask::new(spawn_mock_engine_task(
|
||||
handshake_address.clone(),
|
||||
engine_id.clone(),
|
||||
setup_grpc_service_with_engine_script(
|
||||
engine_id,
|
||||
default_ready_response(),
|
||||
backend,
|
||||
move |dealer, push| {
|
||||
boxed_test_future(async move {
|
||||
let add = recv_engine_message(dealer).await;
|
||||
@@ -311,6 +310,33 @@ where
|
||||
.await;
|
||||
})
|
||||
},
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn setup_grpc_service_with_engine_script<F>(
|
||||
engine_id: impl Into<EngineId>,
|
||||
ready: EngineCoreReadyResponse,
|
||||
backend: Arc<dyn ChatTextBackend>,
|
||||
script: F,
|
||||
) -> (
|
||||
InferenceServer<InferenceServiceImpl>,
|
||||
ControlServer<ControlServiceImpl>,
|
||||
tokio::sync::watch::Receiver<bool>,
|
||||
MockEngineTask,
|
||||
)
|
||||
where
|
||||
F: for<'a> FnOnce(&'a mut DealerSocket, &'a mut PushSocket) -> TestFuture<'a> + Send + 'static,
|
||||
{
|
||||
let ipc = IpcNamespace::new().expect("create ipc namespace");
|
||||
let handshake_address = ipc.handshake_endpoint();
|
||||
let engine_id = engine_id.into();
|
||||
|
||||
let engine_task = MockEngineTask::new(spawn_mock_engine_task_with_ready(
|
||||
handshake_address.clone(),
|
||||
engine_id.clone(),
|
||||
ready,
|
||||
script,
|
||||
));
|
||||
|
||||
let client = EngineCoreClient::connect(
|
||||
@@ -1509,6 +1535,11 @@ async fn control_reports_server_and_model_info() {
|
||||
assert_eq!(parallelism.data_parallel_size, 1);
|
||||
assert_eq!(parallelism.data_parallel_rank, 0);
|
||||
assert_eq!(parallelism.decode_context_parallel_size, 1);
|
||||
let rl = server.rl_capabilities.expect("RL capabilities");
|
||||
assert!(!rl.weight_transfer_enabled);
|
||||
assert!(rl.weight_transfer_backend.is_empty());
|
||||
assert!(!rl.sleep_mode_enabled);
|
||||
assert!(!rl.draft_weight_updates_enabled);
|
||||
|
||||
let model = client
|
||||
.get_model_info(pb::GetModelInfoRequest {})
|
||||
@@ -1527,6 +1558,65 @@ async fn control_reports_server_and_model_info() {
|
||||
server_task.abort();
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
#[serial]
|
||||
async fn control_forwards_weight_update_without_pause_guard() {
|
||||
let mut ready = default_ready_response();
|
||||
ready.weight_transfer_backend = Some("nccl".to_string());
|
||||
let (inference_service, control_service, engine_health, engine_task) =
|
||||
setup_grpc_service_with_engine_script(
|
||||
b"engine-grpc-rl".to_vec(),
|
||||
ready,
|
||||
Arc::new(FakeTextBackend),
|
||||
|dealer, push| {
|
||||
boxed_test_future(async move {
|
||||
let frames = recv_engine_message(dealer).await;
|
||||
assert_eq!(frames[0].as_ref(), &[0x03]);
|
||||
let payload = decode_value(&frames[1]).expect("decode utility payload");
|
||||
let fields = payload.as_array().expect("utility payload array");
|
||||
let call_id = fields[1].as_u64().expect("utility call id");
|
||||
assert_eq!(fields[2].as_str(), Some("collective_rpc"));
|
||||
let args = fields[3].as_array().expect("collective_rpc arguments");
|
||||
assert_eq!(args[0].as_str(), Some("update_weights"));
|
||||
send_outputs(
|
||||
push,
|
||||
UtilityCallOutput {
|
||||
output: UtilityOutput {
|
||||
call_id: call_id.into(),
|
||||
failure_message: None,
|
||||
result: Some(UtilityResultEnvelope::without_type_info(
|
||||
Value::Array(vec![Value::Nil]),
|
||||
)),
|
||||
},
|
||||
..Default::default()
|
||||
}
|
||||
.into(),
|
||||
)
|
||||
.await;
|
||||
})
|
||||
},
|
||||
)
|
||||
.await;
|
||||
let (channel, server_task) = start_grpc_test_server(
|
||||
inference_service,
|
||||
control_service,
|
||||
engine_health,
|
||||
tokio_util::sync::CancellationToken::new(),
|
||||
)
|
||||
.await;
|
||||
let mut client = ControlClient::new(channel);
|
||||
|
||||
client
|
||||
.update_weights(pb::UpdateWeightsRequest {
|
||||
update_info_json: br#"{"names":["model.weight"]}"#.to_vec(),
|
||||
})
|
||||
.await
|
||||
.expect("forward weight update without a pause probe");
|
||||
|
||||
engine_task.await.expect("mock engine task");
|
||||
server_task.abort();
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
async fn control_aggregates_multi_engine_capacity() {
|
||||
let ipc = IpcNamespace::new().expect("create ipc namespace");
|
||||
@@ -1536,6 +1626,9 @@ async fn control_aggregates_multi_engine_capacity() {
|
||||
ready_0.max_model_len = 8_192;
|
||||
ready_0.num_gpu_blocks = 10;
|
||||
ready_0.data_parallel_size = 2;
|
||||
ready_0.weight_transfer_backend = Some("nccl".to_string());
|
||||
ready_0.enable_sleep_mode = true;
|
||||
ready_0.supports_draft_weight_updates = true;
|
||||
|
||||
let mut ready_1 = default_ready_response();
|
||||
ready_1.max_model_len = 4_096;
|
||||
@@ -1586,6 +1679,11 @@ async fn control_aggregates_multi_engine_capacity() {
|
||||
.into_inner();
|
||||
assert_eq!(server.max_model_len, 4_096);
|
||||
assert_eq!(server.total_kv_blocks, 30);
|
||||
let rl = server.rl_capabilities.expect("RL capabilities");
|
||||
assert!(!rl.weight_transfer_enabled);
|
||||
assert!(rl.weight_transfer_backend.is_empty());
|
||||
assert!(!rl.sleep_mode_enabled);
|
||||
assert!(!rl.draft_weight_updates_enabled);
|
||||
assert_eq!(server.parallelism.unwrap().data_parallel_size, 4);
|
||||
|
||||
drop(engine_tasks);
|
||||
|
||||
@@ -217,7 +217,8 @@ where
|
||||
health_reporter.set_serving::<grpc::InferenceGrpcService>().await;
|
||||
health_reporter.set_serving::<grpc::ControlGrpcService>().await;
|
||||
let control_service =
|
||||
grpc::ControlGrpcService::new(grpc::ControlServiceImpl::new(state.clone()));
|
||||
grpc::ControlGrpcService::new(grpc::ControlServiceImpl::new(state.clone()))
|
||||
.max_decoding_message_size(DEFAULT_REQUEST_BODY_LIMIT_BYTES);
|
||||
let inference_service =
|
||||
grpc::InferenceGrpcService::new(grpc::InferenceServiceImpl::new(state.clone()))
|
||||
.max_decoding_message_size(DEFAULT_REQUEST_BODY_LIMIT_BYTES);
|
||||
|
||||
Reference in New Issue
Block a user