[Rust Frontend][gRPC] Add explicit data-parallel rank routing (#51178)
Signed-off-by: Connor Carpenter <connorc@nvidia.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
3c6b2e9c08
commit
fa722b9f01
@@ -58,6 +58,7 @@ async fn main() -> Result<()> {
|
||||
local_input_address: None,
|
||||
local_output_address: None,
|
||||
},
|
||||
data_parallel_size: args.engine_count,
|
||||
coordinator_mode: CoordinatorMode::MaybeInProc,
|
||||
model: args.model,
|
||||
served_model_name: vec![],
|
||||
|
||||
@@ -161,6 +161,8 @@ impl TlsConfig {
|
||||
pub struct Config {
|
||||
/// Frontend-to-engine transport setup.
|
||||
pub transport_mode: TransportMode,
|
||||
/// Deployment-wide data-parallel size retained by the frontend.
|
||||
pub data_parallel_size: usize,
|
||||
/// Requested frontend-side coordinator behavior.
|
||||
pub coordinator_mode: CoordinatorMode,
|
||||
/// Backend model identifier used for engine-core loading.
|
||||
@@ -237,6 +239,46 @@ impl Config {
|
||||
max_logprobs
|
||||
);
|
||||
}
|
||||
if self.data_parallel_size == 0 {
|
||||
bail!("data parallel size must be at least 1");
|
||||
}
|
||||
if self.data_parallel_size > usize::from(u16::MAX) + 1 {
|
||||
bail!(
|
||||
"data parallel size ({}) exceeds the two-byte engine identity limit",
|
||||
self.data_parallel_size
|
||||
);
|
||||
}
|
||||
match &self.transport_mode {
|
||||
TransportMode::HandshakeOwner { engine_count, .. } => {
|
||||
if *engine_count != self.data_parallel_size {
|
||||
bail!(
|
||||
"managed frontend engine count ({engine_count}) must equal data parallel size ({})",
|
||||
self.data_parallel_size
|
||||
);
|
||||
}
|
||||
}
|
||||
TransportMode::Bootstrapped {
|
||||
engine_start_index,
|
||||
engine_count,
|
||||
..
|
||||
} => {
|
||||
if *engine_count == 0 {
|
||||
bail!("engine count must be at least 1");
|
||||
}
|
||||
let engine_start_index = usize::try_from(*engine_start_index)
|
||||
.map_err(|_| anyhow::anyhow!("engine start index does not fit usize"))?;
|
||||
let engine_end_index =
|
||||
engine_start_index.checked_add(*engine_count).ok_or_else(|| {
|
||||
anyhow::anyhow!("engine start index + engine count overflows")
|
||||
})?;
|
||||
if engine_end_index > self.data_parallel_size {
|
||||
bail!(
|
||||
"connected engine range [{engine_start_index}, {engine_end_index}) exceeds data parallel size {}",
|
||||
self.data_parallel_size
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -31,7 +31,7 @@ impl ControlServiceImpl {
|
||||
pb::ParallelismInfo {
|
||||
tensor_parallel_size: ready.tensor_parallel_size,
|
||||
pipeline_parallel_size: ready.pipeline_parallel_size,
|
||||
data_parallel_size: ready.data_parallel_size.min(u64::from(u32::MAX)) as u32,
|
||||
data_parallel_size: self.state.data_parallel_size().min(u32::MAX as usize) as u32,
|
||||
data_parallel_rank: ready.data_parallel_rank,
|
||||
decode_context_parallel_size: ready.decode_context_parallel_size,
|
||||
}
|
||||
|
||||
@@ -22,6 +22,8 @@ use crate::state::AppState;
|
||||
|
||||
pub(crate) type InferenceGrpcService = InferenceServer<InferenceServiceImpl>;
|
||||
|
||||
const DATA_PARALLEL_RANK_METADATA_KEY: &str = "x-data-parallel-rank";
|
||||
|
||||
/// gRPC inference service backed by the shared application state.
|
||||
pub struct InferenceServiceImpl {
|
||||
state: Arc<AppState>,
|
||||
@@ -41,6 +43,7 @@ impl InferenceServiceImpl {
|
||||
async fn prepare_request(
|
||||
&self,
|
||||
mut proto_request: pb::GenerateRequest,
|
||||
data_parallel_rank: Option<u32>,
|
||||
stream: bool,
|
||||
rpc: &'static str,
|
||||
) -> Result<PreparedGrpcRequest, Status> {
|
||||
@@ -62,6 +65,7 @@ impl InferenceServiceImpl {
|
||||
rpc,
|
||||
%model,
|
||||
media_count,
|
||||
?data_parallel_rank,
|
||||
);
|
||||
info!(parent: &request_span, "gRPC inference request received");
|
||||
|
||||
@@ -70,6 +74,7 @@ impl InferenceServiceImpl {
|
||||
let mut text_request =
|
||||
convert::to_text_request(proto_request, stream, self.state.served_model_names())?;
|
||||
text_request.arrival_time = Some(arrival_time);
|
||||
text_request.data_parallel_rank = data_parallel_rank;
|
||||
|
||||
let media = convert::media_parts_from_request(media)?;
|
||||
if !media.is_empty() {
|
||||
@@ -115,6 +120,20 @@ impl InferenceServiceImpl {
|
||||
}
|
||||
}
|
||||
|
||||
fn data_parallel_rank_from_metadata(
|
||||
request: &Request<pb::GenerateRequest>,
|
||||
) -> Result<Option<u32>, Status> {
|
||||
let Some(value) = request.metadata().get(DATA_PARALLEL_RANK_METADATA_KEY) else {
|
||||
return Ok(None);
|
||||
};
|
||||
let value = value.to_str().map_err(|_| {
|
||||
Status::invalid_argument("x-data-parallel-rank metadata must be an unsigned 32-bit integer")
|
||||
})?;
|
||||
value.trim().parse::<u32>().map(Some).map_err(|_| {
|
||||
Status::invalid_argument("x-data-parallel-rank metadata must be an unsigned 32-bit integer")
|
||||
})
|
||||
}
|
||||
|
||||
#[tonic::async_trait]
|
||||
impl pb::inference_server::Inference for InferenceServiceImpl {
|
||||
type GenerateStreamStream =
|
||||
@@ -125,13 +144,14 @@ impl pb::inference_server::Inference for InferenceServiceImpl {
|
||||
&self,
|
||||
request: Request<pb::GenerateRequest>,
|
||||
) -> Result<Response<pb::GenerateResponse>, Status> {
|
||||
let data_parallel_rank = data_parallel_rank_from_metadata(&request)?;
|
||||
let proto_req = request.into_inner();
|
||||
let response_opts = ResponseOpts::from_proto(proto_req.response.as_ref());
|
||||
let PreparedGrpcRequest {
|
||||
text_request,
|
||||
request_span,
|
||||
started_at,
|
||||
} = self.prepare_request(proto_req, false, "Generate").await?;
|
||||
} = self.prepare_request(proto_req, data_parallel_rank, false, "Generate").await?;
|
||||
|
||||
let stream = self
|
||||
.state
|
||||
@@ -186,13 +206,16 @@ impl pb::inference_server::Inference for InferenceServiceImpl {
|
||||
&self,
|
||||
request: Request<pb::GenerateRequest>,
|
||||
) -> Result<Response<Self::GenerateStreamStream>, Status> {
|
||||
let data_parallel_rank = data_parallel_rank_from_metadata(&request)?;
|
||||
let proto_req = request.into_inner();
|
||||
let response_opts = ResponseOpts::from_proto(proto_req.response.as_ref());
|
||||
let PreparedGrpcRequest {
|
||||
text_request,
|
||||
request_span,
|
||||
started_at,
|
||||
} = self.prepare_request(proto_req, true, "GenerateStream").await?;
|
||||
} = self
|
||||
.prepare_request(proto_req, data_parallel_rank, true, "GenerateStream")
|
||||
.await?;
|
||||
|
||||
let stream = self
|
||||
.state
|
||||
|
||||
@@ -843,6 +843,37 @@ async fn unary_generate_missing_prompt_returns_invalid_argument() {
|
||||
server_task.abort();
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
#[serial]
|
||||
async fn unary_generate_unconnected_data_parallel_rank_returns_invalid_argument() {
|
||||
let (mut client, server_task, _engine_task) = grpc_test_server(
|
||||
EngineId::from_engine_index(3),
|
||||
default_stream_output_specs(),
|
||||
)
|
||||
.await;
|
||||
|
||||
let mut request = tonic::Request::new(pb::GenerateRequest {
|
||||
request_id: "test-unconnected-dp-rank".to_string(),
|
||||
model: "test-model".to_string(),
|
||||
prompt: Some(pb::generate_request::Prompt::Text("hello".to_string())),
|
||||
..Default::default()
|
||||
});
|
||||
request.metadata_mut().insert(
|
||||
"x-data-parallel-rank",
|
||||
"0".parse().expect("valid metadata value"),
|
||||
);
|
||||
|
||||
let status = client
|
||||
.generate(request)
|
||||
.await
|
||||
.expect_err("rank 0 should not select globally ranked engine 3");
|
||||
|
||||
assert_eq!(status.code(), tonic::Code::InvalidArgument);
|
||||
assert!(status.message().contains("connected ranks: [3]"));
|
||||
|
||||
server_task.abort();
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
#[serial]
|
||||
async fn unary_generate_min_tokens_above_max_tokens_returns_invalid_argument() {
|
||||
@@ -1523,7 +1554,9 @@ async fn control_aggregates_multi_engine_capacity() {
|
||||
ready_1.data_parallel_rank = 1;
|
||||
|
||||
let engine_tasks = [ready_0, ready_1].map(|ready| {
|
||||
let engine_id = EngineId::from_engine_index(ready.data_parallel_rank);
|
||||
let engine_id = EngineId::from_engine_index(
|
||||
ready.data_parallel_rank.try_into().expect("test rank fits engine identity"),
|
||||
);
|
||||
MockEngineTask::new(spawn_mock_engine_task_with_ready(
|
||||
handshake_address.clone(),
|
||||
engine_id,
|
||||
@@ -1551,10 +1584,8 @@ async fn control_aggregates_multi_engine_capacity() {
|
||||
Llm::new(client),
|
||||
Arc::new(FakeTextBackend) as Arc<dyn ChatTextBackend>,
|
||||
);
|
||||
let service = ControlServiceImpl::new(Arc::new(AppState::new(
|
||||
vec!["test-model".to_string()],
|
||||
chat,
|
||||
)));
|
||||
let state = AppState::new(vec!["test-model".to_string()], chat).with_data_parallel_size(4);
|
||||
let service = ControlServiceImpl::new(Arc::new(state));
|
||||
|
||||
let server = pb::control_server::Control::get_server_info(
|
||||
&service,
|
||||
@@ -1565,6 +1596,7 @@ async fn control_aggregates_multi_engine_capacity() {
|
||||
.into_inner();
|
||||
assert_eq!(server.max_model_len, 4_096);
|
||||
assert_eq!(server.total_kv_blocks, 30);
|
||||
assert_eq!(server.parallelism.unwrap().data_parallel_size, 4);
|
||||
|
||||
drop(engine_tasks);
|
||||
}
|
||||
|
||||
@@ -141,6 +141,7 @@ async fn build_state(config: &Config) -> Result<Arc<AppState>> {
|
||||
.with_model_path(config.model.clone())
|
||||
.with_api_server_options(config.api_server_options)
|
||||
.with_server_info(ServerInfoSnapshot::from_config(config))
|
||||
.with_data_parallel_size(config.data_parallel_size)
|
||||
.with_api_keys(config.api_keys.clone())
|
||||
.with_cors(config.cors.clone())
|
||||
.with_profiler(config.profiler.clone()),
|
||||
|
||||
@@ -725,11 +725,11 @@ async fn test_app_with_dev_mode(dev_mode_enabled: bool) -> axum::Router {
|
||||
/// stays alive for the duration of the test.
|
||||
async fn test_dev_mode_app_with_ready(
|
||||
ready_response: vllm_engine_core_client::protocol::handshake::EngineCoreReadyResponse,
|
||||
data_parallel_size: usize,
|
||||
) -> (axum::Router, MockEngineTask) {
|
||||
let ipc = IpcNamespace::new().expect("create ipc namespace");
|
||||
let handshake_address = ipc.handshake_endpoint();
|
||||
let engine_id = b"engine-world-size".to_vec();
|
||||
|
||||
let engine_task = MockEngineTask::new(spawn_mock_engine_task_with_ready(
|
||||
handshake_address.clone(),
|
||||
engine_id.clone(),
|
||||
@@ -750,10 +750,10 @@ async fn test_dev_mode_app_with_ready(
|
||||
|
||||
let chat = ChatLlm::from_shared_backend(test_llm(client), Arc::new(FakeChatBackend::new()));
|
||||
let app = build_router_with_dev_mode(
|
||||
Arc::new(AppState::new(
|
||||
vec!["Qwen/Qwen1.5-0.5B-Chat".to_string()],
|
||||
chat,
|
||||
)),
|
||||
Arc::new(
|
||||
AppState::new(vec!["Qwen/Qwen1.5-0.5B-Chat".to_string()], chat)
|
||||
.with_data_parallel_size(data_parallel_size),
|
||||
),
|
||||
true,
|
||||
);
|
||||
(app, engine_task)
|
||||
@@ -6313,7 +6313,7 @@ async fn tokenize_allows_prompts_at_or_above_max_model_len() {
|
||||
max_model_len: 4,
|
||||
..default_ready_response()
|
||||
};
|
||||
let (mut app, _engine_task) = test_dev_mode_app_with_ready(ready).await;
|
||||
let (mut app, _engine_task) = test_dev_mode_app_with_ready(ready, 1).await;
|
||||
|
||||
let (completion_status, completion_json) = post_json(
|
||||
&mut app,
|
||||
@@ -6569,10 +6569,10 @@ async fn world_size_endpoint_is_dev_mode_only() {
|
||||
async fn world_size_includes_data_parallelism_by_default() {
|
||||
let ready = vllm_engine_core_client::protocol::handshake::EngineCoreReadyResponse {
|
||||
world_size: 2,
|
||||
data_parallel_size: 4,
|
||||
data_parallel_size: 1,
|
||||
..default_ready_response()
|
||||
};
|
||||
let (mut app, _engine_task) = test_dev_mode_app_with_ready(ready).await;
|
||||
let (mut app, _engine_task) = test_dev_mode_app_with_ready(ready, 4).await;
|
||||
|
||||
let response = app
|
||||
.call(
|
||||
@@ -6598,7 +6598,7 @@ async fn world_size_excludes_data_parallelism_when_include_dp_false() {
|
||||
data_parallel_size: 4,
|
||||
..default_ready_response()
|
||||
};
|
||||
let (mut app, _engine_task) = test_dev_mode_app_with_ready(ready).await;
|
||||
let (mut app, _engine_task) = test_dev_mode_app_with_ready(ready, 4).await;
|
||||
|
||||
let response = app
|
||||
.call(
|
||||
|
||||
@@ -47,7 +47,7 @@ pub async fn get_world_size(
|
||||
let ws = client.world_size();
|
||||
|
||||
let world_size = if params.include_dp {
|
||||
let dp = client.data_parallel_size();
|
||||
let dp = state.data_parallel_size() as u64;
|
||||
ws * dp
|
||||
} else {
|
||||
ws
|
||||
|
||||
@@ -40,6 +40,8 @@ pub struct AppState {
|
||||
pub cors: CorsConfig,
|
||||
/// Runtime server information returned by `/server_info`, when available.
|
||||
server_info: Option<ServerInfoSnapshot>,
|
||||
/// Deployment-wide data-parallel size retained by the frontend.
|
||||
data_parallel_size: usize,
|
||||
/// SHA-256 hashes of API keys accepted as bearer tokens for guarded routes.
|
||||
api_key_hashes: Vec<ApiKeyHash>,
|
||||
/// Number of in-flight inference requests currently owned by this frontend.
|
||||
@@ -69,12 +71,14 @@ impl AppState {
|
||||
!served_model_names.is_empty(),
|
||||
"served_model_names must not be empty"
|
||||
);
|
||||
let data_parallel_size = chat.engine_core_client().engine_count();
|
||||
Self {
|
||||
served_model_names,
|
||||
chat,
|
||||
api_server_options: ApiServerOptions::default(),
|
||||
cors: CorsConfig::default(),
|
||||
server_info: None,
|
||||
data_parallel_size,
|
||||
api_key_hashes: Vec::new(),
|
||||
server_load: AtomicU64::new(0),
|
||||
lora_manager: LoraManager::new(),
|
||||
@@ -114,6 +118,17 @@ impl AppState {
|
||||
self
|
||||
}
|
||||
|
||||
/// Set the deployment-wide data-parallel size reported by frontend APIs.
|
||||
pub(crate) fn with_data_parallel_size(mut self, size: usize) -> Self {
|
||||
self.data_parallel_size = size;
|
||||
self
|
||||
}
|
||||
|
||||
/// Return the deployment-wide data-parallel size.
|
||||
pub(crate) fn data_parallel_size(&self) -> usize {
|
||||
self.data_parallel_size
|
||||
}
|
||||
|
||||
/// Build a `/server_info` response payload.
|
||||
pub(crate) fn server_info_response(
|
||||
&self,
|
||||
|
||||
Reference in New Issue
Block a user