[Rust Frontend][gRPC] Add explicit data-parallel rank routing (#51178)

Signed-off-by: Connor Carpenter <connorc@nvidia.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
Connor Carpenter
2026-08-10 15:05:24 -07:00
committed by GitHub
co-authored by mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent 3c6b2e9c08
commit fa722b9f01
19 changed files with 215 additions and 60 deletions
+8
View File
@@ -457,6 +457,7 @@ impl SharedRuntimeArgs {
coordinator_address: Option<String>,
engine_start_index: u32,
engine_count: usize,
data_parallel_size: usize,
) -> Config {
let ready_timeout = self.ready_timeout();
let shutdown_timeout = self.shutdown_timeout();
@@ -474,6 +475,7 @@ impl SharedRuntimeArgs {
engine_count,
ready_timeout,
},
data_parallel_size,
coordinator_mode: match coordinator_address {
Some(address) => CoordinatorMode::External { address },
None => CoordinatorMode::None,
@@ -530,6 +532,7 @@ impl SharedRuntimeArgs {
local_input_address,
local_output_address,
},
data_parallel_size: engine_count,
coordinator_mode: CoordinatorMode::MaybeInProc,
model: self.model,
served_model_name: self.served_model_name,
@@ -661,6 +664,9 @@ pub struct FrontendArgs {
/// Total number of data-parallel engines expected for this frontend.
#[arg(long, default_value_t = 1)]
pub engine_count: usize,
/// Deployment-wide configured data-parallel size. Defaults to engine count.
#[arg(long)]
pub data_parallel_size: Option<usize>,
/// Shared frontend arguments as one JSON object.
#[arg(long = "args-json", value_parser = parse_runtime_args_json, value_name = "JSON")]
@@ -670,6 +676,7 @@ pub struct FrontendArgs {
impl FrontendArgs {
/// Convert the CLI arguments into the OpenAI server's runtime config.
pub fn into_config(self) -> Config {
let data_parallel_size = self.data_parallel_size.unwrap_or(self.engine_count);
self.runtime.into_bootstrapped_config(
self.listen_fd,
self.input_address,
@@ -677,6 +684,7 @@ impl FrontendArgs {
self.coordinator_address,
self.engine_start_index,
self.engine_count,
data_parallel_size,
)
}
}
+9
View File
@@ -785,6 +785,7 @@ fn frontend_args_accept_json() {
),
engine_start_index: 0,
engine_count: 1,
data_parallel_size: None,
runtime: SharedRuntimeArgs {
model: "Qwen/Qwen3-0.6B",
engine_ready_timeout_secs: 600,
@@ -846,6 +847,8 @@ fn frontend_args_json_applies_defaults() {
"ipc:///tmp/input.sock",
"--output-address",
"ipc:///tmp/output.sock",
"--engine-count",
"4",
"--args-json",
r#"{"model_tag":"Qwen/Qwen3-0.6B"}"#,
])
@@ -861,6 +864,7 @@ fn frontend_args_json_applies_defaults() {
assert_eq!(args.runtime.renderer, RendererSelection::Auto);
assert_eq!(args.runtime.max_logprobs, None);
assert_eq!(args.runtime.shutdown_timeout, 0);
assert_eq!(args.into_config().data_parallel_size, 4);
}
#[test]
@@ -1517,6 +1521,7 @@ fn serve_frontend_config_uses_dp_address_as_advertised_host() {
"<ipc output>",
),
},
data_parallel_size: 4,
coordinator_mode: MaybeInProc,
model: "Qwen/Qwen3-0.6B",
served_model_name: [],
@@ -1602,6 +1607,7 @@ fn serve_frontend_config_keeps_tcp_transport_for_non_local_only_topology() {
local_input_address: None,
local_output_address: None,
},
data_parallel_size: 4,
coordinator_mode: MaybeInProc,
model: "Qwen/Qwen3-0.6B",
served_model_name: [],
@@ -1685,6 +1691,8 @@ fn frontend_config_uses_external_coordinator_when_coordinator_address_is_present
"3",
"--engine-count",
"1",
"--data-parallel-size",
"4",
"--args-json",
r#"{"model_tag":"Qwen/Qwen3-0.6B"}"#,
])
@@ -1704,6 +1712,7 @@ fn frontend_config_uses_external_coordinator_when_coordinator_address_is_present
engine_count: 1,
ready_timeout: 600s,
},
data_parallel_size: 4,
coordinator_mode: External {
address: "tcp://127.0.0.1:7000",
},