[Rust Frontend][gRPC] Add explicit data-parallel rank routing (#51178)
Signed-off-by: Connor Carpenter <connorc@nvidia.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
3c6b2e9c08
commit
fa722b9f01
@@ -457,6 +457,7 @@ impl SharedRuntimeArgs {
|
||||
coordinator_address: Option<String>,
|
||||
engine_start_index: u32,
|
||||
engine_count: usize,
|
||||
data_parallel_size: usize,
|
||||
) -> Config {
|
||||
let ready_timeout = self.ready_timeout();
|
||||
let shutdown_timeout = self.shutdown_timeout();
|
||||
@@ -474,6 +475,7 @@ impl SharedRuntimeArgs {
|
||||
engine_count,
|
||||
ready_timeout,
|
||||
},
|
||||
data_parallel_size,
|
||||
coordinator_mode: match coordinator_address {
|
||||
Some(address) => CoordinatorMode::External { address },
|
||||
None => CoordinatorMode::None,
|
||||
@@ -530,6 +532,7 @@ impl SharedRuntimeArgs {
|
||||
local_input_address,
|
||||
local_output_address,
|
||||
},
|
||||
data_parallel_size: engine_count,
|
||||
coordinator_mode: CoordinatorMode::MaybeInProc,
|
||||
model: self.model,
|
||||
served_model_name: self.served_model_name,
|
||||
@@ -661,6 +664,9 @@ pub struct FrontendArgs {
|
||||
/// Total number of data-parallel engines expected for this frontend.
|
||||
#[arg(long, default_value_t = 1)]
|
||||
pub engine_count: usize,
|
||||
/// Deployment-wide configured data-parallel size. Defaults to engine count.
|
||||
#[arg(long)]
|
||||
pub data_parallel_size: Option<usize>,
|
||||
|
||||
/// Shared frontend arguments as one JSON object.
|
||||
#[arg(long = "args-json", value_parser = parse_runtime_args_json, value_name = "JSON")]
|
||||
@@ -670,6 +676,7 @@ pub struct FrontendArgs {
|
||||
impl FrontendArgs {
|
||||
/// Convert the CLI arguments into the OpenAI server's runtime config.
|
||||
pub fn into_config(self) -> Config {
|
||||
let data_parallel_size = self.data_parallel_size.unwrap_or(self.engine_count);
|
||||
self.runtime.into_bootstrapped_config(
|
||||
self.listen_fd,
|
||||
self.input_address,
|
||||
@@ -677,6 +684,7 @@ impl FrontendArgs {
|
||||
self.coordinator_address,
|
||||
self.engine_start_index,
|
||||
self.engine_count,
|
||||
data_parallel_size,
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -785,6 +785,7 @@ fn frontend_args_accept_json() {
|
||||
),
|
||||
engine_start_index: 0,
|
||||
engine_count: 1,
|
||||
data_parallel_size: None,
|
||||
runtime: SharedRuntimeArgs {
|
||||
model: "Qwen/Qwen3-0.6B",
|
||||
engine_ready_timeout_secs: 600,
|
||||
@@ -846,6 +847,8 @@ fn frontend_args_json_applies_defaults() {
|
||||
"ipc:///tmp/input.sock",
|
||||
"--output-address",
|
||||
"ipc:///tmp/output.sock",
|
||||
"--engine-count",
|
||||
"4",
|
||||
"--args-json",
|
||||
r#"{"model_tag":"Qwen/Qwen3-0.6B"}"#,
|
||||
])
|
||||
@@ -861,6 +864,7 @@ fn frontend_args_json_applies_defaults() {
|
||||
assert_eq!(args.runtime.renderer, RendererSelection::Auto);
|
||||
assert_eq!(args.runtime.max_logprobs, None);
|
||||
assert_eq!(args.runtime.shutdown_timeout, 0);
|
||||
assert_eq!(args.into_config().data_parallel_size, 4);
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -1517,6 +1521,7 @@ fn serve_frontend_config_uses_dp_address_as_advertised_host() {
|
||||
"<ipc output>",
|
||||
),
|
||||
},
|
||||
data_parallel_size: 4,
|
||||
coordinator_mode: MaybeInProc,
|
||||
model: "Qwen/Qwen3-0.6B",
|
||||
served_model_name: [],
|
||||
@@ -1602,6 +1607,7 @@ fn serve_frontend_config_keeps_tcp_transport_for_non_local_only_topology() {
|
||||
local_input_address: None,
|
||||
local_output_address: None,
|
||||
},
|
||||
data_parallel_size: 4,
|
||||
coordinator_mode: MaybeInProc,
|
||||
model: "Qwen/Qwen3-0.6B",
|
||||
served_model_name: [],
|
||||
@@ -1685,6 +1691,8 @@ fn frontend_config_uses_external_coordinator_when_coordinator_address_is_present
|
||||
"3",
|
||||
"--engine-count",
|
||||
"1",
|
||||
"--data-parallel-size",
|
||||
"4",
|
||||
"--args-json",
|
||||
r#"{"model_tag":"Qwen/Qwen3-0.6B"}"#,
|
||||
])
|
||||
@@ -1704,6 +1712,7 @@ fn frontend_config_uses_external_coordinator_when_coordinator_address_is_present
|
||||
engine_count: 1,
|
||||
ready_timeout: 600s,
|
||||
},
|
||||
data_parallel_size: 4,
|
||||
coordinator_mode: External {
|
||||
address: "tcp://127.0.0.1:7000",
|
||||
},
|
||||
|
||||
Reference in New Issue
Block a user