[Rust Frontend] Simplify data-parallel size ownership (#52575)
Signed-off-by: Bugen Zhao <i@bugenzhao.com> Co-authored-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
@@ -2417,6 +2417,7 @@ steps:
|
||||
- vllm/v1/engine/
|
||||
- vllm/v1/worker/
|
||||
- tests/utils.py
|
||||
- tests/v1/distributed/test_dense_dp_world_size.py
|
||||
- tests/v1/distributed/test_external_lb_dp.py
|
||||
- tests/v1/distributed/test_hybrid_lb_dp.py
|
||||
- tests/v1/distributed/test_internal_lb_dp.py
|
||||
@@ -2424,6 +2425,7 @@ steps:
|
||||
commands:
|
||||
- export VLLM_USE_RUST_FRONTEND=1
|
||||
- export VLLM_WORKER_MULTIPROC_METHOD=spawn
|
||||
- TP_SIZE=1 DP_SIZE=4 pytest -v -s v1/distributed/test_dense_dp_world_size.py
|
||||
- TP_SIZE=1 DP_SIZE=4 pytest -v -s v1/distributed/test_internal_lb_dp.py -k "not 4 and not server_info"
|
||||
- TP_SIZE=1 DP_SIZE=2 pytest -v -s v1/distributed/test_external_lb_dp.py -k "not 4 and not server_info"
|
||||
- TP_SIZE=1 DP_SIZE=4 pytest -v -s v1/distributed/test_hybrid_lb_dp.py -k "not 4 and not server_info"
|
||||
@@ -2882,6 +2884,7 @@ steps:
|
||||
- TP_SIZE=2 DP_SIZE=2 pytest -v -s v1/distributed/test_async_llm_dp.py
|
||||
- TP_SIZE=2 DP_SIZE=2 pytest -v -s v1/distributed/test_eagle_dp.py
|
||||
- TP_SIZE=2 DP_SIZE=2 pytest -v -s v1/distributed/test_external_lb_dp.py
|
||||
- TP_SIZE=1 DP_SIZE=4 pytest -v -s v1/distributed/test_dense_dp_world_size.py
|
||||
- TP_SIZE=1 DP_SIZE=4 pytest -v -s v1/distributed/test_internal_lb_dp.py
|
||||
- TP_SIZE=1 DP_SIZE=4 pytest -v -s v1/distributed/test_hybrid_lb_dp.py
|
||||
- pytest -v -s v1/engine/test_engine_core_client.py::test_kv_cache_events_dp
|
||||
|
||||
@@ -154,6 +154,7 @@ steps:
|
||||
- TP_SIZE=2 DP_SIZE=2 pytest -v -s v1/distributed/test_async_llm_dp.py
|
||||
- TP_SIZE=2 DP_SIZE=2 pytest -v -s v1/distributed/test_eagle_dp.py
|
||||
- TP_SIZE=2 DP_SIZE=2 pytest -v -s v1/distributed/test_external_lb_dp.py
|
||||
- TP_SIZE=1 DP_SIZE=4 pytest -v -s v1/distributed/test_dense_dp_world_size.py
|
||||
- TP_SIZE=1 DP_SIZE=4 pytest -v -s v1/distributed/test_internal_lb_dp.py
|
||||
- TP_SIZE=1 DP_SIZE=4 pytest -v -s v1/distributed/test_hybrid_lb_dp.py
|
||||
- pytest -v -s v1/engine/test_engine_core_client.py::test_kv_cache_events_dp
|
||||
|
||||
@@ -108,6 +108,7 @@ steps:
|
||||
- vllm/v1/engine/
|
||||
- vllm/v1/worker/
|
||||
- tests/utils.py
|
||||
- tests/v1/distributed/test_dense_dp_world_size.py
|
||||
- tests/v1/distributed/test_external_lb_dp.py
|
||||
- tests/v1/distributed/test_hybrid_lb_dp.py
|
||||
- tests/v1/distributed/test_internal_lb_dp.py
|
||||
@@ -115,6 +116,7 @@ steps:
|
||||
- export VLLM_USE_RUST_FRONTEND=1
|
||||
- export VLLM_WORKER_MULTIPROC_METHOD=spawn
|
||||
- export NCCL_CUMEM_HOST_ENABLE=0
|
||||
- TP_SIZE=1 DP_SIZE=4 pytest -v -s v1/distributed/test_dense_dp_world_size.py
|
||||
- TP_SIZE=1 DP_SIZE=4 pytest -v -s v1/distributed/test_internal_lb_dp.py -k "not 4 and not server_info"
|
||||
- TP_SIZE=1 DP_SIZE=2 pytest -v -s v1/distributed/test_external_lb_dp.py -k "not 4 and not server_info"
|
||||
- TP_SIZE=1 DP_SIZE=4 pytest -v -s v1/distributed/test_hybrid_lb_dp.py -k "not 4 and not server_info"
|
||||
|
||||
@@ -474,9 +474,9 @@ impl SharedRuntimeArgs {
|
||||
output_address,
|
||||
engine_start_index,
|
||||
engine_count,
|
||||
data_parallel_size,
|
||||
ready_timeout,
|
||||
},
|
||||
data_parallel_size,
|
||||
coordinator_mode: match coordinator_address {
|
||||
Some(address) => CoordinatorMode::External { address },
|
||||
None => CoordinatorMode::None,
|
||||
@@ -533,7 +533,6 @@ impl SharedRuntimeArgs {
|
||||
local_input_address,
|
||||
local_output_address,
|
||||
},
|
||||
data_parallel_size: engine_count,
|
||||
coordinator_mode: CoordinatorMode::MaybeInProc,
|
||||
model: self.model,
|
||||
served_model_name: self.served_model_name,
|
||||
|
||||
@@ -864,7 +864,7 @@ fn frontend_args_json_applies_defaults() {
|
||||
assert_eq!(args.runtime.renderer, RendererSelection::Auto);
|
||||
assert_eq!(args.runtime.max_logprobs, None);
|
||||
assert_eq!(args.runtime.shutdown_timeout, 0);
|
||||
assert_eq!(args.into_config().data_parallel_size, 4);
|
||||
assert_eq!(args.into_config().transport_mode.data_parallel_size(), 4);
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -1521,7 +1521,6 @@ fn serve_frontend_config_uses_dp_address_as_advertised_host() {
|
||||
"<ipc output>",
|
||||
),
|
||||
},
|
||||
data_parallel_size: 4,
|
||||
coordinator_mode: MaybeInProc,
|
||||
model: "Qwen/Qwen3-0.6B",
|
||||
served_model_name: [],
|
||||
@@ -1607,7 +1606,6 @@ fn serve_frontend_config_keeps_tcp_transport_for_non_local_only_topology() {
|
||||
local_input_address: None,
|
||||
local_output_address: None,
|
||||
},
|
||||
data_parallel_size: 4,
|
||||
coordinator_mode: MaybeInProc,
|
||||
model: "Qwen/Qwen3-0.6B",
|
||||
served_model_name: [],
|
||||
@@ -1710,9 +1708,9 @@ fn frontend_config_uses_external_coordinator_when_coordinator_address_is_present
|
||||
output_address: "ipc:///tmp/output.sock",
|
||||
engine_start_index: 3,
|
||||
engine_count: 1,
|
||||
data_parallel_size: 4,
|
||||
ready_timeout: 600s,
|
||||
},
|
||||
data_parallel_size: 4,
|
||||
coordinator_mode: External {
|
||||
address: "tcp://127.0.0.1:7000",
|
||||
},
|
||||
|
||||
@@ -15,7 +15,7 @@ use tracing::{debug, info, trace};
|
||||
|
||||
use crate::client::imp::{ClientInner, run_abort_loop, run_output_dispatcher_loop};
|
||||
use crate::coordinator::CoordinatorHandle;
|
||||
use crate::error::{Error, Result};
|
||||
use crate::error::{Error, Result, bail_invalid_client_config};
|
||||
use crate::protocol::dtype::ModelDtype;
|
||||
use crate::protocol::handshake::EngineCoreReadyResponse;
|
||||
use crate::protocol::lora::LoraRequest;
|
||||
@@ -68,11 +68,70 @@ pub enum TransportMode {
|
||||
engine_start_index: u32,
|
||||
/// Total number of engines expected to register on this transport.
|
||||
engine_count: usize,
|
||||
/// Deployment-wide data-parallel size. This may be larger than
|
||||
/// `engine_count` when a supervisor partitions ranks across frontends.
|
||||
data_parallel_size: usize,
|
||||
/// Maximum time to wait for all expected engines to register.
|
||||
ready_timeout: Duration,
|
||||
},
|
||||
}
|
||||
|
||||
impl TransportMode {
|
||||
/// Return the deployment-wide data-parallel size for this transport.
|
||||
pub fn data_parallel_size(&self) -> usize {
|
||||
match self {
|
||||
Self::HandshakeOwner { engine_count, .. } => *engine_count,
|
||||
Self::Bootstrapped {
|
||||
data_parallel_size, ..
|
||||
} => *data_parallel_size,
|
||||
}
|
||||
}
|
||||
|
||||
/// Validate the transport topology before opening any sockets.
|
||||
pub fn validate(&self) -> Result<()> {
|
||||
let data_parallel_size = self.data_parallel_size();
|
||||
if data_parallel_size == 0 {
|
||||
bail_invalid_client_config!("data parallel size must be at least 1");
|
||||
}
|
||||
if data_parallel_size > usize::from(u16::MAX) + 1 {
|
||||
bail_invalid_client_config!(
|
||||
"data parallel size ({data_parallel_size}) exceeds the two-byte engine identity limit"
|
||||
);
|
||||
}
|
||||
|
||||
match self {
|
||||
Self::HandshakeOwner { .. } => {}
|
||||
Self::Bootstrapped {
|
||||
engine_start_index,
|
||||
engine_count,
|
||||
..
|
||||
} => {
|
||||
if *engine_count == 0 {
|
||||
bail_invalid_client_config!("engine count must be at least 1");
|
||||
}
|
||||
let engine_start_index = usize::try_from(*engine_start_index).map_err(|_| {
|
||||
Error::InvalidClientConfig {
|
||||
message: "engine start index does not fit usize".to_string(),
|
||||
}
|
||||
})?;
|
||||
let engine_end_index =
|
||||
engine_start_index.checked_add(*engine_count).ok_or_else(|| {
|
||||
Error::InvalidClientConfig {
|
||||
message: "engine start index + engine count overflows".to_string(),
|
||||
}
|
||||
})?;
|
||||
if engine_end_index > data_parallel_size {
|
||||
bail_invalid_client_config!(
|
||||
"connected engine range [{engine_start_index}, {engine_end_index}) exceeds data parallel size ({data_parallel_size})"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
/// Which coordinator implementation should be active when one is present for a
|
||||
/// frontend client.
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
@@ -117,6 +176,11 @@ impl EngineCoreClientConfig {
|
||||
}
|
||||
}
|
||||
|
||||
/// Validate the client topology before opening any transport sockets.
|
||||
pub fn validate(&self) -> Result<()> {
|
||||
self.transport_mode.validate()
|
||||
}
|
||||
|
||||
/// Set the model name used by frontend-side metrics and diagnostics.
|
||||
pub fn with_model_name(mut self, model_name: impl Into<String>) -> Self {
|
||||
self.model_name = model_name.into();
|
||||
@@ -226,6 +290,7 @@ impl EngineCoreClient {
|
||||
/// handshake. In bootstrapped mode it binds the provided frontend
|
||||
/// sockets and waits for the expected engine registration frames.
|
||||
pub async fn connect(config: EngineCoreClientConfig) -> Result<Self> {
|
||||
config.validate()?;
|
||||
let connected = match &config.transport_mode {
|
||||
TransportMode::HandshakeOwner {
|
||||
handshake_address,
|
||||
@@ -261,6 +326,7 @@ impl EngineCoreClient {
|
||||
engine_start_index,
|
||||
engine_count,
|
||||
ready_timeout,
|
||||
..
|
||||
} => {
|
||||
if let Some(CoordinatorMode::InProc) = config.coordinator_mode {
|
||||
panic!("cannot use in-process coordinator with bootstrapped transport mode")
|
||||
@@ -378,6 +444,12 @@ impl EngineCoreClient {
|
||||
self.engines.len()
|
||||
}
|
||||
|
||||
/// Return the deployment-wide data-parallel size configured for this
|
||||
/// client.
|
||||
pub fn data_parallel_size(&self) -> usize {
|
||||
self.config.transport_mode.data_parallel_size()
|
||||
}
|
||||
|
||||
/// Return the engine-side indices connected to this client.
|
||||
pub fn engine_indices(&self) -> Vec<u32> {
|
||||
self.engines
|
||||
|
||||
@@ -49,6 +49,8 @@ pub enum Error {
|
||||
UnexpectedHandshakeIdentity { expected: Vec<u8>, actual: Vec<u8> },
|
||||
#[error("unexpected startup handshake message: {message}")]
|
||||
UnexpectedHandshakeMessage { message: String },
|
||||
#[error("invalid engine-core client configuration: {message}")]
|
||||
InvalidClientConfig { message: String },
|
||||
#[error("unexpected non-control output on coordinator path: {message}")]
|
||||
UnexpectedCoordinatorOutput { message: String },
|
||||
#[error("unexpected output on main dispatcher path: {message}")]
|
||||
|
||||
@@ -56,7 +56,7 @@ pub fn default_ready_response() -> EngineCoreReadyResponse {
|
||||
dtype: ModelDtype::Float32,
|
||||
vllm_version: "test-vllm-version".to_string(),
|
||||
world_size: 1,
|
||||
data_parallel_size: 1,
|
||||
effective_data_parallel_size: 1,
|
||||
tensor_parallel_size: 1,
|
||||
pipeline_parallel_size: 1,
|
||||
decode_context_parallel_size: 1,
|
||||
|
||||
@@ -63,8 +63,11 @@ pub struct EngineCoreReadyResponse {
|
||||
pub vllm_version: String,
|
||||
/// World size (TP * PP) from the parallel config.
|
||||
pub world_size: u64,
|
||||
/// Data parallelism size from the parallel config.
|
||||
pub data_parallel_size: u64,
|
||||
/// Data-parallel size from this EngineCore's effective parallel config.
|
||||
/// Dense independent-DP ranks are reconfigured to report `1`; the client
|
||||
/// transport owns the deployment-wide data-parallel size.
|
||||
#[serde(rename = "data_parallel_size")]
|
||||
pub effective_data_parallel_size: u64,
|
||||
// Required discovery metadata; EngineCore and client versions must match.
|
||||
/// Tensor-parallel size of this engine.
|
||||
pub tensor_parallel_size: u32,
|
||||
|
||||
@@ -303,6 +303,7 @@ fn bootstrapped_test_config(
|
||||
output_address,
|
||||
engine_start_index: 0,
|
||||
engine_count,
|
||||
data_parallel_size: engine_count,
|
||||
ready_timeout,
|
||||
},
|
||||
coordinator_mode,
|
||||
@@ -330,15 +331,47 @@ fn bootstrapped_test_config_with_start_index(
|
||||
);
|
||||
let TransportMode::Bootstrapped {
|
||||
engine_start_index: start,
|
||||
data_parallel_size,
|
||||
..
|
||||
} = &mut config.transport_mode
|
||||
else {
|
||||
unreachable!("bootstrapped_test_config returns bootstrapped transport")
|
||||
};
|
||||
*start = engine_start_index;
|
||||
*data_parallel_size = usize::try_from(engine_start_index)
|
||||
.expect("test start index fits usize")
|
||||
.checked_add(engine_count)
|
||||
.expect("test engine range fits usize");
|
||||
config
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn client_config_validates_bootstrapped_dp_range() {
|
||||
let mut config = bootstrapped_test_config_with_start_index(
|
||||
"ipc://unused-input".to_string(),
|
||||
"ipc://unused-output".to_string(),
|
||||
1,
|
||||
2,
|
||||
Duration::from_secs(1),
|
||||
0,
|
||||
None,
|
||||
);
|
||||
config.validate().expect("frontend may own a subset of global DP ranks");
|
||||
|
||||
let TransportMode::Bootstrapped {
|
||||
data_parallel_size, ..
|
||||
} = &mut config.transport_mode
|
||||
else {
|
||||
unreachable!("expected bootstrapped transport")
|
||||
};
|
||||
*data_parallel_size = 2;
|
||||
let error = config.validate().expect_err("engine range above DP size must fail");
|
||||
expect_test::expect![[
|
||||
"invalid engine-core client configuration: connected engine range [1, 3) exceeds data parallel size (2)"
|
||||
]]
|
||||
.assert_eq(&error.to_string());
|
||||
}
|
||||
|
||||
async fn recv_xpub_message(xpub: &mut XPubSocket) -> Vec<bytes::Bytes> {
|
||||
xpub.recv().await.unwrap().into_vec()
|
||||
}
|
||||
|
||||
@@ -58,7 +58,6 @@ async fn main() -> Result<()> {
|
||||
local_input_address: None,
|
||||
local_output_address: None,
|
||||
},
|
||||
data_parallel_size: args.engine_count,
|
||||
coordinator_mode: CoordinatorMode::MaybeInProc,
|
||||
model: args.model,
|
||||
served_model_name: vec![],
|
||||
|
||||
@@ -161,8 +161,6 @@ impl TlsConfig {
|
||||
pub struct Config {
|
||||
/// Frontend-to-engine transport setup.
|
||||
pub transport_mode: TransportMode,
|
||||
/// Deployment-wide data-parallel size retained by the frontend.
|
||||
pub data_parallel_size: usize,
|
||||
/// Requested frontend-side coordinator behavior.
|
||||
pub coordinator_mode: CoordinatorMode,
|
||||
/// Backend model identifier used for engine-core loading.
|
||||
@@ -239,46 +237,7 @@ impl Config {
|
||||
max_logprobs
|
||||
);
|
||||
}
|
||||
if self.data_parallel_size == 0 {
|
||||
bail!("data parallel size must be at least 1");
|
||||
}
|
||||
if self.data_parallel_size > usize::from(u16::MAX) + 1 {
|
||||
bail!(
|
||||
"data parallel size ({}) exceeds the two-byte engine identity limit",
|
||||
self.data_parallel_size
|
||||
);
|
||||
}
|
||||
match &self.transport_mode {
|
||||
TransportMode::HandshakeOwner { engine_count, .. } => {
|
||||
if *engine_count != self.data_parallel_size {
|
||||
bail!(
|
||||
"managed frontend engine count ({engine_count}) must equal data parallel size ({})",
|
||||
self.data_parallel_size
|
||||
);
|
||||
}
|
||||
}
|
||||
TransportMode::Bootstrapped {
|
||||
engine_start_index,
|
||||
engine_count,
|
||||
..
|
||||
} => {
|
||||
if *engine_count == 0 {
|
||||
bail!("engine count must be at least 1");
|
||||
}
|
||||
let engine_start_index = usize::try_from(*engine_start_index)
|
||||
.map_err(|_| anyhow::anyhow!("engine start index does not fit usize"))?;
|
||||
let engine_end_index =
|
||||
engine_start_index.checked_add(*engine_count).ok_or_else(|| {
|
||||
anyhow::anyhow!("engine start index + engine count overflows")
|
||||
})?;
|
||||
if engine_end_index > self.data_parallel_size {
|
||||
bail!(
|
||||
"connected engine range [{engine_start_index}, {engine_end_index}) exceeds data parallel size {}",
|
||||
self.data_parallel_size
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
self.transport_mode.validate()?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -43,7 +43,7 @@ impl ControlServiceImpl {
|
||||
pb::ParallelismInfo {
|
||||
tensor_parallel_size: ready.tensor_parallel_size,
|
||||
pipeline_parallel_size: ready.pipeline_parallel_size,
|
||||
data_parallel_size: self.state.data_parallel_size().min(u32::MAX as usize) as u32,
|
||||
data_parallel_size: self.client().data_parallel_size().min(u32::MAX as usize) as u32,
|
||||
data_parallel_rank: ready.data_parallel_rank,
|
||||
decode_context_parallel_size: ready.decode_context_parallel_size,
|
||||
}
|
||||
|
||||
@@ -1625,7 +1625,7 @@ async fn control_aggregates_multi_engine_capacity() {
|
||||
let mut ready_0 = default_ready_response();
|
||||
ready_0.max_model_len = 8_192;
|
||||
ready_0.num_gpu_blocks = 10;
|
||||
ready_0.data_parallel_size = 2;
|
||||
ready_0.effective_data_parallel_size = 2;
|
||||
ready_0.weight_transfer_backend = Some("nccl".to_string());
|
||||
ready_0.enable_sleep_mode = true;
|
||||
ready_0.supports_draft_weight_updates = true;
|
||||
@@ -1633,7 +1633,7 @@ async fn control_aggregates_multi_engine_capacity() {
|
||||
let mut ready_1 = default_ready_response();
|
||||
ready_1.max_model_len = 4_096;
|
||||
ready_1.num_gpu_blocks = 20;
|
||||
ready_1.data_parallel_size = 2;
|
||||
ready_1.effective_data_parallel_size = 2;
|
||||
ready_1.data_parallel_rank = 1;
|
||||
|
||||
let engine_tasks = [ready_0, ready_1].map(|ready| {
|
||||
@@ -1667,7 +1667,7 @@ async fn control_aggregates_multi_engine_capacity() {
|
||||
Llm::new(client),
|
||||
Arc::new(FakeTextBackend) as Arc<dyn ChatTextBackend>,
|
||||
);
|
||||
let state = AppState::new(vec!["test-model".to_string()], chat).with_data_parallel_size(4);
|
||||
let state = AppState::new(vec!["test-model".to_string()], chat);
|
||||
let service = ControlServiceImpl::new(Arc::new(state));
|
||||
|
||||
let server = pb::control_server::Control::get_server_info(
|
||||
@@ -1684,7 +1684,7 @@ async fn control_aggregates_multi_engine_capacity() {
|
||||
assert!(rl.weight_transfer_backend.is_empty());
|
||||
assert!(!rl.sleep_mode_enabled);
|
||||
assert!(!rl.draft_weight_updates_enabled);
|
||||
assert_eq!(server.parallelism.unwrap().data_parallel_size, 4);
|
||||
assert_eq!(server.parallelism.unwrap().data_parallel_size, 2);
|
||||
|
||||
drop(engine_tasks);
|
||||
}
|
||||
|
||||
@@ -141,7 +141,6 @@ async fn build_state(config: &Config) -> Result<Arc<AppState>> {
|
||||
.with_model_path(config.model.clone())
|
||||
.with_api_server_options(config.api_server_options)
|
||||
.with_server_info(ServerInfoSnapshot::from_config(config))
|
||||
.with_data_parallel_size(config.data_parallel_size)
|
||||
.with_api_keys(config.api_keys.clone())
|
||||
.with_cors(config.cors.clone())
|
||||
.with_profiler(config.profiler.clone()),
|
||||
|
||||
@@ -704,14 +704,13 @@ async fn test_app_with_dev_mode(dev_mode_enabled: bool) -> axum::Router {
|
||||
/// stays alive for the duration of the test.
|
||||
async fn test_dev_mode_app_with_ready(
|
||||
ready_response: vllm_engine_core_client::protocol::handshake::EngineCoreReadyResponse,
|
||||
data_parallel_size: usize,
|
||||
) -> (axum::Router, MockEngineTask) {
|
||||
let ipc = IpcNamespace::new().expect("create ipc namespace");
|
||||
let handshake_address = ipc.handshake_endpoint();
|
||||
let engine_id = b"engine-world-size".to_vec();
|
||||
let engine_task = MockEngineTask::new(spawn_mock_engine_task_with_ready(
|
||||
handshake_address.clone(),
|
||||
engine_id.clone(),
|
||||
engine_id,
|
||||
ready_response,
|
||||
|_dealer, _push| boxed_test_future(async {}),
|
||||
));
|
||||
@@ -729,10 +728,10 @@ async fn test_dev_mode_app_with_ready(
|
||||
|
||||
let chat = ChatLlm::from_shared_backend(test_llm(client), Arc::new(FakeChatBackend::new()));
|
||||
let app = build_router_with_dev_mode(
|
||||
Arc::new(
|
||||
AppState::new(vec!["Qwen/Qwen1.5-0.5B-Chat".to_string()], chat)
|
||||
.with_data_parallel_size(data_parallel_size),
|
||||
),
|
||||
Arc::new(AppState::new(
|
||||
vec!["Qwen/Qwen1.5-0.5B-Chat".to_string()],
|
||||
chat,
|
||||
)),
|
||||
true,
|
||||
);
|
||||
(app, engine_task)
|
||||
@@ -6292,7 +6291,7 @@ async fn tokenize_allows_prompts_at_or_above_max_model_len() {
|
||||
max_model_len: 4,
|
||||
..default_ready_response()
|
||||
};
|
||||
let (mut app, _engine_task) = test_dev_mode_app_with_ready(ready, 1).await;
|
||||
let (mut app, _engine_task) = test_dev_mode_app_with_ready(ready).await;
|
||||
|
||||
let (completion_status, completion_json) = post_json(
|
||||
&mut app,
|
||||
@@ -6543,58 +6542,6 @@ async fn world_size_endpoint_is_dev_mode_only() {
|
||||
assert_eq!(response.status(), StatusCode::NOT_FOUND);
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
#[serial]
|
||||
async fn world_size_includes_data_parallelism_by_default() {
|
||||
let ready = vllm_engine_core_client::protocol::handshake::EngineCoreReadyResponse {
|
||||
world_size: 2,
|
||||
data_parallel_size: 1,
|
||||
..default_ready_response()
|
||||
};
|
||||
let (mut app, _engine_task) = test_dev_mode_app_with_ready(ready, 4).await;
|
||||
|
||||
let response = app
|
||||
.call(
|
||||
Request::builder()
|
||||
.uri("/get_world_size")
|
||||
.body(Body::empty())
|
||||
.expect("build request"),
|
||||
)
|
||||
.await
|
||||
.expect("call app");
|
||||
|
||||
assert_eq!(response.status(), StatusCode::OK);
|
||||
let body = to_bytes(response.into_body(), usize::MAX).await.expect("read body");
|
||||
let json: serde_json::Value = serde_json::from_slice(&body).expect("decode json");
|
||||
assert_eq!(json, json!({"world_size": 8}));
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
#[serial]
|
||||
async fn world_size_excludes_data_parallelism_when_include_dp_false() {
|
||||
let ready = vllm_engine_core_client::protocol::handshake::EngineCoreReadyResponse {
|
||||
world_size: 2,
|
||||
data_parallel_size: 4,
|
||||
..default_ready_response()
|
||||
};
|
||||
let (mut app, _engine_task) = test_dev_mode_app_with_ready(ready, 4).await;
|
||||
|
||||
let response = app
|
||||
.call(
|
||||
Request::builder()
|
||||
.uri("/get_world_size?include_dp=false")
|
||||
.body(Body::empty())
|
||||
.expect("build request"),
|
||||
)
|
||||
.await
|
||||
.expect("call app");
|
||||
|
||||
assert_eq!(response.status(), StatusCode::OK);
|
||||
let body = to_bytes(response.into_body(), usize::MAX).await.expect("read body");
|
||||
let json: serde_json::Value = serde_json::from_slice(&body).expect("decode json");
|
||||
assert_eq!(json, json!({"world_size": 2}));
|
||||
}
|
||||
|
||||
// ========================= Profiler route tests =========================
|
||||
|
||||
async fn test_profiling_app_with_engine_script<F>(script: F) -> (axum::Router, MockEngineTask)
|
||||
|
||||
@@ -47,7 +47,7 @@ pub async fn get_world_size(
|
||||
let ws = client.world_size();
|
||||
|
||||
let world_size = if params.include_dp {
|
||||
let dp = state.data_parallel_size() as u64;
|
||||
let dp = client.data_parallel_size() as u64;
|
||||
ws * dp
|
||||
} else {
|
||||
ws
|
||||
|
||||
@@ -40,8 +40,6 @@ pub struct AppState {
|
||||
pub cors: CorsConfig,
|
||||
/// Runtime server information returned by `/server_info`, when available.
|
||||
server_info: Option<ServerInfoSnapshot>,
|
||||
/// Deployment-wide data-parallel size retained by the frontend.
|
||||
data_parallel_size: usize,
|
||||
/// SHA-256 hashes of API keys accepted as bearer tokens for guarded routes.
|
||||
api_key_hashes: Vec<ApiKeyHash>,
|
||||
/// Number of in-flight inference requests currently owned by this frontend.
|
||||
@@ -71,14 +69,12 @@ impl AppState {
|
||||
!served_model_names.is_empty(),
|
||||
"served_model_names must not be empty"
|
||||
);
|
||||
let data_parallel_size = chat.engine_core_client().engine_count();
|
||||
Self {
|
||||
served_model_names,
|
||||
chat,
|
||||
api_server_options: ApiServerOptions::default(),
|
||||
cors: CorsConfig::default(),
|
||||
server_info: None,
|
||||
data_parallel_size,
|
||||
api_key_hashes: Vec::new(),
|
||||
server_load: AtomicU64::new(0),
|
||||
lora_manager: LoraManager::new(),
|
||||
@@ -118,17 +114,6 @@ impl AppState {
|
||||
self
|
||||
}
|
||||
|
||||
/// Set the deployment-wide data-parallel size reported by frontend APIs.
|
||||
pub(crate) fn with_data_parallel_size(mut self, size: usize) -> Self {
|
||||
self.data_parallel_size = size;
|
||||
self
|
||||
}
|
||||
|
||||
/// Return the deployment-wide data-parallel size.
|
||||
pub(crate) fn data_parallel_size(&self) -> usize {
|
||||
self.data_parallel_size
|
||||
}
|
||||
|
||||
/// Build a `/server_info` response payload.
|
||||
pub(crate) fn server_info_response(
|
||||
&self,
|
||||
|
||||
@@ -0,0 +1,52 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import os
|
||||
|
||||
import requests
|
||||
|
||||
from tests.utils import RemoteOpenAIServer
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
MODEL_NAME = "Qwen/Qwen3-0.6B"
|
||||
DP_SIZE = int(os.getenv("DP_SIZE", "2"))
|
||||
TP_SIZE = int(os.getenv("TP_SIZE", "1"))
|
||||
|
||||
|
||||
def test_dense_dp_world_size():
|
||||
server_args = [
|
||||
"--dtype",
|
||||
"bfloat16",
|
||||
"--max-model-len",
|
||||
"2048",
|
||||
"--max-num-seqs",
|
||||
"128",
|
||||
"--enforce-eager",
|
||||
"--data-parallel-size",
|
||||
str(DP_SIZE),
|
||||
"--data-parallel-size-local",
|
||||
str(DP_SIZE),
|
||||
"--tensor-parallel-size",
|
||||
str(TP_SIZE),
|
||||
]
|
||||
env_dict = {
|
||||
"VLLM_SERVER_DEV_MODE": "1",
|
||||
current_platform.device_control_env_var: ",".join(
|
||||
str(current_platform.device_id_to_physical_device_id(i))
|
||||
for i in range(DP_SIZE * TP_SIZE)
|
||||
),
|
||||
}
|
||||
|
||||
with RemoteOpenAIServer(
|
||||
MODEL_NAME,
|
||||
server_args,
|
||||
env_dict=env_dict,
|
||||
) as server:
|
||||
response = requests.get(server.url_for("get_world_size"))
|
||||
response.raise_for_status()
|
||||
assert response.json() == {"world_size": TP_SIZE * DP_SIZE}
|
||||
|
||||
response = requests.get(
|
||||
server.url_for("get_world_size"), params={"include_dp": "false"}
|
||||
)
|
||||
response.raise_for_status()
|
||||
assert response.json() == {"world_size": TP_SIZE}
|
||||
Reference in New Issue
Block a user