[Rust][Benchmark] Integrate vllm-bench to vllm-rs & vllm CLI (#48930)
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> Signed-off-by: Bugen Zhao <i@bugenzhao.com>
This commit is contained in:
co-authored by
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
823eaf667d
commit
c01618fdc8
@@ -29,6 +29,7 @@ tokio-util.workspace = true
|
||||
tracing.workspace = true
|
||||
tracing-subscriber.workspace = true
|
||||
uuid.workspace = true
|
||||
vllm-bench.workspace = true
|
||||
vllm-chat.workspace = true
|
||||
vllm-engine-core-client.workspace = true
|
||||
vllm-managed-engine.workspace = true
|
||||
|
||||
+11
-1
@@ -79,13 +79,23 @@ impl Cli {
|
||||
}
|
||||
|
||||
/// Supported top-level CLI commands.
|
||||
#[derive(Debug, Subcommand, PartialEq, Eq)]
|
||||
#[derive(Debug, Subcommand)]
|
||||
pub enum Command {
|
||||
/// Run the Rust OpenAI frontend as a Python-supervised worker.
|
||||
Frontend(FrontendArgs),
|
||||
/// Launch a managed Python headless engine, then run the Rust OpenAI
|
||||
/// frontend.
|
||||
Serve(ServeArgs),
|
||||
/// Run vLLM benchmarks.
|
||||
#[command(subcommand)]
|
||||
Bench(BenchCommand),
|
||||
}
|
||||
|
||||
/// Supported benchmark commands.
|
||||
#[derive(Debug, Subcommand)]
|
||||
pub enum BenchCommand {
|
||||
/// Benchmark online serving throughput.
|
||||
Serve(vllm_bench::BenchServeArgs),
|
||||
}
|
||||
|
||||
/// A JSON-encoded list of strings, matching Python's `json.loads` CLI type for
|
||||
|
||||
@@ -5,7 +5,27 @@ use expect_test::expect;
|
||||
use vllm_engine_core_client::TransportMode;
|
||||
use vllm_server::{Config, HttpListenerMode, ParserSelection, RendererSelection};
|
||||
|
||||
use super::{Cli, Command};
|
||||
use super::{BenchCommand, Cli, Command};
|
||||
|
||||
#[test]
|
||||
fn bench_serve_args_parse_without_managed_engine_repartition() {
|
||||
let cli = Cli::try_parse_from([
|
||||
"vllm-rs",
|
||||
"bench",
|
||||
"serve",
|
||||
"--backend",
|
||||
"openai-chat",
|
||||
"--request-rate",
|
||||
"inf",
|
||||
])
|
||||
.unwrap();
|
||||
|
||||
let Command::Bench(BenchCommand::Serve(args)) = cli.command else {
|
||||
panic!("expected bench serve args");
|
||||
};
|
||||
assert_eq!(args.backend, vllm_bench::BackendKind::OpenaiChat);
|
||||
assert!(args.request_rate.is_infinite());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn serve_args_forward_python_flags_with_separator() {
|
||||
|
||||
@@ -12,7 +12,7 @@ use tokio_util::sync::CancellationToken;
|
||||
use tracing::{info, warn};
|
||||
use vllm_managed_engine::ManagedEngineHandle;
|
||||
|
||||
use crate::cli::{Cli, Command};
|
||||
use crate::cli::{BenchCommand, Cli, Command};
|
||||
|
||||
#[global_allocator]
|
||||
static GLOBAL: mimalloc::MiMalloc = mimalloc::MiMalloc;
|
||||
@@ -100,6 +100,10 @@ fn main() -> Result<()> {
|
||||
async fn async_main(cli: Cli) -> Result<()> {
|
||||
match cli.command {
|
||||
Command::Frontend(args) => vllm_server::serve(args.into_config(), shutdown_signal()).await,
|
||||
Command::Bench(BenchCommand::Serve(bench_args)) => {
|
||||
vllm_bench::prepare_process();
|
||||
vllm_bench::run(bench_args).await
|
||||
}
|
||||
Command::Serve(args) => {
|
||||
let handshake_port = args.managed_engine.resolve_handshake_port()?;
|
||||
|
||||
|
||||
Reference in New Issue
Block a user