[Rust Frontend] Move code from vllm-frontend-rs (#43283)
Signed-off-by: Bugen Zhao <i@bugenzhao.com> Signed-off-by: Nick Hill <nickhill123@gmail.com> Signed-off-by: Eric Curtin <eric.curtin@docker.com> Signed-off-by: Dev-X25874 <283057883+Dev-X25874@users.noreply.github.com> Signed-off-by: Will.hou <1205157517@qq.com> Signed-off-by: Will.hou <willamhou@ceresman.com> Co-authored-by: Nick Hill <nickhill123@gmail.com> Co-authored-by: Eric Curtin <eric.curtin@docker.com> Co-authored-by: Dev-X25874 <283057883+Dev-X25874@users.noreply.github.com> Co-authored-by: Will.hou <1205157517@qq.com> Co-authored-by: Will.hou <willamhou@ceresman.com> Please see https://github.com/Inferact/vllm-frontend-rs for full original commit history.
This commit is contained in:
co-authored by
Nick Hill
Eric Curtin
Dev-X25874
Will.hou
Will.hou
parent
39d5fa96a7
commit
39910f2b25
@@ -0,0 +1,53 @@
|
||||
# Engine-Core Smoke Tests
|
||||
|
||||
Start headless `vllm`:
|
||||
|
||||
```bash
|
||||
source ../vllm/.venv/bin/activate
|
||||
HF_HUB_OFFLINE=1 \
|
||||
VLLM_LOGGING_LEVEL=DEBUG \
|
||||
VLLM_CPU_KVCACHE_SPACE=2 \
|
||||
VLLM_HOST_IP=127.0.0.1 \
|
||||
VLLM_LOOPBACK_IP=127.0.0.1 \
|
||||
python3 -m vllm.entrypoints.cli.main serve Qwen/Qwen3-0.6B \
|
||||
--headless \
|
||||
--enable-sleep-mode \
|
||||
--data-parallel-address 127.0.0.1 \
|
||||
--data-parallel-rpc-port 62100 \
|
||||
--data-parallel-size-local 1 \
|
||||
--max-model-len 512 \
|
||||
--dtype float16
|
||||
```
|
||||
|
||||
Run the Rust smoke test through the `vllm-engine-core-client` utility interface:
|
||||
|
||||
```bash
|
||||
cargo run -p vllm-engine-core-client --example external_engine_utility_call -- \
|
||||
--handshake-address tcp://127.0.0.1:62100 \
|
||||
--host 127.0.0.1
|
||||
```
|
||||
|
||||
If your current engine setup does not support sleep mode, skip the `sleep` / `wake_up` part of the
|
||||
smoke:
|
||||
|
||||
```bash
|
||||
cargo run -p vllm-engine-core-client --example external_engine_utility_call -- \
|
||||
--handshake-address tcp://127.0.0.1:62100 \
|
||||
--host 127.0.0.1 \
|
||||
--skip-sleep-wake
|
||||
```
|
||||
|
||||
Run the Rust smoke test for sample logprobs decoding through the raw engine-core request path:
|
||||
|
||||
```bash
|
||||
cargo run -p vllm-engine-core-client --example external_engine_logprobs -- \
|
||||
--handshake-address tcp://127.0.0.1:62100 \
|
||||
--host 127.0.0.1
|
||||
```
|
||||
|
||||
This smoke requests a small generated-token `logprobs` payload plus prompt logprobs over a much
|
||||
longer prompt, so it exercises both the inline and aux-frame decode paths against a real engine.
|
||||
The Rust client decodes those payloads into semantic per-position records rather than exposing the
|
||||
raw ndarray/tensor wire shape.
|
||||
|
||||
IMPORTANT: You must restart `vllm` each time you run the smoke test, as the vLLM engine cannot manage frontend closures and subsequent reconnects. In other words, do not reuse existing `vllm` instances, if any.
|
||||
@@ -0,0 +1,190 @@
|
||||
use std::time::Duration;
|
||||
|
||||
use anyhow::{Context, Result, bail};
|
||||
use clap::Parser;
|
||||
use futures::StreamExt as _;
|
||||
use tokio::time::timeout;
|
||||
use tracing_subscriber::EnvFilter;
|
||||
use vllm_engine_core_client::protocol::{
|
||||
EngineCoreFinishReason, EngineCoreRequest, EngineCoreSamplingParams,
|
||||
};
|
||||
use vllm_engine_core_client::{
|
||||
EngineCoreClient, EngineCoreClientConfig, EngineCoreStreamOutput, TransportMode,
|
||||
};
|
||||
|
||||
const BASE_PROMPT_TOKEN_IDS: &[u32] = &[20841, 448, 6896, 25, 23811];
|
||||
|
||||
#[derive(Debug, Parser)]
|
||||
#[command(about = "Smoke-test engine-core sample logprobs against an external vLLM engine.")]
|
||||
struct Args {
|
||||
#[arg(long)]
|
||||
handshake_address: String,
|
||||
#[arg(long, default_value_t = 1)]
|
||||
engine_count: usize,
|
||||
#[arg(long, default_value = "Qwen/Qwen3-0.6B")]
|
||||
model: String,
|
||||
#[arg(long, default_value = "127.0.0.1")]
|
||||
host: String,
|
||||
#[arg(long, default_value_t = 0)]
|
||||
client_index: u32,
|
||||
#[arg(long, default_value_t = 30)]
|
||||
ready_timeout_secs: u64,
|
||||
#[arg(long, default_value_t = 120)]
|
||||
output_timeout_secs: u64,
|
||||
#[arg(long, default_value_t = 1)]
|
||||
max_tokens: u32,
|
||||
#[arg(long, default_value_t = 2)]
|
||||
logprobs: i32,
|
||||
#[arg(long, default_value_t = 1)]
|
||||
prompt_logprobs: i32,
|
||||
#[arg(long, default_value_t = 96)]
|
||||
prompt_repeats: usize,
|
||||
}
|
||||
|
||||
fn init_tracing() {
|
||||
let filter = EnvFilter::try_from_default_env()
|
||||
.unwrap_or_else(|_| EnvFilter::new("vllm_engine_core_client=debug"));
|
||||
let _ = tracing_subscriber::fmt().with_env_filter(filter).try_init();
|
||||
}
|
||||
|
||||
fn unique_request_id() -> String {
|
||||
let nanos = std::time::SystemTime::now()
|
||||
.duration_since(std::time::UNIX_EPOCH)
|
||||
.expect("system clock should be after unix epoch")
|
||||
.as_nanos();
|
||||
format!("rust-engine-core-logprobs-{nanos}")
|
||||
}
|
||||
|
||||
fn build_prompt_token_ids(prompt_repeats: usize) -> Vec<u32> {
|
||||
let repeats = prompt_repeats.max(1);
|
||||
BASE_PROMPT_TOKEN_IDS.repeat(repeats)
|
||||
}
|
||||
|
||||
fn build_request(
|
||||
request_id: String,
|
||||
prompt_token_ids: Vec<u32>,
|
||||
max_tokens: u32,
|
||||
logprobs: i32,
|
||||
prompt_logprobs: i32,
|
||||
client_index: u32,
|
||||
) -> EngineCoreRequest {
|
||||
EngineCoreRequest {
|
||||
request_id,
|
||||
prompt_token_ids: Some(prompt_token_ids),
|
||||
sampling_params: Some(EngineCoreSamplingParams {
|
||||
max_tokens,
|
||||
logprobs: Some(logprobs),
|
||||
prompt_logprobs: Some(prompt_logprobs),
|
||||
..EngineCoreSamplingParams::for_test()
|
||||
}),
|
||||
arrival_time: 0.0,
|
||||
client_index,
|
||||
..EngineCoreRequest::default()
|
||||
}
|
||||
}
|
||||
|
||||
async fn wait_for_final_output(
|
||||
mut stream: vllm_engine_core_client::EngineCoreOutputStream,
|
||||
) -> Result<EngineCoreStreamOutput> {
|
||||
while let Some(output) = stream.next().await {
|
||||
let output = output.context("failed to receive engine-core output")?;
|
||||
if output.finished() {
|
||||
return Ok(output);
|
||||
}
|
||||
}
|
||||
bail!("request stream ended without a final output")
|
||||
}
|
||||
|
||||
#[tokio::main(flavor = "multi_thread")]
|
||||
async fn main() -> Result<()> {
|
||||
init_tracing();
|
||||
let args = Args::parse();
|
||||
let ready_timeout = Duration::from_secs(args.ready_timeout_secs);
|
||||
let output_timeout = Duration::from_secs(args.output_timeout_secs);
|
||||
let request_id = unique_request_id();
|
||||
let prompt_token_ids = build_prompt_token_ids(args.prompt_repeats);
|
||||
let client = EngineCoreClient::connect(EngineCoreClientConfig {
|
||||
transport_mode: TransportMode::HandshakeOwner {
|
||||
handshake_address: args.handshake_address.clone(),
|
||||
advertised_host: args.host.clone(),
|
||||
engine_count: args.engine_count,
|
||||
ready_timeout,
|
||||
local_input_address: None,
|
||||
local_output_address: None,
|
||||
},
|
||||
coordinator_mode: None,
|
||||
model_name: args.model.clone(),
|
||||
client_index: args.client_index,
|
||||
})
|
||||
.await
|
||||
.context("failed to connect to external vLLM engine")?;
|
||||
|
||||
println!("model={}", args.model);
|
||||
println!("handshake_address={}", args.handshake_address);
|
||||
println!("engine_count={}", args.engine_count);
|
||||
println!("input_address={}", client.input_address());
|
||||
println!("output_address={}", client.output_address());
|
||||
println!("engine_identities={:x?}", client.engine_identities());
|
||||
|
||||
let request = build_request(
|
||||
request_id.clone(),
|
||||
prompt_token_ids.clone(),
|
||||
args.max_tokens,
|
||||
args.logprobs,
|
||||
args.prompt_logprobs,
|
||||
args.client_index,
|
||||
);
|
||||
println!("request_id={request_id}");
|
||||
println!("prompt_len={}", prompt_token_ids.len());
|
||||
println!("base_prompt_len={}", BASE_PROMPT_TOKEN_IDS.len());
|
||||
println!("prompt_repeats={}", args.prompt_repeats);
|
||||
println!("requested_logprobs={}", args.logprobs);
|
||||
println!("requested_prompt_logprobs={}", args.prompt_logprobs);
|
||||
|
||||
let stream = client.call(request).await.context("failed to submit engine-core request")?;
|
||||
let output = timeout(output_timeout, wait_for_final_output(stream))
|
||||
.await
|
||||
.context("timed out waiting for final output")??;
|
||||
|
||||
let finish_reason = output.finish_reason;
|
||||
let token_ids = output.new_token_ids.clone();
|
||||
let logprobs = output
|
||||
.new_logprobs
|
||||
.as_ref()
|
||||
.and_then(|value| value.as_direct())
|
||||
.context("engine output did not include decoded sample logprobs")?;
|
||||
let prompt_logprobs = output
|
||||
.new_prompt_logprobs_tensors
|
||||
.as_ref()
|
||||
.and_then(|value| value.as_direct())
|
||||
.context("engine output did not include decoded prompt logprobs")?;
|
||||
|
||||
println!("token_ids={token_ids:?}");
|
||||
println!("finish_reason={finish_reason:?}");
|
||||
println!("new_logprobs={logprobs:#?}");
|
||||
println!("new_prompt_logprobs_tensors={prompt_logprobs:#?}");
|
||||
|
||||
client.shutdown().await.context("failed to shut down engine-core client")?;
|
||||
|
||||
if finish_reason != Some(EngineCoreFinishReason::Length) {
|
||||
bail!("unexpected finish_reason: expected Length, got {finish_reason:?}");
|
||||
}
|
||||
if token_ids.is_empty() {
|
||||
bail!("engine returned no generated token ids");
|
||||
}
|
||||
if logprobs.is_empty() {
|
||||
bail!("decoded logprobs payload is unexpectedly empty");
|
||||
}
|
||||
if prompt_logprobs.is_empty() {
|
||||
bail!("decoded prompt logprobs payload is unexpectedly empty");
|
||||
}
|
||||
if prompt_logprobs.len() + 1 < prompt_token_ids.len() {
|
||||
bail!(
|
||||
"prompt logprobs rows look too short: prompt_len={}, rows={}",
|
||||
prompt_token_ids.len(),
|
||||
prompt_logprobs.len()
|
||||
);
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
@@ -0,0 +1,143 @@
|
||||
use std::time::Duration;
|
||||
|
||||
use anyhow::{Context, Result, bail};
|
||||
use clap::Parser;
|
||||
use tracing_subscriber::EnvFilter;
|
||||
use vllm_engine_core_client::{EngineCoreClient, EngineCoreClientConfig, TransportMode};
|
||||
|
||||
#[derive(Debug, Parser)]
|
||||
#[command(about = "Smoke-test EngineCoreClient utility calls against an external vLLM engine.")]
|
||||
struct Args {
|
||||
#[arg(long)]
|
||||
handshake_address: String,
|
||||
#[arg(long, default_value_t = 1)]
|
||||
engine_count: usize,
|
||||
#[arg(long, default_value = "Qwen/Qwen3-0.6B")]
|
||||
model: String,
|
||||
#[arg(long, default_value = "127.0.0.1")]
|
||||
host: String,
|
||||
#[arg(long, default_value_t = 0)]
|
||||
client_index: u32,
|
||||
#[arg(long, default_value_t = 30)]
|
||||
ready_timeout_secs: u64,
|
||||
#[arg(
|
||||
long,
|
||||
default_value_t = false,
|
||||
help = "Expected initial result of is_sleeping() before running the smoke steps."
|
||||
)]
|
||||
expected_is_sleeping: bool,
|
||||
#[arg(long, default_value_t = false)]
|
||||
reset_running_requests: bool,
|
||||
#[arg(long, default_value_t = false)]
|
||||
reset_external: bool,
|
||||
#[arg(long, default_value_t = 1)]
|
||||
sleep_level: u32,
|
||||
#[arg(long, default_value = "abort")]
|
||||
sleep_mode: String,
|
||||
#[arg(
|
||||
long,
|
||||
default_value_t = false,
|
||||
help = "Skip sleep/wake_up calls when the engine was not started with sleep-mode support."
|
||||
)]
|
||||
skip_sleep_wake: bool,
|
||||
}
|
||||
|
||||
fn init_tracing() {
|
||||
let filter = EnvFilter::try_from_default_env()
|
||||
.unwrap_or_else(|_| EnvFilter::new("vllm_engine_core_client=debug"));
|
||||
let _ = tracing_subscriber::fmt().with_env_filter(filter).try_init();
|
||||
}
|
||||
|
||||
#[tokio::main(flavor = "multi_thread")]
|
||||
async fn main() -> Result<()> {
|
||||
init_tracing();
|
||||
let args = Args::parse();
|
||||
let client = EngineCoreClient::connect(EngineCoreClientConfig {
|
||||
transport_mode: TransportMode::HandshakeOwner {
|
||||
handshake_address: args.handshake_address.clone(),
|
||||
advertised_host: args.host.clone(),
|
||||
engine_count: args.engine_count,
|
||||
ready_timeout: Duration::from_secs(args.ready_timeout_secs),
|
||||
local_input_address: None,
|
||||
local_output_address: None,
|
||||
},
|
||||
coordinator_mode: None,
|
||||
model_name: args.model.clone(),
|
||||
client_index: args.client_index,
|
||||
})
|
||||
.await
|
||||
.context("failed to connect to external vLLM engine")?;
|
||||
|
||||
println!("model={}", args.model);
|
||||
println!("handshake_address={}", args.handshake_address);
|
||||
println!("engine_count={}", args.engine_count);
|
||||
println!("input_address={}", client.input_address());
|
||||
println!("output_address={}", client.output_address());
|
||||
println!("engine_identities={:x?}", client.engine_identities());
|
||||
|
||||
let initial_is_sleeping =
|
||||
client.is_sleeping().await.context("failed to call is_sleeping utility")?;
|
||||
|
||||
println!("is_sleeping(initial)={initial_is_sleeping}");
|
||||
|
||||
if initial_is_sleeping != args.expected_is_sleeping {
|
||||
bail!(
|
||||
"unexpected initial is_sleeping state: expected {}, got {}",
|
||||
args.expected_is_sleeping,
|
||||
initial_is_sleeping
|
||||
);
|
||||
}
|
||||
|
||||
let reset_prefix_cache = client
|
||||
.reset_prefix_cache(args.reset_running_requests, args.reset_external)
|
||||
.await
|
||||
.context("failed to call reset_prefix_cache utility")?;
|
||||
println!("reset_prefix_cache={reset_prefix_cache}");
|
||||
|
||||
client.reset_mm_cache().await.context("failed to call reset_mm_cache utility")?;
|
||||
println!("reset_mm_cache=ok");
|
||||
|
||||
client
|
||||
.reset_encoder_cache()
|
||||
.await
|
||||
.context("failed to call reset_encoder_cache utility")?;
|
||||
println!("reset_encoder_cache=ok");
|
||||
|
||||
if args.skip_sleep_wake {
|
||||
println!("sleep_wake=skipped");
|
||||
} else {
|
||||
client.sleep(args.sleep_level, &args.sleep_mode).await.with_context(|| {
|
||||
format!(
|
||||
"failed to call sleep utility with level={} mode={}",
|
||||
args.sleep_level, args.sleep_mode
|
||||
)
|
||||
})?;
|
||||
println!(
|
||||
"sleep=ok level={} mode={}",
|
||||
args.sleep_level, args.sleep_mode
|
||||
);
|
||||
|
||||
let sleeping_after_sleep =
|
||||
client.is_sleeping().await.context("failed to call is_sleeping after sleep")?;
|
||||
println!("is_sleeping(after_sleep)={sleeping_after_sleep}");
|
||||
|
||||
if !sleeping_after_sleep {
|
||||
bail!("engine should report sleeping=true after sleep()");
|
||||
}
|
||||
|
||||
client.wake_up(None).await.context("failed to call wake_up utility")?;
|
||||
println!("wake_up=ok");
|
||||
|
||||
let sleeping_after_wake =
|
||||
client.is_sleeping().await.context("failed to call is_sleeping after wake_up")?;
|
||||
println!("is_sleeping(after_wake)={sleeping_after_wake}");
|
||||
|
||||
if sleeping_after_wake {
|
||||
bail!("engine should report sleeping=false after wake_up()");
|
||||
}
|
||||
}
|
||||
|
||||
client.shutdown().await.context("failed to shut down engine-core client")?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
Reference in New Issue
Block a user