[Rust Frontend] Move code from vllm-frontend-rs (#43283)

Signed-off-by: Bugen Zhao <i@bugenzhao.com>
Signed-off-by: Nick Hill <nickhill123@gmail.com>
Signed-off-by: Eric Curtin <eric.curtin@docker.com>
Signed-off-by: Dev-X25874 <283057883+Dev-X25874@users.noreply.github.com>
Signed-off-by: Will.hou <1205157517@qq.com>
Signed-off-by: Will.hou <willamhou@ceresman.com>
Co-authored-by: Nick Hill <nickhill123@gmail.com>
Co-authored-by: Eric Curtin <eric.curtin@docker.com>
Co-authored-by: Dev-X25874 <283057883+Dev-X25874@users.noreply.github.com>
Co-authored-by: Will.hou <1205157517@qq.com>
Co-authored-by: Will.hou <willamhou@ceresman.com>

Please see https://github.com/Inferact/vllm-frontend-rs for full original commit history.
This commit is contained in:
Bugen Zhao
2026-05-21 17:21:48 -07:00
committed by GitHub
co-authored by Nick Hill Eric Curtin Dev-X25874 Will.hou Will.hou
parent 39d5fa96a7
commit 39910f2b25
284 changed files with 72569 additions and 90 deletions
@@ -0,0 +1,53 @@
# Engine-Core Smoke Tests
Start headless `vllm`:
```bash
source ../vllm/.venv/bin/activate
HF_HUB_OFFLINE=1 \
VLLM_LOGGING_LEVEL=DEBUG \
VLLM_CPU_KVCACHE_SPACE=2 \
VLLM_HOST_IP=127.0.0.1 \
VLLM_LOOPBACK_IP=127.0.0.1 \
python3 -m vllm.entrypoints.cli.main serve Qwen/Qwen3-0.6B \
--headless \
--enable-sleep-mode \
--data-parallel-address 127.0.0.1 \
--data-parallel-rpc-port 62100 \
--data-parallel-size-local 1 \
--max-model-len 512 \
--dtype float16
```
Run the Rust smoke test through the `vllm-engine-core-client` utility interface:
```bash
cargo run -p vllm-engine-core-client --example external_engine_utility_call -- \
--handshake-address tcp://127.0.0.1:62100 \
--host 127.0.0.1
```
If your current engine setup does not support sleep mode, skip the `sleep` / `wake_up` part of the
smoke:
```bash
cargo run -p vllm-engine-core-client --example external_engine_utility_call -- \
--handshake-address tcp://127.0.0.1:62100 \
--host 127.0.0.1 \
--skip-sleep-wake
```
Run the Rust smoke test for sample logprobs decoding through the raw engine-core request path:
```bash
cargo run -p vllm-engine-core-client --example external_engine_logprobs -- \
--handshake-address tcp://127.0.0.1:62100 \
--host 127.0.0.1
```
This smoke requests a small generated-token `logprobs` payload plus prompt logprobs over a much
longer prompt, so it exercises both the inline and aux-frame decode paths against a real engine.
The Rust client decodes those payloads into semantic per-position records rather than exposing the
raw ndarray/tensor wire shape.
IMPORTANT: You must restart `vllm` each time you run the smoke test, as the vLLM engine cannot manage frontend closures and subsequent reconnects. In other words, do not reuse existing `vllm` instances, if any.
@@ -0,0 +1,190 @@
use std::time::Duration;
use anyhow::{Context, Result, bail};
use clap::Parser;
use futures::StreamExt as _;
use tokio::time::timeout;
use tracing_subscriber::EnvFilter;
use vllm_engine_core_client::protocol::{
EngineCoreFinishReason, EngineCoreRequest, EngineCoreSamplingParams,
};
use vllm_engine_core_client::{
EngineCoreClient, EngineCoreClientConfig, EngineCoreStreamOutput, TransportMode,
};
const BASE_PROMPT_TOKEN_IDS: &[u32] = &[20841, 448, 6896, 25, 23811];
#[derive(Debug, Parser)]
#[command(about = "Smoke-test engine-core sample logprobs against an external vLLM engine.")]
struct Args {
#[arg(long)]
handshake_address: String,
#[arg(long, default_value_t = 1)]
engine_count: usize,
#[arg(long, default_value = "Qwen/Qwen3-0.6B")]
model: String,
#[arg(long, default_value = "127.0.0.1")]
host: String,
#[arg(long, default_value_t = 0)]
client_index: u32,
#[arg(long, default_value_t = 30)]
ready_timeout_secs: u64,
#[arg(long, default_value_t = 120)]
output_timeout_secs: u64,
#[arg(long, default_value_t = 1)]
max_tokens: u32,
#[arg(long, default_value_t = 2)]
logprobs: i32,
#[arg(long, default_value_t = 1)]
prompt_logprobs: i32,
#[arg(long, default_value_t = 96)]
prompt_repeats: usize,
}
fn init_tracing() {
let filter = EnvFilter::try_from_default_env()
.unwrap_or_else(|_| EnvFilter::new("vllm_engine_core_client=debug"));
let _ = tracing_subscriber::fmt().with_env_filter(filter).try_init();
}
fn unique_request_id() -> String {
let nanos = std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.expect("system clock should be after unix epoch")
.as_nanos();
format!("rust-engine-core-logprobs-{nanos}")
}
fn build_prompt_token_ids(prompt_repeats: usize) -> Vec<u32> {
let repeats = prompt_repeats.max(1);
BASE_PROMPT_TOKEN_IDS.repeat(repeats)
}
fn build_request(
request_id: String,
prompt_token_ids: Vec<u32>,
max_tokens: u32,
logprobs: i32,
prompt_logprobs: i32,
client_index: u32,
) -> EngineCoreRequest {
EngineCoreRequest {
request_id,
prompt_token_ids: Some(prompt_token_ids),
sampling_params: Some(EngineCoreSamplingParams {
max_tokens,
logprobs: Some(logprobs),
prompt_logprobs: Some(prompt_logprobs),
..EngineCoreSamplingParams::for_test()
}),
arrival_time: 0.0,
client_index,
..EngineCoreRequest::default()
}
}
async fn wait_for_final_output(
mut stream: vllm_engine_core_client::EngineCoreOutputStream,
) -> Result<EngineCoreStreamOutput> {
while let Some(output) = stream.next().await {
let output = output.context("failed to receive engine-core output")?;
if output.finished() {
return Ok(output);
}
}
bail!("request stream ended without a final output")
}
#[tokio::main(flavor = "multi_thread")]
async fn main() -> Result<()> {
init_tracing();
let args = Args::parse();
let ready_timeout = Duration::from_secs(args.ready_timeout_secs);
let output_timeout = Duration::from_secs(args.output_timeout_secs);
let request_id = unique_request_id();
let prompt_token_ids = build_prompt_token_ids(args.prompt_repeats);
let client = EngineCoreClient::connect(EngineCoreClientConfig {
transport_mode: TransportMode::HandshakeOwner {
handshake_address: args.handshake_address.clone(),
advertised_host: args.host.clone(),
engine_count: args.engine_count,
ready_timeout,
local_input_address: None,
local_output_address: None,
},
coordinator_mode: None,
model_name: args.model.clone(),
client_index: args.client_index,
})
.await
.context("failed to connect to external vLLM engine")?;
println!("model={}", args.model);
println!("handshake_address={}", args.handshake_address);
println!("engine_count={}", args.engine_count);
println!("input_address={}", client.input_address());
println!("output_address={}", client.output_address());
println!("engine_identities={:x?}", client.engine_identities());
let request = build_request(
request_id.clone(),
prompt_token_ids.clone(),
args.max_tokens,
args.logprobs,
args.prompt_logprobs,
args.client_index,
);
println!("request_id={request_id}");
println!("prompt_len={}", prompt_token_ids.len());
println!("base_prompt_len={}", BASE_PROMPT_TOKEN_IDS.len());
println!("prompt_repeats={}", args.prompt_repeats);
println!("requested_logprobs={}", args.logprobs);
println!("requested_prompt_logprobs={}", args.prompt_logprobs);
let stream = client.call(request).await.context("failed to submit engine-core request")?;
let output = timeout(output_timeout, wait_for_final_output(stream))
.await
.context("timed out waiting for final output")??;
let finish_reason = output.finish_reason;
let token_ids = output.new_token_ids.clone();
let logprobs = output
.new_logprobs
.as_ref()
.and_then(|value| value.as_direct())
.context("engine output did not include decoded sample logprobs")?;
let prompt_logprobs = output
.new_prompt_logprobs_tensors
.as_ref()
.and_then(|value| value.as_direct())
.context("engine output did not include decoded prompt logprobs")?;
println!("token_ids={token_ids:?}");
println!("finish_reason={finish_reason:?}");
println!("new_logprobs={logprobs:#?}");
println!("new_prompt_logprobs_tensors={prompt_logprobs:#?}");
client.shutdown().await.context("failed to shut down engine-core client")?;
if finish_reason != Some(EngineCoreFinishReason::Length) {
bail!("unexpected finish_reason: expected Length, got {finish_reason:?}");
}
if token_ids.is_empty() {
bail!("engine returned no generated token ids");
}
if logprobs.is_empty() {
bail!("decoded logprobs payload is unexpectedly empty");
}
if prompt_logprobs.is_empty() {
bail!("decoded prompt logprobs payload is unexpectedly empty");
}
if prompt_logprobs.len() + 1 < prompt_token_ids.len() {
bail!(
"prompt logprobs rows look too short: prompt_len={}, rows={}",
prompt_token_ids.len(),
prompt_logprobs.len()
);
}
Ok(())
}
@@ -0,0 +1,143 @@
use std::time::Duration;
use anyhow::{Context, Result, bail};
use clap::Parser;
use tracing_subscriber::EnvFilter;
use vllm_engine_core_client::{EngineCoreClient, EngineCoreClientConfig, TransportMode};
#[derive(Debug, Parser)]
#[command(about = "Smoke-test EngineCoreClient utility calls against an external vLLM engine.")]
struct Args {
#[arg(long)]
handshake_address: String,
#[arg(long, default_value_t = 1)]
engine_count: usize,
#[arg(long, default_value = "Qwen/Qwen3-0.6B")]
model: String,
#[arg(long, default_value = "127.0.0.1")]
host: String,
#[arg(long, default_value_t = 0)]
client_index: u32,
#[arg(long, default_value_t = 30)]
ready_timeout_secs: u64,
#[arg(
long,
default_value_t = false,
help = "Expected initial result of is_sleeping() before running the smoke steps."
)]
expected_is_sleeping: bool,
#[arg(long, default_value_t = false)]
reset_running_requests: bool,
#[arg(long, default_value_t = false)]
reset_external: bool,
#[arg(long, default_value_t = 1)]
sleep_level: u32,
#[arg(long, default_value = "abort")]
sleep_mode: String,
#[arg(
long,
default_value_t = false,
help = "Skip sleep/wake_up calls when the engine was not started with sleep-mode support."
)]
skip_sleep_wake: bool,
}
fn init_tracing() {
let filter = EnvFilter::try_from_default_env()
.unwrap_or_else(|_| EnvFilter::new("vllm_engine_core_client=debug"));
let _ = tracing_subscriber::fmt().with_env_filter(filter).try_init();
}
#[tokio::main(flavor = "multi_thread")]
async fn main() -> Result<()> {
init_tracing();
let args = Args::parse();
let client = EngineCoreClient::connect(EngineCoreClientConfig {
transport_mode: TransportMode::HandshakeOwner {
handshake_address: args.handshake_address.clone(),
advertised_host: args.host.clone(),
engine_count: args.engine_count,
ready_timeout: Duration::from_secs(args.ready_timeout_secs),
local_input_address: None,
local_output_address: None,
},
coordinator_mode: None,
model_name: args.model.clone(),
client_index: args.client_index,
})
.await
.context("failed to connect to external vLLM engine")?;
println!("model={}", args.model);
println!("handshake_address={}", args.handshake_address);
println!("engine_count={}", args.engine_count);
println!("input_address={}", client.input_address());
println!("output_address={}", client.output_address());
println!("engine_identities={:x?}", client.engine_identities());
let initial_is_sleeping =
client.is_sleeping().await.context("failed to call is_sleeping utility")?;
println!("is_sleeping(initial)={initial_is_sleeping}");
if initial_is_sleeping != args.expected_is_sleeping {
bail!(
"unexpected initial is_sleeping state: expected {}, got {}",
args.expected_is_sleeping,
initial_is_sleeping
);
}
let reset_prefix_cache = client
.reset_prefix_cache(args.reset_running_requests, args.reset_external)
.await
.context("failed to call reset_prefix_cache utility")?;
println!("reset_prefix_cache={reset_prefix_cache}");
client.reset_mm_cache().await.context("failed to call reset_mm_cache utility")?;
println!("reset_mm_cache=ok");
client
.reset_encoder_cache()
.await
.context("failed to call reset_encoder_cache utility")?;
println!("reset_encoder_cache=ok");
if args.skip_sleep_wake {
println!("sleep_wake=skipped");
} else {
client.sleep(args.sleep_level, &args.sleep_mode).await.with_context(|| {
format!(
"failed to call sleep utility with level={} mode={}",
args.sleep_level, args.sleep_mode
)
})?;
println!(
"sleep=ok level={} mode={}",
args.sleep_level, args.sleep_mode
);
let sleeping_after_sleep =
client.is_sleeping().await.context("failed to call is_sleeping after sleep")?;
println!("is_sleeping(after_sleep)={sleeping_after_sleep}");
if !sleeping_after_sleep {
bail!("engine should report sleeping=true after sleep()");
}
client.wake_up(None).await.context("failed to call wake_up utility")?;
println!("wake_up=ok");
let sleeping_after_wake =
client.is_sleeping().await.context("failed to call is_sleeping after wake_up")?;
println!("is_sleeping(after_wake)={sleeping_after_wake}");
if sleeping_after_wake {
bail!("engine should report sleeping=false after wake_up()");
}
}
client.shutdown().await.context("failed to shut down engine-core client")?;
Ok(())
}