[Rust Frontend] Move code from vllm-frontend-rs (#43283)

Signed-off-by: Bugen Zhao <i@bugenzhao.com>
Signed-off-by: Nick Hill <nickhill123@gmail.com>
Signed-off-by: Eric Curtin <eric.curtin@docker.com>
Signed-off-by: Dev-X25874 <283057883+Dev-X25874@users.noreply.github.com>
Signed-off-by: Will.hou <1205157517@qq.com>
Signed-off-by: Will.hou <willamhou@ceresman.com>
Co-authored-by: Nick Hill <nickhill123@gmail.com>
Co-authored-by: Eric Curtin <eric.curtin@docker.com>
Co-authored-by: Dev-X25874 <283057883+Dev-X25874@users.noreply.github.com>
Co-authored-by: Will.hou <1205157517@qq.com>
Co-authored-by: Will.hou <willamhou@ceresman.com>

Please see https://github.com/Inferact/vllm-frontend-rs for full original commit history.
This commit is contained in:
Bugen Zhao
2026-05-21 17:21:48 -07:00
committed by GitHub
co-authored by Nick Hill Eric Curtin Dev-X25874 Will.hou Will.hou
parent 39d5fa96a7
commit 39910f2b25
284 changed files with 72569 additions and 90 deletions
+39
View File
@@ -0,0 +1,39 @@
# Chat Smoke Test
Start a fresh headless `vllm` engine:
```bash
source ../vllm/.venv/bin/activate
HF_HUB_OFFLINE=1 \
VLLM_LOGGING_LEVEL=DEBUG \
VLLM_CPU_KVCACHE_SPACE=2 \
VLLM_HOST_IP=127.0.0.1 \
VLLM_LOOPBACK_IP=127.0.0.1 \
python3 -m vllm.entrypoints.cli.main serve Qwen/Qwen3-0.6B \
--headless \
--data-parallel-address 127.0.0.1 \
--data-parallel-rpc-port 62100 \
--data-parallel-size-local 1 \
--max-model-len 512 \
--dtype float16
```
Run the Rust chat smoke test through the `vllm-chat` interface:
```bash
cargo run -p vllm-chat --example external_engine_chat_qwen -- \
--handshake-address tcp://127.0.0.1:62100 \
--host 127.0.0.1 \
--prompt 'What is the capital of France? Answer with one word.'
```
The example now defaults to `Qwen/Qwen3-0.6B`. The current `vllm-chat`
request model stays text-first and supports either plain string content or
OpenAI-style text blocks, while the output side now emits structured assistant
events and automatically separates reasoning blocks for supported models. Tool
use and multimodal inputs are still out of scope. It uses the Rust
`tokenizers` library for the tokenizer itself, plus standard Hugging Face
config files to load the chat template and EOS metadata.
IMPORTANT: Restart `vllm` each time you run the smoke test. The current headless
engine cannot safely handle frontend reconnects after the client shuts down.
@@ -0,0 +1,178 @@
use std::time::Duration;
use anyhow::{Context, Result, bail};
use clap::Parser;
use futures::StreamExt as _;
use tracing_subscriber::EnvFilter;
use vllm_chat::{
AssistantBlockKind, AssistantMessageExt as _, ChatEvent, ChatLlm, ChatMessage, ChatRequest,
ChatRole, SamplingParams, load_model_backends,
};
use vllm_engine_core_client::{EngineCoreClient, EngineCoreClientConfig, TransportMode};
use vllm_llm::Llm;
use vllm_text::TextLlm;
#[derive(Debug, Parser)]
#[command(about = "Smoke-test the Rust chat facade against an external Qwen vLLM engine.")]
struct Args {
#[arg(long)]
handshake_address: String,
#[arg(long, default_value_t = 1)]
engine_count: usize,
#[arg(long, default_value = "Qwen/Qwen3-0.6B")]
model: String,
#[arg(long, default_value = "127.0.0.1")]
host: String,
#[arg(long, default_value_t = 30)]
ready_timeout_secs: u64,
#[arg(long)]
prompt: String,
}
const CLIENT_INDEX: u32 = 0;
const OUTPUT_TIMEOUT_SECS: u64 = 120;
fn unique_request_id() -> String {
format!("rust-chat-smoke-{}", uuid::Uuid::new_v4())
}
fn init_tracing() {
let filter = EnvFilter::try_from_default_env()
.unwrap_or_else(|_| EnvFilter::new("vllm_engine_core_client=debug"));
let _ = tracing_subscriber::fmt().with_env_filter(filter).try_init();
}
#[tokio::main(flavor = "multi_thread")]
async fn main() -> Result<()> {
init_tracing();
let args = Args::parse();
let loaded = load_model_backends(&args.model, Default::default())
.await
.with_context(|| format!("failed to load backends for {}", args.model))?;
let text_backend = loaded.text_backend;
let chat_backend = loaded.chat_backend;
let ready_timeout = Duration::from_secs(args.ready_timeout_secs);
let output_timeout = Duration::from_secs(OUTPUT_TIMEOUT_SECS);
let request_id = unique_request_id();
let client = EngineCoreClient::connect(EngineCoreClientConfig {
transport_mode: TransportMode::HandshakeOwner {
handshake_address: args.handshake_address.clone(),
advertised_host: args.host.clone(),
engine_count: args.engine_count,
ready_timeout,
local_input_address: None,
local_output_address: None,
},
coordinator_mode: None,
model_name: args.model.clone(),
client_index: CLIENT_INDEX,
})
.await
.context("failed to connect to external vLLM engine")?;
println!("model={}", args.model);
println!("tokenizer_source=tokenizers + hf-hub");
println!("chat_template_source=tokenizer_config.json or adjacent chat template file");
println!("handshake_address={}", args.handshake_address);
println!("engine_count={}", args.engine_count);
println!("input_address={}", client.input_address());
println!("output_address={}", client.output_address());
println!("engine_identities={:x?}", client.engine_identities());
let llm = Llm::new(client);
let chat = ChatLlm::new(TextLlm::new(llm, text_backend), chat_backend);
let request = ChatRequest {
messages: vec![ChatMessage::text(ChatRole::User, args.prompt.clone())],
sampling_params: SamplingParams {
temperature: Some(0.0),
..Default::default()
},
request_id: request_id.clone(),
..ChatRequest::for_test()
};
println!("request_id={request_id}");
println!("prompt={}", args.prompt);
let mut stream = chat.chat(request).await.context("failed to submit chat request")?;
let output = tokio::time::timeout(output_timeout, async {
let mut final_reasoning = String::new();
let mut final_text = String::new();
let mut final_output_token_count = 0usize;
let mut finish_reason = None;
let mut saw_start = false;
let mut saw_stream_output = false;
while let Some(event) = stream.next().await.transpose()? {
match event {
ChatEvent::Start { .. } => {
saw_start = true;
}
ChatEvent::BlockStart { kind, .. } => {
if saw_stream_output {
println!();
}
match kind {
AssistantBlockKind::Reasoning => print!("[reasoning] "),
AssistantBlockKind::Text => print!("[answer] "),
AssistantBlockKind::ToolCall => {}
}
saw_stream_output = true;
}
ChatEvent::ToolCallStart { name, .. } => {
if saw_stream_output {
println!();
}
print!("[tool:{name}] ");
saw_stream_output = true;
}
ChatEvent::LogprobsDelta { .. } => {}
ChatEvent::Done {
message,
output_token_count,
finish_reason: reason,
..
} => {
final_reasoning = message.reasoning().unwrap_or_default();
final_text = message.text();
final_output_token_count = output_token_count;
finish_reason = Some(reason);
break;
}
ChatEvent::BlockDelta { kind, delta, .. } => match kind {
AssistantBlockKind::Reasoning | AssistantBlockKind::Text => {
print!("{delta}");
}
AssistantBlockKind::ToolCall => {}
},
ChatEvent::ToolCallArgumentsDelta { delta, .. } => print!("{delta}"),
ChatEvent::BlockEnd { .. } | ChatEvent::ToolCallEnd { .. } => {}
}
}
println!();
if !saw_start {
bail!("chat stream ended without a start event");
}
Ok::<_, anyhow::Error>((
final_reasoning,
final_text,
final_output_token_count,
finish_reason,
))
})
.await
.context("timed out waiting for chat output")??;
chat.shutdown().await.context("failed to shut down chat client")?;
println!("final_reasoning={:?}", output.0);
println!("final_text={:?}", output.1);
println!("final_output_token_count={:?}", output.2);
println!("finish_reason={:?}", output.3);
Ok(())
}