[Rust Frontend] Move code from vllm-frontend-rs (#43283)
Signed-off-by: Bugen Zhao <i@bugenzhao.com> Signed-off-by: Nick Hill <nickhill123@gmail.com> Signed-off-by: Eric Curtin <eric.curtin@docker.com> Signed-off-by: Dev-X25874 <283057883+Dev-X25874@users.noreply.github.com> Signed-off-by: Will.hou <1205157517@qq.com> Signed-off-by: Will.hou <willamhou@ceresman.com> Co-authored-by: Nick Hill <nickhill123@gmail.com> Co-authored-by: Eric Curtin <eric.curtin@docker.com> Co-authored-by: Dev-X25874 <283057883+Dev-X25874@users.noreply.github.com> Co-authored-by: Will.hou <1205157517@qq.com> Co-authored-by: Will.hou <willamhou@ceresman.com> Please see https://github.com/Inferact/vllm-frontend-rs for full original commit history.
This commit is contained in:
co-authored by
Nick Hill
Eric Curtin
Dev-X25874
Will.hou
Will.hou
parent
39d5fa96a7
commit
39910f2b25
@@ -0,0 +1,39 @@
|
||||
# Chat Smoke Test
|
||||
|
||||
Start a fresh headless `vllm` engine:
|
||||
|
||||
```bash
|
||||
source ../vllm/.venv/bin/activate
|
||||
HF_HUB_OFFLINE=1 \
|
||||
VLLM_LOGGING_LEVEL=DEBUG \
|
||||
VLLM_CPU_KVCACHE_SPACE=2 \
|
||||
VLLM_HOST_IP=127.0.0.1 \
|
||||
VLLM_LOOPBACK_IP=127.0.0.1 \
|
||||
python3 -m vllm.entrypoints.cli.main serve Qwen/Qwen3-0.6B \
|
||||
--headless \
|
||||
--data-parallel-address 127.0.0.1 \
|
||||
--data-parallel-rpc-port 62100 \
|
||||
--data-parallel-size-local 1 \
|
||||
--max-model-len 512 \
|
||||
--dtype float16
|
||||
```
|
||||
|
||||
Run the Rust chat smoke test through the `vllm-chat` interface:
|
||||
|
||||
```bash
|
||||
cargo run -p vllm-chat --example external_engine_chat_qwen -- \
|
||||
--handshake-address tcp://127.0.0.1:62100 \
|
||||
--host 127.0.0.1 \
|
||||
--prompt 'What is the capital of France? Answer with one word.'
|
||||
```
|
||||
|
||||
The example now defaults to `Qwen/Qwen3-0.6B`. The current `vllm-chat`
|
||||
request model stays text-first and supports either plain string content or
|
||||
OpenAI-style text blocks, while the output side now emits structured assistant
|
||||
events and automatically separates reasoning blocks for supported models. Tool
|
||||
use and multimodal inputs are still out of scope. It uses the Rust
|
||||
`tokenizers` library for the tokenizer itself, plus standard Hugging Face
|
||||
config files to load the chat template and EOS metadata.
|
||||
|
||||
IMPORTANT: Restart `vllm` each time you run the smoke test. The current headless
|
||||
engine cannot safely handle frontend reconnects after the client shuts down.
|
||||
@@ -0,0 +1,178 @@
|
||||
use std::time::Duration;
|
||||
|
||||
use anyhow::{Context, Result, bail};
|
||||
use clap::Parser;
|
||||
use futures::StreamExt as _;
|
||||
use tracing_subscriber::EnvFilter;
|
||||
use vllm_chat::{
|
||||
AssistantBlockKind, AssistantMessageExt as _, ChatEvent, ChatLlm, ChatMessage, ChatRequest,
|
||||
ChatRole, SamplingParams, load_model_backends,
|
||||
};
|
||||
use vllm_engine_core_client::{EngineCoreClient, EngineCoreClientConfig, TransportMode};
|
||||
use vllm_llm::Llm;
|
||||
use vllm_text::TextLlm;
|
||||
|
||||
#[derive(Debug, Parser)]
|
||||
#[command(about = "Smoke-test the Rust chat facade against an external Qwen vLLM engine.")]
|
||||
struct Args {
|
||||
#[arg(long)]
|
||||
handshake_address: String,
|
||||
#[arg(long, default_value_t = 1)]
|
||||
engine_count: usize,
|
||||
#[arg(long, default_value = "Qwen/Qwen3-0.6B")]
|
||||
model: String,
|
||||
#[arg(long, default_value = "127.0.0.1")]
|
||||
host: String,
|
||||
#[arg(long, default_value_t = 30)]
|
||||
ready_timeout_secs: u64,
|
||||
#[arg(long)]
|
||||
prompt: String,
|
||||
}
|
||||
|
||||
const CLIENT_INDEX: u32 = 0;
|
||||
const OUTPUT_TIMEOUT_SECS: u64 = 120;
|
||||
|
||||
fn unique_request_id() -> String {
|
||||
format!("rust-chat-smoke-{}", uuid::Uuid::new_v4())
|
||||
}
|
||||
|
||||
fn init_tracing() {
|
||||
let filter = EnvFilter::try_from_default_env()
|
||||
.unwrap_or_else(|_| EnvFilter::new("vllm_engine_core_client=debug"));
|
||||
let _ = tracing_subscriber::fmt().with_env_filter(filter).try_init();
|
||||
}
|
||||
|
||||
#[tokio::main(flavor = "multi_thread")]
|
||||
async fn main() -> Result<()> {
|
||||
init_tracing();
|
||||
let args = Args::parse();
|
||||
let loaded = load_model_backends(&args.model, Default::default())
|
||||
.await
|
||||
.with_context(|| format!("failed to load backends for {}", args.model))?;
|
||||
let text_backend = loaded.text_backend;
|
||||
let chat_backend = loaded.chat_backend;
|
||||
|
||||
let ready_timeout = Duration::from_secs(args.ready_timeout_secs);
|
||||
let output_timeout = Duration::from_secs(OUTPUT_TIMEOUT_SECS);
|
||||
let request_id = unique_request_id();
|
||||
let client = EngineCoreClient::connect(EngineCoreClientConfig {
|
||||
transport_mode: TransportMode::HandshakeOwner {
|
||||
handshake_address: args.handshake_address.clone(),
|
||||
advertised_host: args.host.clone(),
|
||||
engine_count: args.engine_count,
|
||||
ready_timeout,
|
||||
local_input_address: None,
|
||||
local_output_address: None,
|
||||
},
|
||||
coordinator_mode: None,
|
||||
model_name: args.model.clone(),
|
||||
client_index: CLIENT_INDEX,
|
||||
})
|
||||
.await
|
||||
.context("failed to connect to external vLLM engine")?;
|
||||
|
||||
println!("model={}", args.model);
|
||||
println!("tokenizer_source=tokenizers + hf-hub");
|
||||
println!("chat_template_source=tokenizer_config.json or adjacent chat template file");
|
||||
println!("handshake_address={}", args.handshake_address);
|
||||
println!("engine_count={}", args.engine_count);
|
||||
println!("input_address={}", client.input_address());
|
||||
println!("output_address={}", client.output_address());
|
||||
println!("engine_identities={:x?}", client.engine_identities());
|
||||
|
||||
let llm = Llm::new(client);
|
||||
let chat = ChatLlm::new(TextLlm::new(llm, text_backend), chat_backend);
|
||||
|
||||
let request = ChatRequest {
|
||||
messages: vec![ChatMessage::text(ChatRole::User, args.prompt.clone())],
|
||||
sampling_params: SamplingParams {
|
||||
temperature: Some(0.0),
|
||||
..Default::default()
|
||||
},
|
||||
request_id: request_id.clone(),
|
||||
..ChatRequest::for_test()
|
||||
};
|
||||
|
||||
println!("request_id={request_id}");
|
||||
println!("prompt={}", args.prompt);
|
||||
|
||||
let mut stream = chat.chat(request).await.context("failed to submit chat request")?;
|
||||
let output = tokio::time::timeout(output_timeout, async {
|
||||
let mut final_reasoning = String::new();
|
||||
let mut final_text = String::new();
|
||||
let mut final_output_token_count = 0usize;
|
||||
let mut finish_reason = None;
|
||||
let mut saw_start = false;
|
||||
let mut saw_stream_output = false;
|
||||
|
||||
while let Some(event) = stream.next().await.transpose()? {
|
||||
match event {
|
||||
ChatEvent::Start { .. } => {
|
||||
saw_start = true;
|
||||
}
|
||||
ChatEvent::BlockStart { kind, .. } => {
|
||||
if saw_stream_output {
|
||||
println!();
|
||||
}
|
||||
match kind {
|
||||
AssistantBlockKind::Reasoning => print!("[reasoning] "),
|
||||
AssistantBlockKind::Text => print!("[answer] "),
|
||||
AssistantBlockKind::ToolCall => {}
|
||||
}
|
||||
saw_stream_output = true;
|
||||
}
|
||||
ChatEvent::ToolCallStart { name, .. } => {
|
||||
if saw_stream_output {
|
||||
println!();
|
||||
}
|
||||
print!("[tool:{name}] ");
|
||||
saw_stream_output = true;
|
||||
}
|
||||
ChatEvent::LogprobsDelta { .. } => {}
|
||||
ChatEvent::Done {
|
||||
message,
|
||||
output_token_count,
|
||||
finish_reason: reason,
|
||||
..
|
||||
} => {
|
||||
final_reasoning = message.reasoning().unwrap_or_default();
|
||||
final_text = message.text();
|
||||
final_output_token_count = output_token_count;
|
||||
finish_reason = Some(reason);
|
||||
break;
|
||||
}
|
||||
ChatEvent::BlockDelta { kind, delta, .. } => match kind {
|
||||
AssistantBlockKind::Reasoning | AssistantBlockKind::Text => {
|
||||
print!("{delta}");
|
||||
}
|
||||
AssistantBlockKind::ToolCall => {}
|
||||
},
|
||||
ChatEvent::ToolCallArgumentsDelta { delta, .. } => print!("{delta}"),
|
||||
ChatEvent::BlockEnd { .. } | ChatEvent::ToolCallEnd { .. } => {}
|
||||
}
|
||||
}
|
||||
|
||||
println!();
|
||||
|
||||
if !saw_start {
|
||||
bail!("chat stream ended without a start event");
|
||||
}
|
||||
Ok::<_, anyhow::Error>((
|
||||
final_reasoning,
|
||||
final_text,
|
||||
final_output_token_count,
|
||||
finish_reason,
|
||||
))
|
||||
})
|
||||
.await
|
||||
.context("timed out waiting for chat output")??;
|
||||
|
||||
chat.shutdown().await.context("failed to shut down chat client")?;
|
||||
|
||||
println!("final_reasoning={:?}", output.0);
|
||||
println!("final_text={:?}", output.1);
|
||||
println!("final_output_token_count={:?}", output.2);
|
||||
println!("finish_reason={:?}", output.3);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
Reference in New Issue
Block a user