[Rust Frontend] Introduce mock engine for benchmark baseline (#43469)

Signed-off-by: Bugen Zhao <i@bugenzhao.com>
This commit is contained in:
Bugen Zhao
2026-05-28 01:40:35 +00:00
committed by GitHub
parent 1223732dda
commit c87f62ccf8
19 changed files with 1462 additions and 204 deletions
+30
View File
@@ -0,0 +1,30 @@
[package]
name = "vllm-mock-engine"
version.workspace = true
edition.workspace = true
license.workspace = true
[[bin]]
name = "vllm-mock-engine"
path = "src/main.rs"
[dependencies]
anyhow.workspace = true
asynk-strim-attr.workspace = true
clap.workspace = true
futures.workspace = true
rand.workspace = true
rmpv.workspace = true
serde.workspace = true
tokio = { workspace = true, features = ["signal"] }
tokio-util.workspace = true
tracing.workspace = true
tracing-subscriber.workspace = true
vllm-engine-core-client.workspace = true
zeromq.workspace = true
[dev-dependencies]
vllm-engine-core-client = { workspace = true, features = ["test-util"] }
[lints]
workspace = true
+104
View File
@@ -0,0 +1,104 @@
# vLLM Mock Engine
`vllm-mock-engine` is a small engine-side process for frontend stress testing. It
joins a frontend-owned startup handshake, reports a large ready response, treats
prefill as instant, and emits random decode tokens until each request reaches
its `max_tokens`.
The frontend must own the handshake socket. Start the frontend first, then start
the mock engine with the same handshake address.
## Start the mock engine
```bash
cargo run -p vllm-mock-engine -- \
--handshake-address tcp://127.0.0.1:29550 \
--engine-count 1 \
--output-token-chunk-size 1 \
--vocab-size 32000 \
--seed 0 \
--log-requests
```
Useful knobs:
- `--engine-count` must match the frontend's expected data-parallel engine
count.
- `--output-token-chunk-size` controls how many token IDs appear in one
`EngineCoreOutput`; values greater than 1 are useful for MTP/spec-decode
shaped frontend tests.
- `--vocab-size` should stay within the tokenizer vocabulary of the model used
by the frontend.
Stop it with Ctrl-C.
## Rust Frontend
Terminal 1:
```bash
cargo run --bin vllm-rs -- serve Qwen/Qwen3-0.6B \
--data-parallel-size 1 \
--data-parallel-size-local 0 \
--handshake-port 29550
```
Terminal 2:
```bash
cargo run -p vllm-mock-engine -- \
--handshake-address tcp://127.0.0.1:29550
```
For multiple mock engines, set both sides to the same count:
```bash
cargo run --bin vllm-rs -- serve Qwen/Qwen3-0.6B \
--data-parallel-size 4 \
--data-parallel-size-local 0 \
--handshake-port 29550
cargo run -p vllm-mock-engine -- \
--handshake-address tcp://127.0.0.1:29550 \
--engine-count 4
```
## Python Frontend
Use `vllm serve` with `--data-parallel-size-local 0` so the Python process runs
as a frontend/API server and waits for external engines on
`--data-parallel-rpc-port`.
Terminal 1:
```bash
vllm serve Qwen/Qwen3-0.6B \
--data-parallel-address 127.0.0.1 \
--data-parallel-rpc-port 29550 \
--data-parallel-size 1 \
--data-parallel-size-local 0
```
Terminal 2:
```bash
cargo run -p vllm-mock-engine -- \
--handshake-address tcp://127.0.0.1:29550
```
## Smoke Request
After either frontend is ready:
```bash
curl http://127.0.0.1:8000/v1/chat/completions \
-H "Content-Type: application/json" \
-d '{
"model": "Qwen/Qwen3-0.6B",
"messages": [{"role": "user", "content": "hello"}],
"max_tokens": 16,
"stream": true
}'
```
Always pass `max_tokens`; the mock engine stops by length.
+416
View File
@@ -0,0 +1,416 @@
use std::collections::{BTreeMap, BTreeSet, HashMap};
use std::hash::{Hash as _, Hasher as _};
use std::time::{SystemTime, UNIX_EPOCH};
use anyhow::{Result, anyhow};
use rand::rngs::StdRng;
use rand::{Rng as _, SeedableRng as _};
use rmpv::Value;
use serde::Serialize;
use tokio::sync::mpsc;
use tokio::task::yield_now;
use tokio_util::sync::CancellationToken;
use tracing::{debug, info, warn};
use vllm_engine_core_client::protocol::utility::{
EngineCoreUtilityRequest, UtilityOutput, UtilityResultEnvelope,
};
use vllm_engine_core_client::protocol::{
EngineCoreFinishReason, EngineCoreOutput, EngineCoreOutputs, EngineCoreRequest,
};
use super::Opt;
/// Derive a stable per-request seed from the CLI seed, engine, and request id.
fn request_seed(base_seed: u64, engine_index: u32, request_id: &str) -> u64 {
let mut hasher = std::hash::DefaultHasher::new();
base_seed.hash(&mut hasher);
engine_index.hash(&mut hasher);
request_id.hash(&mut hasher);
hasher.finish()
}
/// Current UNIX timestamp in seconds for engine-core output envelopes.
fn now_secs() -> f64 {
SystemTime::now()
.duration_since(UNIX_EPOCH)
.map(|duration| duration.as_secs_f64())
.unwrap_or_default()
}
/// Build one request output with only token IDs and terminal status populated.
fn request_output(
request_id: String,
new_token_ids: Vec<u32>,
finish_reason: Option<EngineCoreFinishReason>,
) -> EngineCoreOutput {
EngineCoreOutput {
request_id,
new_token_ids,
finish_reason,
..Default::default()
}
}
/// Produce an empty output with a terminal finish reason for an invalid request.
fn empty_finish_outputs(
engine_index: u32,
request_id: String,
finish_reason: EngineCoreFinishReason,
) -> EngineCoreOutputs {
let output = request_output(request_id, Vec::new(), Some(finish_reason));
let finished_requests = BTreeSet::from([output.request_id.clone()]);
EngineCoreOutputs {
engine_index,
outputs: vec![output],
timestamp: now_secs(),
finished_requests: Some(finished_requests),
..Default::default()
}
}
/// Encode a utility result into the protocol's msgpack value envelope.
fn utility_envelope<T>(value: T) -> Result<UtilityResultEnvelope>
where
T: Serialize,
{
Ok(UtilityResultEnvelope::without_type_info(
rmpv::ext::to_value(value)?,
))
}
/// Produce the minimal utility responses needed by the Rust frontend.
fn utility_response(
engine_index: u32,
request: EngineCoreUtilityRequest,
) -> Result<EngineCoreOutputs> {
let result = match request.method_name.as_str() {
"get_supported_tasks" => utility_envelope(vec!["generate"]),
"is_sleeping" => utility_envelope(false),
"reset_prefix_cache" => utility_envelope(true),
"reset_mm_cache"
| "reset_encoder_cache"
| "profile"
| "sleep"
| "wake_up"
| "execute_dummy_batch" => utility_envelope(()),
_ => utility_envelope(Value::Nil),
}?;
Ok(EngineCoreOutputs {
engine_index,
utility_output: Some(UtilityOutput {
call_id: request.call_id,
failure_message: None,
result: Some(result),
}),
timestamp: now_secs(),
..Default::default()
})
}
/// Message sent from the frontend to the mock engine task to drive the engine loop.
pub(crate) enum EngineInput {
Request(Box<EngineCoreRequest>),
Abort(Vec<String>),
Utility(EngineCoreUtilityRequest),
StartDpWave,
}
/// Message sent from the mock engine task to the frontend for one engine output batch.
pub(crate) struct EngineOutput {
pub client_index: u32,
pub outputs: EngineCoreOutputs,
}
/// Per-request decode state owned by one mock engine.
#[derive(Debug)]
struct ActiveRequest {
request_id: String,
client_index: u32,
prompt_len: usize,
max_tokens: usize,
generated: usize,
rng: StdRng,
}
impl ActiveRequest {
/// Create a new active request from an incoming EngineCoreRequest, or return an immediate
/// finish reason if the request is invalid.
fn new(
engine_index: u32,
request: Box<EngineCoreRequest>,
opt: &Opt,
) -> Result<Self, EngineCoreFinishReason> {
let request_id = request.request_id;
let client_index = request.client_index;
let prompt_len = request.prompt_token_ids.as_ref().map(Vec::len).unwrap_or_default();
let Some(sampling_params) = request.sampling_params else {
warn!(
request_id,
"request has no sampling params; returning engine error"
);
return Err(EngineCoreFinishReason::Error);
};
let max_tokens = sampling_params.max_tokens as usize;
if opt.log_requests {
info!(
request_id,
prompt_len,
max_tokens,
chunk_size = opt.output_token_chunk_size,
"mock request started"
);
}
if max_tokens == 0 {
return Err(EngineCoreFinishReason::Length);
}
Ok(ActiveRequest {
rng: StdRng::seed_from_u64(request_seed(opt.seed, engine_index, &request_id)),
request_id,
client_index,
prompt_len,
max_tokens,
generated: 0,
})
}
/// Advance this request by one mock engine step.
fn step(&mut self, opt: &Opt) -> EngineCoreOutput {
let remaining = self.max_tokens - self.generated;
let chunk_len = remaining.min(opt.output_token_chunk_size);
let mut new_token_ids = Vec::with_capacity(chunk_len);
for _ in 0..chunk_len {
new_token_ids.push(self.rng.random_range(0..opt.vocab_size));
}
self.generated += chunk_len;
let finished = self.generated >= self.max_tokens;
request_output(
self.request_id.clone(),
new_token_ids,
finished.then_some(EngineCoreFinishReason::Length),
)
}
}
/// Internal state for one mock engine instance, owned by the engine loop task.
struct Engine {
engine_index: u32,
opt: Opt,
active_requests: HashMap<String, ActiveRequest>,
}
impl Engine {
/// Drain one frontend request message received on the input DEALER socket.
fn handle_input(&mut self, input: EngineInput) -> Result<Vec<EngineOutput>> {
let mut outputs = Vec::new();
match input {
EngineInput::Request(request) => {
let request_id = request.request_id.clone();
let client_index = request.client_index;
if self.active_requests.contains_key(&request_id) {
warn!(
engine_index = self.engine_index,
request_id, "duplicate mock request id"
);
return Ok(vec![EngineOutput {
client_index,
outputs: empty_finish_outputs(
self.engine_index,
request_id,
EngineCoreFinishReason::Error,
),
}]);
}
match ActiveRequest::new(self.engine_index, request, &self.opt) {
Ok(request) => {
self.active_requests.insert(request_id, request);
}
Err(finish_reason) => {
return Ok(vec![EngineOutput {
client_index,
outputs: empty_finish_outputs(
self.engine_index,
request_id,
finish_reason,
),
}]);
}
}
}
EngineInput::Abort(request_ids) => {
let mut outputs_by_client =
BTreeMap::<u32, (Vec<EngineCoreOutput>, BTreeSet<String>)>::new();
for request_id in request_ids {
if let Some(request) = self.active_requests.remove(&request_id) {
let output = request_output(
request_id.clone(),
Vec::new(),
Some(EngineCoreFinishReason::Abort),
);
let (outputs, finished_requests) = outputs_by_client
.entry(request.client_index)
.or_insert_with(|| (Vec::new(), BTreeSet::new()));
outputs.push(output);
finished_requests.insert(request_id.clone());
if self.opt.log_requests {
info!(request_id, finish_reason = "abort", "mock request aborted");
}
}
}
for (client_index, (client_outputs, finished_requests)) in outputs_by_client {
outputs.push({
let outputs = EngineCoreOutputs {
engine_index: self.engine_index,
outputs: client_outputs,
timestamp: now_secs(),
finished_requests: Some(finished_requests),
..Default::default()
};
EngineOutput {
client_index,
outputs,
}
});
}
}
EngineInput::Utility(request) => {
debug!(
engine_index = self.engine_index,
call_id = %request.call_id,
method = request.method_name,
"mock utility request"
);
let client_index = request.client_index;
outputs.push({
let outputs = utility_response(self.engine_index, request)?;
EngineOutput {
client_index,
outputs,
}
});
}
EngineInput::StartDpWave => {
debug!(
engine_index = self.engine_index,
"ignoring START_DP_WAVE in mock engine"
);
}
}
Ok(outputs)
}
/// Advance active requests once and return one batched engine output.
fn step(&mut self) -> Vec<EngineOutput> {
if self.active_requests.is_empty() {
return Vec::new();
}
let mut outputs_by_client =
BTreeMap::<u32, (Vec<EngineCoreOutput>, BTreeSet<String>)>::new();
let mut all_finished_requests = BTreeSet::new();
for request in self.active_requests.values_mut() {
let client_index = request.client_index;
let output = request.step(&self.opt);
let request_id = request.request_id.clone();
let finished = output.finished();
if output.finished() {
all_finished_requests.insert(request_id.clone());
if self.opt.log_requests {
info!(
request_id,
prompt_len = request.prompt_len,
output_tokens = request.generated,
finish_reason = "length",
"mock request finished"
);
}
}
let (outputs, finished_requests) = outputs_by_client
.entry(client_index)
.or_insert_with(|| (Vec::new(), BTreeSet::new()));
if finished {
finished_requests.insert(request_id.clone());
}
outputs.push(output);
}
for request_id in &all_finished_requests {
self.active_requests.remove(request_id);
}
outputs_by_client
.into_iter()
.filter_map(|(client_index, (outputs, finished_requests))| {
(!outputs.is_empty()).then(|| EngineOutput {
client_index,
outputs: EngineCoreOutputs {
engine_index: self.engine_index,
outputs,
timestamp: now_secs(),
finished_requests: (!finished_requests.is_empty())
.then_some(finished_requests),
..Default::default()
},
})
})
.collect()
}
}
/// Run the main loop for the mock engine, receiving `EngineInput` from `input_rx`
/// and sending `EngineOutput` to `output_tx` until `shutdown` is cancelled.
pub(crate) async fn run_engine_loop(
engine_index: u32,
opt: Opt,
mut input_rx: mpsc::UnboundedReceiver<EngineInput>,
output_tx: mpsc::Sender<EngineOutput>,
shutdown: CancellationToken,
) -> Result<()> {
let mut engine = Engine {
engine_index,
opt,
active_requests: HashMap::new(),
};
loop {
let outputs = tokio::select! {
biased;
_ = shutdown.cancelled() => break,
input = input_rx.recv() => {
let input = input
.ok_or_else(|| anyhow!("mock engine input channel closed"))?;
engine.handle_input(input)?
}
// If there are active requests, step them once after yielding to the scheduler to
// avoid blocking the engine loop while still making steady progress on request outputs.
_ = yield_now(), if !engine.active_requests.is_empty() => {
engine.step()
}
};
for output in outputs {
output_tx
.send(output)
.await
.map_err(|_| anyhow!("mock engine IO task shut down"))?;
}
}
Ok(())
}
+121
View File
@@ -0,0 +1,121 @@
use anyhow::{Context as _, Result, anyhow, bail};
use futures::{Stream, StreamExt as _, stream};
use tokio::sync::mpsc;
use tokio_util::sync::CancellationToken;
use tracing::warn;
use vllm_engine_core_client::mock_engine::MockEngineDataSockets;
use vllm_engine_core_client::protocol::utility::EngineCoreUtilityRequest;
use vllm_engine_core_client::protocol::{
EngineCoreRequest, EngineCoreRequestType, decode_msgpack, encode_msgpack,
};
use zeromq::{DealerSocket, PushSocket, SocketRecv as _, SocketSend as _, ZmqMessage};
use crate::engine::{EngineInput, EngineOutput};
/// Send one engine output batch to the client over the appropriate push socket.
async fn send_engine_outputs_to_client(
push_sockets: &mut [PushSocket],
EngineOutput {
client_index,
outputs,
}: EngineOutput,
) -> Result<()> {
let message = ZmqMessage::from(encode_msgpack(&outputs)?);
push_sockets[client_index as usize].send(message).await?;
Ok(())
}
/// Create a stream of `EngineInput` by continuously receiving messages from the given dealer socket
/// and decoding them into `EngineInput`.
fn dealer_input_stream(dealer: DealerSocket) -> impl Stream<Item = Result<EngineInput>> {
stream::unfold(dealer, |mut dealer| async {
let input = loop {
let message =
match dealer.recv().await.context("failed to receive message from dealer socket") {
Ok(message) => message,
Err(err) => break Err(err),
};
match decode_request(message) {
Ok(input) => break Ok(input),
Err(err) => {
warn!(%err, "failed to decode engine request message; ignoring");
}
}
};
Some((input, dealer))
})
}
/// Decode a `ZmqMessage` into an `EngineInput`. Returns an error if the message is malformed or
/// contains an unknown/unsupported request type.
fn decode_request(message: ZmqMessage) -> Result<EngineInput> {
let frames = message.into_vec();
if frames.is_empty() {
bail!("empty engine request message");
}
if frames.len() != 2 {
bail!("invalid frame count for engine request: {}", frames.len());
}
let request_type_frame = frames[0].as_ref();
let Some(request_type) = EngineCoreRequestType::from_frame(request_type_frame) else {
bail!("unknown engine request type: {:?}", request_type_frame);
};
let input = match request_type {
EngineCoreRequestType::Add => {
let request: Box<EngineCoreRequest> = decode_msgpack(frames[1].as_ref())?;
EngineInput::Request(request)
}
EngineCoreRequestType::Abort => {
let request_ids: Vec<String> = decode_msgpack(frames[1].as_ref())?;
EngineInput::Abort(request_ids)
}
EngineCoreRequestType::Utility => {
let request: EngineCoreUtilityRequest = decode_msgpack(frames[1].as_ref())?;
EngineInput::Utility(request)
}
EngineCoreRequestType::StartDpWave => EngineInput::StartDpWave,
};
Ok(input)
}
/// Run the main IO loop for the mock engine, continuously receiving and decoding raw messages from
/// the dealer sockets, sending them to the engine loop task via `input_tx`, and receiving
/// `EngineOutput` from the engine loop task via `output_rx` and sending them to the client over the
/// appropriate push socket, until `shutdown` is cancelled.
pub(crate) async fn run_io_loop(
data_sockets: Vec<MockEngineDataSockets>,
input_tx: mpsc::UnboundedSender<EngineInput>,
mut output_rx: mpsc::Receiver<EngineOutput>,
shutdown: CancellationToken,
) -> Result<()> {
let (dealers, mut push_sockets): (Vec<_>, Vec<_>) =
data_sockets.into_iter().map(|sockets| (sockets.dealer, sockets.push)).unzip();
let mut input_streams =
stream::select_all(dealers.into_iter().map(dealer_input_stream).map(Box::pin));
loop {
tokio::select! {
biased;
_ = shutdown.cancelled() => return Ok(()),
output = output_rx.recv() => {
let output = output
.ok_or_else(|| anyhow!("mock engine output channel closed"))?;
send_engine_outputs_to_client(&mut push_sockets, output).await?;
}
input = input_streams.next() => {
let input = input
.ok_or_else(|| anyhow!("mock engine input streams closed"))??;
input_tx
.send(input)
.map_err(|_| anyhow!("mock engine state task shut down"))?;
}
}
}
}
+138
View File
@@ -0,0 +1,138 @@
use anyhow::{Context, Result, bail};
use clap::Parser;
use tokio::sync::mpsc;
use tokio::task::JoinSet;
use tokio_util::sync::CancellationToken;
use tracing::{error, info};
use vllm_engine_core_client::EngineId;
use vllm_engine_core_client::mock_engine::{
MockEngineConfig, MockEngineSockets, connect_to_frontend,
};
pub mod engine;
pub mod io;
/// Standalone engine-core protocol emulator for frontend stress testing.
#[derive(Debug, Clone, Parser)]
#[command(
name = "vllm-mock-engine",
about = "Run a mock vLLM headless engine for Rust frontend stress testing."
)]
pub struct Opt {
/// Frontend-owned ZMQ handshake address.
#[arg(long, default_value = "tcp://127.0.0.1:29550")]
pub handshake_address: String,
/// Number of mock engine identities to register with the frontend.
#[arg(long, default_value_t = 1)]
pub engine_count: usize,
/// Number of accepted output tokens included in each EngineCoreOutput.
#[arg(long, default_value_t = 1)]
pub output_token_chunk_size: usize,
/// Random token IDs are sampled uniformly from 0..vocab_size.
#[arg(long, default_value_t = 32_000)]
pub vocab_size: u32,
/// Base seed for deterministic random token generation.
#[arg(long, default_value_t = 0)]
pub seed: u64,
/// Log a summary line for each request.
#[arg(long)]
pub log_requests: bool,
}
/// Run one mock engine until shutdown or transport failure.
async fn run_engine(engine_index: u32, opt: Opt, shutdown: CancellationToken) -> Result<()> {
let MockEngineSockets { data_sockets, .. } = connect_to_frontend(
&opt.handshake_address,
EngineId::from_engine_index(engine_index),
MockEngineConfig::default(),
)
.await
.with_context(|| format!("mock engine {engine_index} failed to connect to frontend"))?;
info!(engine_index, "mock engine connected to frontend");
let (input_tx, input_rx) = mpsc::unbounded_channel();
let (output_tx, output_rx) = mpsc::channel(64);
// IO loop: dealer -> input_tx, output_rx -> push
let mut io_loop = tokio::spawn(io::run_io_loop(
data_sockets,
input_tx,
output_rx,
shutdown.clone(),
));
// Engine loop: input_rx -> engine logic -> output_tx
let mut engine_loop = tokio::spawn(engine::run_engine_loop(
engine_index,
opt,
input_rx,
output_tx,
shutdown.clone(),
));
tokio::select! {
biased;
_ = shutdown.cancelled() => {
io_loop.abort();
engine_loop.abort();
io_loop.await.ok();
engine_loop.await.ok();
}
result = &mut io_loop => {
error!(engine_index, "mock engine IO loop exited unexpectedly");
engine_loop.abort();
engine_loop.await.ok();
result??;
}
result = &mut engine_loop => {
error!(engine_index, "mock engine loop exited unexpectedly");
io_loop.abort();
io_loop.await.ok();
result??;
}
}
info!(engine_index, "mock engine shut down");
Ok(())
}
/// Run all requested mock engines until cancellation or one engine task fails.
pub async fn run(opt: Opt, shutdown: CancellationToken) -> Result<()> {
info!(?opt, "starting mock engine");
let mut engines = JoinSet::new();
for engine_index in 0..opt.engine_count {
engines.spawn(run_engine(
engine_index as u32,
opt.clone(),
shutdown.clone(),
));
}
tokio::select! {
biased;
_ = shutdown.cancelled() => {
engines.abort_all();
while engines.join_next().await.is_some() {}
Ok(())
}
joined = engines.join_next() => {
match joined {
Some(Ok(Ok(()))) => bail!("mock engine exited unexpectedly"),
Some(Ok(Err(error))) => Err(error),
Some(Err(error)) => Err(error).context("mock engine task join failed"),
None => Ok(()),
}
}
}
}
#[cfg(test)]
mod tests;
+38
View File
@@ -0,0 +1,38 @@
use anyhow::{Context, Result};
use clap::Parser as _;
use tokio_util::sync::CancellationToken;
use tracing::{Level, info};
use vllm_mock_engine::Opt;
fn init_tracing() {
tracing_subscriber::fmt().with_max_level(Level::INFO).init();
}
/// Create a cancellation token that is triggered by Ctrl-C.
fn shutdown_signal() -> CancellationToken {
let token = CancellationToken::new();
let shutdown = token.clone();
tokio::spawn(async move {
tokio::signal::ctrl_c().await.expect("failed to install Ctrl-C signal handler");
info!("received shutdown signal (Ctrl-C), shutting down...");
shutdown.cancel();
});
token
}
fn main() -> Result<()> {
init_tracing();
let opt = Opt::parse();
let runtime = tokio::runtime::Builder::new_multi_thread()
.enable_all()
.build()
.context("failed to build Tokio runtime")?;
runtime.block_on(async move {
let shutdown = shutdown_signal();
vllm_mock_engine::run(opt, shutdown).await
})
}
+197
View File
@@ -0,0 +1,197 @@
use std::net::TcpListener;
use std::time::Duration;
use anyhow::Result;
use futures::StreamExt as _;
use tokio::time::timeout;
use tokio_util::sync::CancellationToken;
use vllm_engine_core_client::protocol::{
EngineCoreFinishReason, EngineCoreRequest, EngineCoreSamplingParams,
};
use vllm_engine_core_client::test_utils::IpcNamespace;
use vllm_engine_core_client::{EngineCoreClient, EngineCoreClientConfig, TransportMode};
use crate::{Opt, run};
fn free_tcp_address() -> String {
let listener = TcpListener::bind("127.0.0.1:0").expect("bind free port");
let port = listener.local_addr().expect("local addr").port();
drop(listener);
format!("tcp://127.0.0.1:{port}")
}
fn client_config(handshake_address: String, engine_count: usize) -> EngineCoreClientConfig {
EngineCoreClientConfig {
transport_mode: TransportMode::HandshakeOwner {
handshake_address,
advertised_host: "127.0.0.1".to_string(),
engine_count,
ready_timeout: Duration::from_secs(5),
local_input_address: None,
local_output_address: None,
},
coordinator_mode: None,
model_name: "mock-model".to_string(),
client_index: 0,
}
}
async fn connect_with_mock(
handshake_address: String,
engine_count: usize,
output_token_chunk_size: usize,
) -> (
EngineCoreClient,
CancellationToken,
tokio::task::JoinHandle<Result<()>>,
) {
let shutdown = CancellationToken::new();
let task = tokio::spawn(run(
Opt {
handshake_address: handshake_address.clone(),
engine_count,
output_token_chunk_size,
vocab_size: 32_000,
seed: 0,
log_requests: false,
},
shutdown.clone(),
));
let client = timeout(
Duration::from_secs(5),
EngineCoreClient::connect(client_config(handshake_address, engine_count)),
)
.await
.expect("client connect timeout")
.expect("connect client");
(client, shutdown, task)
}
fn sample_request(request_id: &str, max_tokens: u32) -> EngineCoreRequest {
EngineCoreRequest {
request_id: request_id.to_string(),
prompt_token_ids: Some(vec![1, 2, 3]),
sampling_params: Some(EngineCoreSamplingParams {
max_tokens,
..EngineCoreSamplingParams::for_test()
}),
arrival_time: 0.0,
..Default::default()
}
}
async fn shutdown_mock(
client: EngineCoreClient,
shutdown: CancellationToken,
task: tokio::task::JoinHandle<Result<()>>,
) {
client.shutdown().await.expect("client shutdown");
shutdown.cancel();
task.await.expect("mock join").expect("mock run");
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn mock_engine_connects_over_tcp() {
let handshake_address = free_tcp_address();
let (client, shutdown, task) = connect_with_mock(handshake_address, 1, 1).await;
assert_eq!(client.engine_count(), 1);
assert_eq!(client.engine_identities()[0], &[0, 0]);
assert_eq!(client.max_model_len(), Some(1024 * 1024));
shutdown_mock(client, shutdown, task).await;
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn mock_engine_connects_over_ipc() {
let ipc = IpcNamespace::new().expect("ipc namespace");
let handshake_address = ipc.handshake_endpoint();
let (client, shutdown, task) = connect_with_mock(handshake_address, 1, 1).await;
assert_eq!(client.engine_count(), 1);
assert_eq!(client.engine_identities()[0], &[0, 0]);
shutdown_mock(client, shutdown, task).await;
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn mock_engine_registers_multiple_identities() {
let handshake_address = free_tcp_address();
let (client, shutdown, task) = connect_with_mock(handshake_address, 2, 1).await;
assert_eq!(client.engine_count(), 2);
assert_eq!(client.engine_identities(), vec![&[0, 0][..], &[1, 0][..]]);
shutdown_mock(client, shutdown, task).await;
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn chunk_size_one_outputs_one_token_per_update() {
let handshake_address = free_tcp_address();
let (client, shutdown, task) = connect_with_mock(handshake_address, 1, 1).await;
let mut stream = client.call(sample_request("req-1", 3)).await.expect("call");
let first = stream.next().await.expect("first").expect("first ok");
assert_eq!(first.new_token_ids.len(), 1);
assert_eq!(first.finish_reason, None);
let second = stream.next().await.expect("second").expect("second ok");
assert_eq!(second.new_token_ids.len(), 1);
assert_eq!(second.finish_reason, None);
let third = stream.next().await.expect("third").expect("third ok");
assert_eq!(third.new_token_ids.len(), 1);
assert_eq!(third.finish_reason, Some(EngineCoreFinishReason::Length));
assert!(stream.next().await.is_none());
shutdown_mock(client, shutdown, task).await;
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn chunk_size_clips_final_output_to_max_tokens() {
let handshake_address = free_tcp_address();
let (client, shutdown, task) = connect_with_mock(handshake_address, 1, 4).await;
let mut stream = client.call(sample_request("req-clip", 6)).await.expect("call");
let first = stream.next().await.expect("first").expect("first ok");
assert_eq!(first.new_token_ids.len(), 4);
assert_eq!(first.finish_reason, None);
let second = stream.next().await.expect("second").expect("second ok");
assert_eq!(second.new_token_ids.len(), 2);
assert_eq!(second.finish_reason, Some(EngineCoreFinishReason::Length));
assert!(stream.next().await.is_none());
shutdown_mock(client, shutdown, task).await;
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn abort_cancels_active_request_and_emits_terminal_output() {
let handshake_address = free_tcp_address();
let (client, shutdown, task) = connect_with_mock(handshake_address, 1, 1).await;
let mut stream = client.call(sample_request("req-abort", 1_000_000)).await.expect("call");
let first = stream.next().await.expect("first").expect("first ok");
assert_eq!(first.finish_reason, None);
client.abort(&["req-abort".to_string()]).await.expect("abort");
loop {
let output = timeout(Duration::from_secs(5), stream.next())
.await
.expect("stream timeout")
.expect("terminal output")
.expect("output ok");
if output.finish_reason.is_some() {
assert_eq!(output.finish_reason, Some(EngineCoreFinishReason::Abort));
break;
}
}
shutdown_mock(client, shutdown, task).await;
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn utility_requests_return_minimal_success_responses() {
let handshake_address = free_tcp_address();
let (client, shutdown, task) = connect_with_mock(handshake_address, 1, 1).await;
assert!(!client.is_sleeping().await.expect("is sleeping"));
assert!(client.reset_prefix_cache(false, false).await.expect("reset prefix cache"));
client.reset_mm_cache().await.expect("reset mm cache");
client.reset_encoder_cache().await.expect("reset encoder cache");
shutdown_mock(client, shutdown, task).await;
}