[Rust Frontend] Introduce mock engine for benchmark baseline (#43469)
Signed-off-by: Bugen Zhao <i@bugenzhao.com>
This commit is contained in:
@@ -0,0 +1,30 @@
|
||||
[package]
|
||||
name = "vllm-mock-engine"
|
||||
version.workspace = true
|
||||
edition.workspace = true
|
||||
license.workspace = true
|
||||
|
||||
[[bin]]
|
||||
name = "vllm-mock-engine"
|
||||
path = "src/main.rs"
|
||||
|
||||
[dependencies]
|
||||
anyhow.workspace = true
|
||||
asynk-strim-attr.workspace = true
|
||||
clap.workspace = true
|
||||
futures.workspace = true
|
||||
rand.workspace = true
|
||||
rmpv.workspace = true
|
||||
serde.workspace = true
|
||||
tokio = { workspace = true, features = ["signal"] }
|
||||
tokio-util.workspace = true
|
||||
tracing.workspace = true
|
||||
tracing-subscriber.workspace = true
|
||||
vllm-engine-core-client.workspace = true
|
||||
zeromq.workspace = true
|
||||
|
||||
[dev-dependencies]
|
||||
vllm-engine-core-client = { workspace = true, features = ["test-util"] }
|
||||
|
||||
[lints]
|
||||
workspace = true
|
||||
@@ -0,0 +1,104 @@
|
||||
# vLLM Mock Engine
|
||||
|
||||
`vllm-mock-engine` is a small engine-side process for frontend stress testing. It
|
||||
joins a frontend-owned startup handshake, reports a large ready response, treats
|
||||
prefill as instant, and emits random decode tokens until each request reaches
|
||||
its `max_tokens`.
|
||||
|
||||
The frontend must own the handshake socket. Start the frontend first, then start
|
||||
the mock engine with the same handshake address.
|
||||
|
||||
## Start the mock engine
|
||||
|
||||
```bash
|
||||
cargo run -p vllm-mock-engine -- \
|
||||
--handshake-address tcp://127.0.0.1:29550 \
|
||||
--engine-count 1 \
|
||||
--output-token-chunk-size 1 \
|
||||
--vocab-size 32000 \
|
||||
--seed 0 \
|
||||
--log-requests
|
||||
```
|
||||
|
||||
Useful knobs:
|
||||
|
||||
- `--engine-count` must match the frontend's expected data-parallel engine
|
||||
count.
|
||||
- `--output-token-chunk-size` controls how many token IDs appear in one
|
||||
`EngineCoreOutput`; values greater than 1 are useful for MTP/spec-decode
|
||||
shaped frontend tests.
|
||||
- `--vocab-size` should stay within the tokenizer vocabulary of the model used
|
||||
by the frontend.
|
||||
|
||||
Stop it with Ctrl-C.
|
||||
|
||||
## Rust Frontend
|
||||
|
||||
Terminal 1:
|
||||
|
||||
```bash
|
||||
cargo run --bin vllm-rs -- serve Qwen/Qwen3-0.6B \
|
||||
--data-parallel-size 1 \
|
||||
--data-parallel-size-local 0 \
|
||||
--handshake-port 29550
|
||||
```
|
||||
|
||||
Terminal 2:
|
||||
|
||||
```bash
|
||||
cargo run -p vllm-mock-engine -- \
|
||||
--handshake-address tcp://127.0.0.1:29550
|
||||
```
|
||||
|
||||
For multiple mock engines, set both sides to the same count:
|
||||
|
||||
```bash
|
||||
cargo run --bin vllm-rs -- serve Qwen/Qwen3-0.6B \
|
||||
--data-parallel-size 4 \
|
||||
--data-parallel-size-local 0 \
|
||||
--handshake-port 29550
|
||||
|
||||
cargo run -p vllm-mock-engine -- \
|
||||
--handshake-address tcp://127.0.0.1:29550 \
|
||||
--engine-count 4
|
||||
```
|
||||
|
||||
## Python Frontend
|
||||
|
||||
Use `vllm serve` with `--data-parallel-size-local 0` so the Python process runs
|
||||
as a frontend/API server and waits for external engines on
|
||||
`--data-parallel-rpc-port`.
|
||||
|
||||
Terminal 1:
|
||||
|
||||
```bash
|
||||
vllm serve Qwen/Qwen3-0.6B \
|
||||
--data-parallel-address 127.0.0.1 \
|
||||
--data-parallel-rpc-port 29550 \
|
||||
--data-parallel-size 1 \
|
||||
--data-parallel-size-local 0
|
||||
```
|
||||
|
||||
Terminal 2:
|
||||
|
||||
```bash
|
||||
cargo run -p vllm-mock-engine -- \
|
||||
--handshake-address tcp://127.0.0.1:29550
|
||||
```
|
||||
|
||||
## Smoke Request
|
||||
|
||||
After either frontend is ready:
|
||||
|
||||
```bash
|
||||
curl http://127.0.0.1:8000/v1/chat/completions \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{
|
||||
"model": "Qwen/Qwen3-0.6B",
|
||||
"messages": [{"role": "user", "content": "hello"}],
|
||||
"max_tokens": 16,
|
||||
"stream": true
|
||||
}'
|
||||
```
|
||||
|
||||
Always pass `max_tokens`; the mock engine stops by length.
|
||||
@@ -0,0 +1,416 @@
|
||||
use std::collections::{BTreeMap, BTreeSet, HashMap};
|
||||
use std::hash::{Hash as _, Hasher as _};
|
||||
use std::time::{SystemTime, UNIX_EPOCH};
|
||||
|
||||
use anyhow::{Result, anyhow};
|
||||
use rand::rngs::StdRng;
|
||||
use rand::{Rng as _, SeedableRng as _};
|
||||
use rmpv::Value;
|
||||
use serde::Serialize;
|
||||
use tokio::sync::mpsc;
|
||||
use tokio::task::yield_now;
|
||||
use tokio_util::sync::CancellationToken;
|
||||
use tracing::{debug, info, warn};
|
||||
use vllm_engine_core_client::protocol::utility::{
|
||||
EngineCoreUtilityRequest, UtilityOutput, UtilityResultEnvelope,
|
||||
};
|
||||
use vllm_engine_core_client::protocol::{
|
||||
EngineCoreFinishReason, EngineCoreOutput, EngineCoreOutputs, EngineCoreRequest,
|
||||
};
|
||||
|
||||
use super::Opt;
|
||||
|
||||
/// Derive a stable per-request seed from the CLI seed, engine, and request id.
|
||||
fn request_seed(base_seed: u64, engine_index: u32, request_id: &str) -> u64 {
|
||||
let mut hasher = std::hash::DefaultHasher::new();
|
||||
base_seed.hash(&mut hasher);
|
||||
engine_index.hash(&mut hasher);
|
||||
request_id.hash(&mut hasher);
|
||||
hasher.finish()
|
||||
}
|
||||
|
||||
/// Current UNIX timestamp in seconds for engine-core output envelopes.
|
||||
fn now_secs() -> f64 {
|
||||
SystemTime::now()
|
||||
.duration_since(UNIX_EPOCH)
|
||||
.map(|duration| duration.as_secs_f64())
|
||||
.unwrap_or_default()
|
||||
}
|
||||
|
||||
/// Build one request output with only token IDs and terminal status populated.
|
||||
fn request_output(
|
||||
request_id: String,
|
||||
new_token_ids: Vec<u32>,
|
||||
finish_reason: Option<EngineCoreFinishReason>,
|
||||
) -> EngineCoreOutput {
|
||||
EngineCoreOutput {
|
||||
request_id,
|
||||
new_token_ids,
|
||||
finish_reason,
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
|
||||
/// Produce an empty output with a terminal finish reason for an invalid request.
|
||||
fn empty_finish_outputs(
|
||||
engine_index: u32,
|
||||
request_id: String,
|
||||
finish_reason: EngineCoreFinishReason,
|
||||
) -> EngineCoreOutputs {
|
||||
let output = request_output(request_id, Vec::new(), Some(finish_reason));
|
||||
let finished_requests = BTreeSet::from([output.request_id.clone()]);
|
||||
|
||||
EngineCoreOutputs {
|
||||
engine_index,
|
||||
outputs: vec![output],
|
||||
timestamp: now_secs(),
|
||||
finished_requests: Some(finished_requests),
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
|
||||
/// Encode a utility result into the protocol's msgpack value envelope.
|
||||
fn utility_envelope<T>(value: T) -> Result<UtilityResultEnvelope>
|
||||
where
|
||||
T: Serialize,
|
||||
{
|
||||
Ok(UtilityResultEnvelope::without_type_info(
|
||||
rmpv::ext::to_value(value)?,
|
||||
))
|
||||
}
|
||||
|
||||
/// Produce the minimal utility responses needed by the Rust frontend.
|
||||
fn utility_response(
|
||||
engine_index: u32,
|
||||
request: EngineCoreUtilityRequest,
|
||||
) -> Result<EngineCoreOutputs> {
|
||||
let result = match request.method_name.as_str() {
|
||||
"get_supported_tasks" => utility_envelope(vec!["generate"]),
|
||||
"is_sleeping" => utility_envelope(false),
|
||||
"reset_prefix_cache" => utility_envelope(true),
|
||||
"reset_mm_cache"
|
||||
| "reset_encoder_cache"
|
||||
| "profile"
|
||||
| "sleep"
|
||||
| "wake_up"
|
||||
| "execute_dummy_batch" => utility_envelope(()),
|
||||
_ => utility_envelope(Value::Nil),
|
||||
}?;
|
||||
|
||||
Ok(EngineCoreOutputs {
|
||||
engine_index,
|
||||
utility_output: Some(UtilityOutput {
|
||||
call_id: request.call_id,
|
||||
failure_message: None,
|
||||
result: Some(result),
|
||||
}),
|
||||
timestamp: now_secs(),
|
||||
..Default::default()
|
||||
})
|
||||
}
|
||||
|
||||
/// Message sent from the frontend to the mock engine task to drive the engine loop.
|
||||
pub(crate) enum EngineInput {
|
||||
Request(Box<EngineCoreRequest>),
|
||||
Abort(Vec<String>),
|
||||
Utility(EngineCoreUtilityRequest),
|
||||
StartDpWave,
|
||||
}
|
||||
|
||||
/// Message sent from the mock engine task to the frontend for one engine output batch.
|
||||
pub(crate) struct EngineOutput {
|
||||
pub client_index: u32,
|
||||
pub outputs: EngineCoreOutputs,
|
||||
}
|
||||
|
||||
/// Per-request decode state owned by one mock engine.
|
||||
#[derive(Debug)]
|
||||
struct ActiveRequest {
|
||||
request_id: String,
|
||||
client_index: u32,
|
||||
prompt_len: usize,
|
||||
max_tokens: usize,
|
||||
generated: usize,
|
||||
rng: StdRng,
|
||||
}
|
||||
|
||||
impl ActiveRequest {
|
||||
/// Create a new active request from an incoming EngineCoreRequest, or return an immediate
|
||||
/// finish reason if the request is invalid.
|
||||
fn new(
|
||||
engine_index: u32,
|
||||
request: Box<EngineCoreRequest>,
|
||||
opt: &Opt,
|
||||
) -> Result<Self, EngineCoreFinishReason> {
|
||||
let request_id = request.request_id;
|
||||
let client_index = request.client_index;
|
||||
let prompt_len = request.prompt_token_ids.as_ref().map(Vec::len).unwrap_or_default();
|
||||
|
||||
let Some(sampling_params) = request.sampling_params else {
|
||||
warn!(
|
||||
request_id,
|
||||
"request has no sampling params; returning engine error"
|
||||
);
|
||||
return Err(EngineCoreFinishReason::Error);
|
||||
};
|
||||
let max_tokens = sampling_params.max_tokens as usize;
|
||||
|
||||
if opt.log_requests {
|
||||
info!(
|
||||
request_id,
|
||||
prompt_len,
|
||||
max_tokens,
|
||||
chunk_size = opt.output_token_chunk_size,
|
||||
"mock request started"
|
||||
);
|
||||
}
|
||||
|
||||
if max_tokens == 0 {
|
||||
return Err(EngineCoreFinishReason::Length);
|
||||
}
|
||||
|
||||
Ok(ActiveRequest {
|
||||
rng: StdRng::seed_from_u64(request_seed(opt.seed, engine_index, &request_id)),
|
||||
request_id,
|
||||
client_index,
|
||||
prompt_len,
|
||||
max_tokens,
|
||||
generated: 0,
|
||||
})
|
||||
}
|
||||
|
||||
/// Advance this request by one mock engine step.
|
||||
fn step(&mut self, opt: &Opt) -> EngineCoreOutput {
|
||||
let remaining = self.max_tokens - self.generated;
|
||||
let chunk_len = remaining.min(opt.output_token_chunk_size);
|
||||
let mut new_token_ids = Vec::with_capacity(chunk_len);
|
||||
for _ in 0..chunk_len {
|
||||
new_token_ids.push(self.rng.random_range(0..opt.vocab_size));
|
||||
}
|
||||
self.generated += chunk_len;
|
||||
|
||||
let finished = self.generated >= self.max_tokens;
|
||||
request_output(
|
||||
self.request_id.clone(),
|
||||
new_token_ids,
|
||||
finished.then_some(EngineCoreFinishReason::Length),
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
/// Internal state for one mock engine instance, owned by the engine loop task.
|
||||
struct Engine {
|
||||
engine_index: u32,
|
||||
opt: Opt,
|
||||
active_requests: HashMap<String, ActiveRequest>,
|
||||
}
|
||||
|
||||
impl Engine {
|
||||
/// Drain one frontend request message received on the input DEALER socket.
|
||||
fn handle_input(&mut self, input: EngineInput) -> Result<Vec<EngineOutput>> {
|
||||
let mut outputs = Vec::new();
|
||||
|
||||
match input {
|
||||
EngineInput::Request(request) => {
|
||||
let request_id = request.request_id.clone();
|
||||
let client_index = request.client_index;
|
||||
|
||||
if self.active_requests.contains_key(&request_id) {
|
||||
warn!(
|
||||
engine_index = self.engine_index,
|
||||
request_id, "duplicate mock request id"
|
||||
);
|
||||
return Ok(vec![EngineOutput {
|
||||
client_index,
|
||||
outputs: empty_finish_outputs(
|
||||
self.engine_index,
|
||||
request_id,
|
||||
EngineCoreFinishReason::Error,
|
||||
),
|
||||
}]);
|
||||
}
|
||||
|
||||
match ActiveRequest::new(self.engine_index, request, &self.opt) {
|
||||
Ok(request) => {
|
||||
self.active_requests.insert(request_id, request);
|
||||
}
|
||||
Err(finish_reason) => {
|
||||
return Ok(vec![EngineOutput {
|
||||
client_index,
|
||||
outputs: empty_finish_outputs(
|
||||
self.engine_index,
|
||||
request_id,
|
||||
finish_reason,
|
||||
),
|
||||
}]);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
EngineInput::Abort(request_ids) => {
|
||||
let mut outputs_by_client =
|
||||
BTreeMap::<u32, (Vec<EngineCoreOutput>, BTreeSet<String>)>::new();
|
||||
for request_id in request_ids {
|
||||
if let Some(request) = self.active_requests.remove(&request_id) {
|
||||
let output = request_output(
|
||||
request_id.clone(),
|
||||
Vec::new(),
|
||||
Some(EngineCoreFinishReason::Abort),
|
||||
);
|
||||
let (outputs, finished_requests) = outputs_by_client
|
||||
.entry(request.client_index)
|
||||
.or_insert_with(|| (Vec::new(), BTreeSet::new()));
|
||||
outputs.push(output);
|
||||
finished_requests.insert(request_id.clone());
|
||||
if self.opt.log_requests {
|
||||
info!(request_id, finish_reason = "abort", "mock request aborted");
|
||||
}
|
||||
}
|
||||
}
|
||||
for (client_index, (client_outputs, finished_requests)) in outputs_by_client {
|
||||
outputs.push({
|
||||
let outputs = EngineCoreOutputs {
|
||||
engine_index: self.engine_index,
|
||||
outputs: client_outputs,
|
||||
timestamp: now_secs(),
|
||||
finished_requests: Some(finished_requests),
|
||||
..Default::default()
|
||||
};
|
||||
EngineOutput {
|
||||
client_index,
|
||||
outputs,
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
EngineInput::Utility(request) => {
|
||||
debug!(
|
||||
engine_index = self.engine_index,
|
||||
call_id = %request.call_id,
|
||||
method = request.method_name,
|
||||
"mock utility request"
|
||||
);
|
||||
let client_index = request.client_index;
|
||||
outputs.push({
|
||||
let outputs = utility_response(self.engine_index, request)?;
|
||||
EngineOutput {
|
||||
client_index,
|
||||
outputs,
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
EngineInput::StartDpWave => {
|
||||
debug!(
|
||||
engine_index = self.engine_index,
|
||||
"ignoring START_DP_WAVE in mock engine"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
Ok(outputs)
|
||||
}
|
||||
|
||||
/// Advance active requests once and return one batched engine output.
|
||||
fn step(&mut self) -> Vec<EngineOutput> {
|
||||
if self.active_requests.is_empty() {
|
||||
return Vec::new();
|
||||
}
|
||||
|
||||
let mut outputs_by_client =
|
||||
BTreeMap::<u32, (Vec<EngineCoreOutput>, BTreeSet<String>)>::new();
|
||||
let mut all_finished_requests = BTreeSet::new();
|
||||
|
||||
for request in self.active_requests.values_mut() {
|
||||
let client_index = request.client_index;
|
||||
let output = request.step(&self.opt);
|
||||
let request_id = request.request_id.clone();
|
||||
let finished = output.finished();
|
||||
if output.finished() {
|
||||
all_finished_requests.insert(request_id.clone());
|
||||
if self.opt.log_requests {
|
||||
info!(
|
||||
request_id,
|
||||
prompt_len = request.prompt_len,
|
||||
output_tokens = request.generated,
|
||||
finish_reason = "length",
|
||||
"mock request finished"
|
||||
);
|
||||
}
|
||||
}
|
||||
let (outputs, finished_requests) = outputs_by_client
|
||||
.entry(client_index)
|
||||
.or_insert_with(|| (Vec::new(), BTreeSet::new()));
|
||||
if finished {
|
||||
finished_requests.insert(request_id.clone());
|
||||
}
|
||||
outputs.push(output);
|
||||
}
|
||||
|
||||
for request_id in &all_finished_requests {
|
||||
self.active_requests.remove(request_id);
|
||||
}
|
||||
|
||||
outputs_by_client
|
||||
.into_iter()
|
||||
.filter_map(|(client_index, (outputs, finished_requests))| {
|
||||
(!outputs.is_empty()).then(|| EngineOutput {
|
||||
client_index,
|
||||
outputs: EngineCoreOutputs {
|
||||
engine_index: self.engine_index,
|
||||
outputs,
|
||||
timestamp: now_secs(),
|
||||
finished_requests: (!finished_requests.is_empty())
|
||||
.then_some(finished_requests),
|
||||
..Default::default()
|
||||
},
|
||||
})
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
}
|
||||
|
||||
/// Run the main loop for the mock engine, receiving `EngineInput` from `input_rx`
|
||||
/// and sending `EngineOutput` to `output_tx` until `shutdown` is cancelled.
|
||||
pub(crate) async fn run_engine_loop(
|
||||
engine_index: u32,
|
||||
opt: Opt,
|
||||
mut input_rx: mpsc::UnboundedReceiver<EngineInput>,
|
||||
output_tx: mpsc::Sender<EngineOutput>,
|
||||
shutdown: CancellationToken,
|
||||
) -> Result<()> {
|
||||
let mut engine = Engine {
|
||||
engine_index,
|
||||
opt,
|
||||
active_requests: HashMap::new(),
|
||||
};
|
||||
|
||||
loop {
|
||||
let outputs = tokio::select! {
|
||||
biased;
|
||||
_ = shutdown.cancelled() => break,
|
||||
|
||||
input = input_rx.recv() => {
|
||||
let input = input
|
||||
.ok_or_else(|| anyhow!("mock engine input channel closed"))?;
|
||||
engine.handle_input(input)?
|
||||
}
|
||||
|
||||
// If there are active requests, step them once after yielding to the scheduler to
|
||||
// avoid blocking the engine loop while still making steady progress on request outputs.
|
||||
_ = yield_now(), if !engine.active_requests.is_empty() => {
|
||||
engine.step()
|
||||
}
|
||||
};
|
||||
|
||||
for output in outputs {
|
||||
output_tx
|
||||
.send(output)
|
||||
.await
|
||||
.map_err(|_| anyhow!("mock engine IO task shut down"))?;
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
@@ -0,0 +1,121 @@
|
||||
use anyhow::{Context as _, Result, anyhow, bail};
|
||||
use futures::{Stream, StreamExt as _, stream};
|
||||
use tokio::sync::mpsc;
|
||||
use tokio_util::sync::CancellationToken;
|
||||
use tracing::warn;
|
||||
use vllm_engine_core_client::mock_engine::MockEngineDataSockets;
|
||||
use vllm_engine_core_client::protocol::utility::EngineCoreUtilityRequest;
|
||||
use vllm_engine_core_client::protocol::{
|
||||
EngineCoreRequest, EngineCoreRequestType, decode_msgpack, encode_msgpack,
|
||||
};
|
||||
use zeromq::{DealerSocket, PushSocket, SocketRecv as _, SocketSend as _, ZmqMessage};
|
||||
|
||||
use crate::engine::{EngineInput, EngineOutput};
|
||||
|
||||
/// Send one engine output batch to the client over the appropriate push socket.
|
||||
async fn send_engine_outputs_to_client(
|
||||
push_sockets: &mut [PushSocket],
|
||||
EngineOutput {
|
||||
client_index,
|
||||
outputs,
|
||||
}: EngineOutput,
|
||||
) -> Result<()> {
|
||||
let message = ZmqMessage::from(encode_msgpack(&outputs)?);
|
||||
push_sockets[client_index as usize].send(message).await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Create a stream of `EngineInput` by continuously receiving messages from the given dealer socket
|
||||
/// and decoding them into `EngineInput`.
|
||||
fn dealer_input_stream(dealer: DealerSocket) -> impl Stream<Item = Result<EngineInput>> {
|
||||
stream::unfold(dealer, |mut dealer| async {
|
||||
let input = loop {
|
||||
let message =
|
||||
match dealer.recv().await.context("failed to receive message from dealer socket") {
|
||||
Ok(message) => message,
|
||||
Err(err) => break Err(err),
|
||||
};
|
||||
|
||||
match decode_request(message) {
|
||||
Ok(input) => break Ok(input),
|
||||
Err(err) => {
|
||||
warn!(%err, "failed to decode engine request message; ignoring");
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
Some((input, dealer))
|
||||
})
|
||||
}
|
||||
|
||||
/// Decode a `ZmqMessage` into an `EngineInput`. Returns an error if the message is malformed or
|
||||
/// contains an unknown/unsupported request type.
|
||||
fn decode_request(message: ZmqMessage) -> Result<EngineInput> {
|
||||
let frames = message.into_vec();
|
||||
if frames.is_empty() {
|
||||
bail!("empty engine request message");
|
||||
}
|
||||
if frames.len() != 2 {
|
||||
bail!("invalid frame count for engine request: {}", frames.len());
|
||||
}
|
||||
|
||||
let request_type_frame = frames[0].as_ref();
|
||||
let Some(request_type) = EngineCoreRequestType::from_frame(request_type_frame) else {
|
||||
bail!("unknown engine request type: {:?}", request_type_frame);
|
||||
};
|
||||
|
||||
let input = match request_type {
|
||||
EngineCoreRequestType::Add => {
|
||||
let request: Box<EngineCoreRequest> = decode_msgpack(frames[1].as_ref())?;
|
||||
EngineInput::Request(request)
|
||||
}
|
||||
EngineCoreRequestType::Abort => {
|
||||
let request_ids: Vec<String> = decode_msgpack(frames[1].as_ref())?;
|
||||
EngineInput::Abort(request_ids)
|
||||
}
|
||||
EngineCoreRequestType::Utility => {
|
||||
let request: EngineCoreUtilityRequest = decode_msgpack(frames[1].as_ref())?;
|
||||
EngineInput::Utility(request)
|
||||
}
|
||||
EngineCoreRequestType::StartDpWave => EngineInput::StartDpWave,
|
||||
};
|
||||
|
||||
Ok(input)
|
||||
}
|
||||
|
||||
/// Run the main IO loop for the mock engine, continuously receiving and decoding raw messages from
|
||||
/// the dealer sockets, sending them to the engine loop task via `input_tx`, and receiving
|
||||
/// `EngineOutput` from the engine loop task via `output_rx` and sending them to the client over the
|
||||
/// appropriate push socket, until `shutdown` is cancelled.
|
||||
pub(crate) async fn run_io_loop(
|
||||
data_sockets: Vec<MockEngineDataSockets>,
|
||||
input_tx: mpsc::UnboundedSender<EngineInput>,
|
||||
mut output_rx: mpsc::Receiver<EngineOutput>,
|
||||
shutdown: CancellationToken,
|
||||
) -> Result<()> {
|
||||
let (dealers, mut push_sockets): (Vec<_>, Vec<_>) =
|
||||
data_sockets.into_iter().map(|sockets| (sockets.dealer, sockets.push)).unzip();
|
||||
let mut input_streams =
|
||||
stream::select_all(dealers.into_iter().map(dealer_input_stream).map(Box::pin));
|
||||
|
||||
loop {
|
||||
tokio::select! {
|
||||
biased;
|
||||
_ = shutdown.cancelled() => return Ok(()),
|
||||
|
||||
output = output_rx.recv() => {
|
||||
let output = output
|
||||
.ok_or_else(|| anyhow!("mock engine output channel closed"))?;
|
||||
send_engine_outputs_to_client(&mut push_sockets, output).await?;
|
||||
}
|
||||
|
||||
input = input_streams.next() => {
|
||||
let input = input
|
||||
.ok_or_else(|| anyhow!("mock engine input streams closed"))??;
|
||||
input_tx
|
||||
.send(input)
|
||||
.map_err(|_| anyhow!("mock engine state task shut down"))?;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,138 @@
|
||||
use anyhow::{Context, Result, bail};
|
||||
use clap::Parser;
|
||||
use tokio::sync::mpsc;
|
||||
use tokio::task::JoinSet;
|
||||
use tokio_util::sync::CancellationToken;
|
||||
use tracing::{error, info};
|
||||
use vllm_engine_core_client::EngineId;
|
||||
use vllm_engine_core_client::mock_engine::{
|
||||
MockEngineConfig, MockEngineSockets, connect_to_frontend,
|
||||
};
|
||||
|
||||
pub mod engine;
|
||||
pub mod io;
|
||||
|
||||
/// Standalone engine-core protocol emulator for frontend stress testing.
|
||||
#[derive(Debug, Clone, Parser)]
|
||||
#[command(
|
||||
name = "vllm-mock-engine",
|
||||
about = "Run a mock vLLM headless engine for Rust frontend stress testing."
|
||||
)]
|
||||
pub struct Opt {
|
||||
/// Frontend-owned ZMQ handshake address.
|
||||
#[arg(long, default_value = "tcp://127.0.0.1:29550")]
|
||||
pub handshake_address: String,
|
||||
|
||||
/// Number of mock engine identities to register with the frontend.
|
||||
#[arg(long, default_value_t = 1)]
|
||||
pub engine_count: usize,
|
||||
|
||||
/// Number of accepted output tokens included in each EngineCoreOutput.
|
||||
#[arg(long, default_value_t = 1)]
|
||||
pub output_token_chunk_size: usize,
|
||||
|
||||
/// Random token IDs are sampled uniformly from 0..vocab_size.
|
||||
#[arg(long, default_value_t = 32_000)]
|
||||
pub vocab_size: u32,
|
||||
|
||||
/// Base seed for deterministic random token generation.
|
||||
#[arg(long, default_value_t = 0)]
|
||||
pub seed: u64,
|
||||
|
||||
/// Log a summary line for each request.
|
||||
#[arg(long)]
|
||||
pub log_requests: bool,
|
||||
}
|
||||
|
||||
/// Run one mock engine until shutdown or transport failure.
|
||||
async fn run_engine(engine_index: u32, opt: Opt, shutdown: CancellationToken) -> Result<()> {
|
||||
let MockEngineSockets { data_sockets, .. } = connect_to_frontend(
|
||||
&opt.handshake_address,
|
||||
EngineId::from_engine_index(engine_index),
|
||||
MockEngineConfig::default(),
|
||||
)
|
||||
.await
|
||||
.with_context(|| format!("mock engine {engine_index} failed to connect to frontend"))?;
|
||||
|
||||
info!(engine_index, "mock engine connected to frontend");
|
||||
|
||||
let (input_tx, input_rx) = mpsc::unbounded_channel();
|
||||
let (output_tx, output_rx) = mpsc::channel(64);
|
||||
|
||||
// IO loop: dealer -> input_tx, output_rx -> push
|
||||
let mut io_loop = tokio::spawn(io::run_io_loop(
|
||||
data_sockets,
|
||||
input_tx,
|
||||
output_rx,
|
||||
shutdown.clone(),
|
||||
));
|
||||
// Engine loop: input_rx -> engine logic -> output_tx
|
||||
let mut engine_loop = tokio::spawn(engine::run_engine_loop(
|
||||
engine_index,
|
||||
opt,
|
||||
input_rx,
|
||||
output_tx,
|
||||
shutdown.clone(),
|
||||
));
|
||||
|
||||
tokio::select! {
|
||||
biased;
|
||||
_ = shutdown.cancelled() => {
|
||||
io_loop.abort();
|
||||
engine_loop.abort();
|
||||
io_loop.await.ok();
|
||||
engine_loop.await.ok();
|
||||
}
|
||||
|
||||
result = &mut io_loop => {
|
||||
error!(engine_index, "mock engine IO loop exited unexpectedly");
|
||||
engine_loop.abort();
|
||||
engine_loop.await.ok();
|
||||
result??;
|
||||
}
|
||||
result = &mut engine_loop => {
|
||||
error!(engine_index, "mock engine loop exited unexpectedly");
|
||||
io_loop.abort();
|
||||
io_loop.await.ok();
|
||||
result??;
|
||||
}
|
||||
}
|
||||
|
||||
info!(engine_index, "mock engine shut down");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Run all requested mock engines until cancellation or one engine task fails.
|
||||
pub async fn run(opt: Opt, shutdown: CancellationToken) -> Result<()> {
|
||||
info!(?opt, "starting mock engine");
|
||||
|
||||
let mut engines = JoinSet::new();
|
||||
for engine_index in 0..opt.engine_count {
|
||||
engines.spawn(run_engine(
|
||||
engine_index as u32,
|
||||
opt.clone(),
|
||||
shutdown.clone(),
|
||||
));
|
||||
}
|
||||
|
||||
tokio::select! {
|
||||
biased;
|
||||
_ = shutdown.cancelled() => {
|
||||
engines.abort_all();
|
||||
while engines.join_next().await.is_some() {}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
joined = engines.join_next() => {
|
||||
match joined {
|
||||
Some(Ok(Ok(()))) => bail!("mock engine exited unexpectedly"),
|
||||
Some(Ok(Err(error))) => Err(error),
|
||||
Some(Err(error)) => Err(error).context("mock engine task join failed"),
|
||||
None => Ok(()),
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests;
|
||||
@@ -0,0 +1,38 @@
|
||||
use anyhow::{Context, Result};
|
||||
use clap::Parser as _;
|
||||
use tokio_util::sync::CancellationToken;
|
||||
use tracing::{Level, info};
|
||||
use vllm_mock_engine::Opt;
|
||||
|
||||
fn init_tracing() {
|
||||
tracing_subscriber::fmt().with_max_level(Level::INFO).init();
|
||||
}
|
||||
|
||||
/// Create a cancellation token that is triggered by Ctrl-C.
|
||||
fn shutdown_signal() -> CancellationToken {
|
||||
let token = CancellationToken::new();
|
||||
let shutdown = token.clone();
|
||||
|
||||
tokio::spawn(async move {
|
||||
tokio::signal::ctrl_c().await.expect("failed to install Ctrl-C signal handler");
|
||||
info!("received shutdown signal (Ctrl-C), shutting down...");
|
||||
shutdown.cancel();
|
||||
});
|
||||
|
||||
token
|
||||
}
|
||||
|
||||
fn main() -> Result<()> {
|
||||
init_tracing();
|
||||
let opt = Opt::parse();
|
||||
|
||||
let runtime = tokio::runtime::Builder::new_multi_thread()
|
||||
.enable_all()
|
||||
.build()
|
||||
.context("failed to build Tokio runtime")?;
|
||||
|
||||
runtime.block_on(async move {
|
||||
let shutdown = shutdown_signal();
|
||||
vllm_mock_engine::run(opt, shutdown).await
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,197 @@
|
||||
use std::net::TcpListener;
|
||||
use std::time::Duration;
|
||||
|
||||
use anyhow::Result;
|
||||
use futures::StreamExt as _;
|
||||
use tokio::time::timeout;
|
||||
use tokio_util::sync::CancellationToken;
|
||||
use vllm_engine_core_client::protocol::{
|
||||
EngineCoreFinishReason, EngineCoreRequest, EngineCoreSamplingParams,
|
||||
};
|
||||
use vllm_engine_core_client::test_utils::IpcNamespace;
|
||||
use vllm_engine_core_client::{EngineCoreClient, EngineCoreClientConfig, TransportMode};
|
||||
|
||||
use crate::{Opt, run};
|
||||
|
||||
fn free_tcp_address() -> String {
|
||||
let listener = TcpListener::bind("127.0.0.1:0").expect("bind free port");
|
||||
let port = listener.local_addr().expect("local addr").port();
|
||||
drop(listener);
|
||||
format!("tcp://127.0.0.1:{port}")
|
||||
}
|
||||
|
||||
fn client_config(handshake_address: String, engine_count: usize) -> EngineCoreClientConfig {
|
||||
EngineCoreClientConfig {
|
||||
transport_mode: TransportMode::HandshakeOwner {
|
||||
handshake_address,
|
||||
advertised_host: "127.0.0.1".to_string(),
|
||||
engine_count,
|
||||
ready_timeout: Duration::from_secs(5),
|
||||
local_input_address: None,
|
||||
local_output_address: None,
|
||||
},
|
||||
coordinator_mode: None,
|
||||
model_name: "mock-model".to_string(),
|
||||
client_index: 0,
|
||||
}
|
||||
}
|
||||
|
||||
async fn connect_with_mock(
|
||||
handshake_address: String,
|
||||
engine_count: usize,
|
||||
output_token_chunk_size: usize,
|
||||
) -> (
|
||||
EngineCoreClient,
|
||||
CancellationToken,
|
||||
tokio::task::JoinHandle<Result<()>>,
|
||||
) {
|
||||
let shutdown = CancellationToken::new();
|
||||
let task = tokio::spawn(run(
|
||||
Opt {
|
||||
handshake_address: handshake_address.clone(),
|
||||
engine_count,
|
||||
output_token_chunk_size,
|
||||
vocab_size: 32_000,
|
||||
seed: 0,
|
||||
log_requests: false,
|
||||
},
|
||||
shutdown.clone(),
|
||||
));
|
||||
|
||||
let client = timeout(
|
||||
Duration::from_secs(5),
|
||||
EngineCoreClient::connect(client_config(handshake_address, engine_count)),
|
||||
)
|
||||
.await
|
||||
.expect("client connect timeout")
|
||||
.expect("connect client");
|
||||
|
||||
(client, shutdown, task)
|
||||
}
|
||||
|
||||
fn sample_request(request_id: &str, max_tokens: u32) -> EngineCoreRequest {
|
||||
EngineCoreRequest {
|
||||
request_id: request_id.to_string(),
|
||||
prompt_token_ids: Some(vec![1, 2, 3]),
|
||||
sampling_params: Some(EngineCoreSamplingParams {
|
||||
max_tokens,
|
||||
..EngineCoreSamplingParams::for_test()
|
||||
}),
|
||||
arrival_time: 0.0,
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
|
||||
async fn shutdown_mock(
|
||||
client: EngineCoreClient,
|
||||
shutdown: CancellationToken,
|
||||
task: tokio::task::JoinHandle<Result<()>>,
|
||||
) {
|
||||
client.shutdown().await.expect("client shutdown");
|
||||
shutdown.cancel();
|
||||
task.await.expect("mock join").expect("mock run");
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
async fn mock_engine_connects_over_tcp() {
|
||||
let handshake_address = free_tcp_address();
|
||||
let (client, shutdown, task) = connect_with_mock(handshake_address, 1, 1).await;
|
||||
assert_eq!(client.engine_count(), 1);
|
||||
assert_eq!(client.engine_identities()[0], &[0, 0]);
|
||||
assert_eq!(client.max_model_len(), Some(1024 * 1024));
|
||||
shutdown_mock(client, shutdown, task).await;
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
async fn mock_engine_connects_over_ipc() {
|
||||
let ipc = IpcNamespace::new().expect("ipc namespace");
|
||||
let handshake_address = ipc.handshake_endpoint();
|
||||
let (client, shutdown, task) = connect_with_mock(handshake_address, 1, 1).await;
|
||||
assert_eq!(client.engine_count(), 1);
|
||||
assert_eq!(client.engine_identities()[0], &[0, 0]);
|
||||
shutdown_mock(client, shutdown, task).await;
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
async fn mock_engine_registers_multiple_identities() {
|
||||
let handshake_address = free_tcp_address();
|
||||
let (client, shutdown, task) = connect_with_mock(handshake_address, 2, 1).await;
|
||||
assert_eq!(client.engine_count(), 2);
|
||||
assert_eq!(client.engine_identities(), vec![&[0, 0][..], &[1, 0][..]]);
|
||||
shutdown_mock(client, shutdown, task).await;
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
async fn chunk_size_one_outputs_one_token_per_update() {
|
||||
let handshake_address = free_tcp_address();
|
||||
let (client, shutdown, task) = connect_with_mock(handshake_address, 1, 1).await;
|
||||
let mut stream = client.call(sample_request("req-1", 3)).await.expect("call");
|
||||
|
||||
let first = stream.next().await.expect("first").expect("first ok");
|
||||
assert_eq!(first.new_token_ids.len(), 1);
|
||||
assert_eq!(first.finish_reason, None);
|
||||
let second = stream.next().await.expect("second").expect("second ok");
|
||||
assert_eq!(second.new_token_ids.len(), 1);
|
||||
assert_eq!(second.finish_reason, None);
|
||||
let third = stream.next().await.expect("third").expect("third ok");
|
||||
assert_eq!(third.new_token_ids.len(), 1);
|
||||
assert_eq!(third.finish_reason, Some(EngineCoreFinishReason::Length));
|
||||
assert!(stream.next().await.is_none());
|
||||
|
||||
shutdown_mock(client, shutdown, task).await;
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
async fn chunk_size_clips_final_output_to_max_tokens() {
|
||||
let handshake_address = free_tcp_address();
|
||||
let (client, shutdown, task) = connect_with_mock(handshake_address, 1, 4).await;
|
||||
let mut stream = client.call(sample_request("req-clip", 6)).await.expect("call");
|
||||
|
||||
let first = stream.next().await.expect("first").expect("first ok");
|
||||
assert_eq!(first.new_token_ids.len(), 4);
|
||||
assert_eq!(first.finish_reason, None);
|
||||
let second = stream.next().await.expect("second").expect("second ok");
|
||||
assert_eq!(second.new_token_ids.len(), 2);
|
||||
assert_eq!(second.finish_reason, Some(EngineCoreFinishReason::Length));
|
||||
assert!(stream.next().await.is_none());
|
||||
|
||||
shutdown_mock(client, shutdown, task).await;
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
async fn abort_cancels_active_request_and_emits_terminal_output() {
|
||||
let handshake_address = free_tcp_address();
|
||||
let (client, shutdown, task) = connect_with_mock(handshake_address, 1, 1).await;
|
||||
let mut stream = client.call(sample_request("req-abort", 1_000_000)).await.expect("call");
|
||||
let first = stream.next().await.expect("first").expect("first ok");
|
||||
assert_eq!(first.finish_reason, None);
|
||||
|
||||
client.abort(&["req-abort".to_string()]).await.expect("abort");
|
||||
|
||||
loop {
|
||||
let output = timeout(Duration::from_secs(5), stream.next())
|
||||
.await
|
||||
.expect("stream timeout")
|
||||
.expect("terminal output")
|
||||
.expect("output ok");
|
||||
if output.finish_reason.is_some() {
|
||||
assert_eq!(output.finish_reason, Some(EngineCoreFinishReason::Abort));
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
shutdown_mock(client, shutdown, task).await;
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
async fn utility_requests_return_minimal_success_responses() {
|
||||
let handshake_address = free_tcp_address();
|
||||
let (client, shutdown, task) = connect_with_mock(handshake_address, 1, 1).await;
|
||||
|
||||
assert!(!client.is_sleeping().await.expect("is sleeping"));
|
||||
assert!(client.reset_prefix_cache(false, false).await.expect("reset prefix cache"));
|
||||
client.reset_mm_cache().await.expect("reset mm cache");
|
||||
client.reset_encoder_cache().await.expect("reset encoder cache");
|
||||
|
||||
shutdown_mock(client, shutdown, task).await;
|
||||
}
|
||||
Reference in New Issue
Block a user