[Rust Frontend] Move code from vllm-frontend-rs (#43283)

Signed-off-by: Bugen Zhao <i@bugenzhao.com>
Signed-off-by: Nick Hill <nickhill123@gmail.com>
Signed-off-by: Eric Curtin <eric.curtin@docker.com>
Signed-off-by: Dev-X25874 <283057883+Dev-X25874@users.noreply.github.com>
Signed-off-by: Will.hou <1205157517@qq.com>
Signed-off-by: Will.hou <willamhou@ceresman.com>
Co-authored-by: Nick Hill <nickhill123@gmail.com>
Co-authored-by: Eric Curtin <eric.curtin@docker.com>
Co-authored-by: Dev-X25874 <283057883+Dev-X25874@users.noreply.github.com>
Co-authored-by: Will.hou <1205157517@qq.com>
Co-authored-by: Will.hou <willamhou@ceresman.com>

Please see https://github.com/Inferact/vllm-frontend-rs for full original commit history.
This commit is contained in:
Bugen Zhao
2026-05-21 17:21:48 -07:00
committed by GitHub
co-authored by Nick Hill Eric Curtin Dev-X25874 Will.hou Will.hou
parent 39d5fa96a7
commit 39910f2b25
284 changed files with 72569 additions and 90 deletions
+57
View File
@@ -0,0 +1,57 @@
[package]
name = "vllm-server"
version.workspace = true
edition.workspace = true
license.workspace = true
[dependencies]
anyhow.workspace = true
asynk-strim-attr.workspace = true
axum.workspace = true
futures.workspace = true
http-body.workspace = true
itertools.workspace = true
libc.workspace = true
llm-multimodal.workspace = true
prost.workspace = true
prost-types.workspace = true
rmpv.workspace = true
serde.workspace = true
serde_json.workspace = true
serde_with.workspace = true
socket2.workspace = true
thiserror-ext.workspace = true
tokio.workspace = true
tokio-stream.workspace = true
tokio-util.workspace = true
tonic.workspace = true
tonic-prost.workspace = true
tower-http.workspace = true
tracing.workspace = true
tracing-futures.workspace = true
tracing-subscriber.workspace = true
uuid.workspace = true
validator.workspace = true
vllm-chat.workspace = true
vllm-engine-core-client.workspace = true
vllm-llm.workspace = true
vllm-metrics.workspace = true
vllm-text.workspace = true
[build-dependencies]
tonic-prost-build.workspace = true
[dev-dependencies]
anyhow.workspace = true
async-openai = { workspace = true, features = ["full"] }
bytes.workspace = true
clap.workspace = true
expect-test.workspace = true
rmp-serde.workspace = true
serial_test.workspace = true
tower.workspace = true
vllm-engine-core-client = { workspace = true, features = ["test-util"] }
zeromq.workspace = true
[lints]
workspace = true
+12
View File
@@ -0,0 +1,12 @@
fn main() -> Result<(), Box<dyn std::error::Error>> {
let manifest_dir = std::env::var("CARGO_MANIFEST_DIR").unwrap();
let proto_dir = format!("{manifest_dir}/../../proto");
tonic_prost_build::configure()
.build_server(true)
.build_client(true)
.protoc_arg("--experimental_allow_proto3_optional") // be compatible with old compilers
.compile_protos(&[format!("{proto_dir}/vllm_grpc.proto")], &[proto_dir])?;
Ok(())
}
+39
View File
@@ -0,0 +1,39 @@
# Server Smoke Test
Start a fresh headless `vllm` engine:
```bash
source ../vllm/.venv/bin/activate
HF_HUB_OFFLINE=1 \
VLLM_LOGGING_LEVEL=DEBUG \
VLLM_CPU_KVCACHE_SPACE=2 \
VLLM_HOST_IP=127.0.0.1 \
VLLM_LOOPBACK_IP=127.0.0.1 \
python3 -m vllm.entrypoints.cli.main serve Qwen/Qwen3-0.6B \
--headless \
--data-parallel-address 127.0.0.1 \
--data-parallel-rpc-port 62100 \
--data-parallel-size-local 1 \
--max-model-len 512 \
--dtype float16
```
Run the Rust server smoke test:
```bash
cargo run -p vllm-server --example external_engine_openai_qwen -- \
--handshake-address tcp://127.0.0.1:62100
```
The example starts the Rust OpenAI-compatible server on an ephemeral local port,
connects to it via the `async-openai` Rust client, lists models, and then checks
that a streamed chat completion yields the assistant role chunk, final-answer
content chunks, and a terminal finish chunk. This example intentionally uses
`async-openai`'s standard typed `create_stream` API instead of BYOT, so it does
not inspect the nonstandard `reasoning_content` field even though the Rust
server may emit it for reasoning-capable models such as Qwen3. For reasoning
behavior itself, use the `vllm-chat` smoke test or the `vllm-server`
route tests.
IMPORTANT: Restart `vllm` each time you run the smoke test. The current headless
engine cannot safely handle frontend reconnects after the client shuts down.
@@ -0,0 +1,215 @@
use std::time::Duration;
use anyhow::{Context, Result, bail};
use async_openai::Client;
use async_openai::config::OpenAIConfig;
use async_openai::types::chat::{
ChatCompletionRequestUserMessageArgs, CreateChatCompletionRequest,
CreateChatCompletionRequestArgs,
};
use async_openai::types::models::ListModelResponse;
use clap::Parser;
use futures::StreamExt as _;
use tokio_util::sync::CancellationToken;
use tracing_subscriber::EnvFilter;
use vllm_engine_core_client::TransportMode;
use vllm_server::{
ChatTemplateContentFormatOption, Config, CoordinatorMode, HttpListenerMode, ParserSelection,
RendererSelection, serve,
};
#[derive(Debug, Parser)]
#[command(
about = "Smoke-test the Rust OpenAI server with async-openai against an external Qwen vLLM engine."
)]
struct Args {
#[arg(long)]
handshake_address: String,
#[arg(long, default_value_t = 1)]
engine_count: usize,
#[arg(long, default_value = "Qwen/Qwen3-0.6B")]
model: String,
#[arg(long, default_value = "127.0.0.1")]
host: String,
#[arg(long, default_value_t = 30)]
ready_timeout_secs: u64,
#[arg(
long,
default_value = "What is the capital of France? Answer with one word."
)]
prompt: String,
}
#[tokio::main(flavor = "multi_thread")]
async fn main() -> Result<()> {
init_tracing();
let args = Args::parse();
let port = unique_local_port()?;
let config = Config {
transport_mode: TransportMode::HandshakeOwner {
handshake_address: args.handshake_address,
advertised_host: args.host,
engine_count: args.engine_count,
ready_timeout: Duration::from_secs(args.ready_timeout_secs),
local_input_address: None,
local_output_address: None,
},
coordinator_mode: CoordinatorMode::MaybeInProc,
model: args.model,
served_model_name: vec![],
listener_mode: HttpListenerMode::BindTcp {
host: "127.0.0.1".to_string(),
port,
},
tool_call_parser: ParserSelection::Auto,
reasoning_parser: ParserSelection::Auto,
renderer: RendererSelection::Auto,
chat_template: None,
default_chat_template_kwargs: None,
chat_template_content_format: ChatTemplateContentFormatOption::Auto,
enable_log_requests: false,
disable_log_stats: false,
grpc_port: None,
shutdown_timeout: Duration::ZERO,
};
let bind_address = format!("127.0.0.1:{port}");
let shutdown = CancellationToken::new();
let server_config = config.clone();
let server_shutdown = shutdown.clone();
let server_task = tokio::spawn(async move { serve(server_config, server_shutdown).await });
let client = Client::with_config(
OpenAIConfig::new()
.with_api_key("unused")
.with_api_base(format!("http://{bind_address}/v1")),
);
print_models(&client).await?;
let final_text = stream_completion(&client, &config.model, &args.prompt).await?;
println!();
println!("final_text={final_text:?}");
shutdown_server(server_task, shutdown).await
}
fn init_tracing() {
let filter = EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new("info"));
let _ = tracing_subscriber::fmt().with_env_filter(filter).try_init();
}
fn unique_local_port() -> Result<u16> {
let listener = std::net::TcpListener::bind("127.0.0.1:0")
.context("failed to allocate local smoke-test port")?;
let port = listener.local_addr().context("failed to read local smoke-test port")?.port();
drop(listener);
Ok(port)
}
async fn print_models(client: &Client<OpenAIConfig>) -> Result<()> {
let models = wait_for_models(client).await?;
let model_ids = models.data.into_iter().map(|model| model.id).collect::<Vec<_>>();
println!("models={model_ids:?}");
Ok(())
}
async fn wait_for_models(client: &Client<OpenAIConfig>) -> Result<ListModelResponse> {
let mut last_error = None;
for _ in 0..240 {
match client.models().list().await {
Ok(models) => return Ok(models),
Err(error) => {
last_error = Some(error);
tokio::time::sleep(Duration::from_millis(500)).await;
}
}
}
match last_error {
Some(error) => Err(error).context("OpenAI server did not become ready in time"),
None => bail!("OpenAI server readiness loop finished without a result"),
}
}
async fn stream_completion(
client: &Client<OpenAIConfig>,
model: &str,
prompt: &str,
) -> Result<String> {
// Keep this smoke test on async-openai's standard `create_stream` path so it
// exercises the ordinary typed chat-completions client without BYOT
// request/response types.
//
// The current async-openai chat-completions stream delta type does not expose
// our OpenAI-compatible `reasoning_content` extension field, so this
// example only validates the assistant role chunk, visible `content`
// deltas, and terminal finish chunk. Reasoning coverage lives in our own
// route tests and in the `vllm-chat` smoke example.
let request: CreateChatCompletionRequest = CreateChatCompletionRequestArgs::default()
.model(model)
.stream(true)
.temperature(0.0)
.max_completion_tokens(128u32)
.messages([ChatCompletionRequestUserMessageArgs::default()
.content(prompt)
.build()
.context("failed to build user chat message")?
.into()])
.build()
.context("failed to build chat completion request")?;
let mut stream = client
.chat()
.create_stream(request)
.await
.context("failed to create streaming chat completion")?;
let mut final_text = String::new();
let mut saw_role = false;
let mut saw_finish_reason = false;
let mut saw_text = false;
while let Some(chunk) = stream.next().await {
let chunk = chunk.context("streaming chat completion failed")?;
for choice in chunk.choices {
if choice.delta.role.is_some() {
saw_role = true;
}
if let Some(delta) = choice.delta.content {
if !saw_text {
print!("[answer] ");
}
print!("{delta}");
final_text.push_str(&delta);
saw_text = true;
}
if choice.finish_reason.is_some() {
saw_finish_reason = true;
}
}
}
if !saw_role {
bail!("stream ended without an assistant role chunk");
}
if !saw_finish_reason {
bail!("stream ended without a terminal finish reason");
}
if final_text.is_empty() {
bail!("stream ended without any content deltas");
}
Ok(final_text)
}
async fn shutdown_server(
server_task: tokio::task::JoinHandle<anyhow::Result<()>>,
shutdown: CancellationToken,
) -> Result<()> {
shutdown.cancel();
server_task
.await
.context("server task join failed")?
.context("server task failed")
}
+110
View File
@@ -0,0 +1,110 @@
use std::collections::HashMap;
use std::time::Duration;
use anyhow::Result;
use serde_json::Value;
use vllm_chat::{ChatTemplateContentFormatOption, ParserSelection, RendererSelection};
use vllm_engine_core_client::{CoordinatorMode as EngineCoreCoordinatorMode, TransportMode};
/// How the HTTP server obtains its listening socket.
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum HttpListenerMode {
/// Bind a fresh TCP listener on the given host/port.
BindTcp { host: String, port: u16 },
/// Bind a fresh Unix domain listener on the given filesystem path.
BindUnix { path: String },
/// Adopt an already-open listening socket inherited from a supervisor
/// process.
InheritedFd { fd: i32 },
}
/// Which coordinator implementation should be active when one is present for a
/// frontend client.
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum CoordinatorMode {
/// Do not run a coordinator at all.
None,
/// Run the Rust in-process coordinator for managed `serve` deployments, if
/// there are multiple engines and the model is MoE.
MaybeInProc,
/// Connect to an external coordinator owned by another process.
External { address: String },
}
/// Normalized runtime configuration for the minimal OpenAI-compatible server.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct Config {
/// Frontend-to-engine transport setup.
pub transport_mode: TransportMode,
/// Requested frontend-side coordinator behavior.
pub coordinator_mode: CoordinatorMode,
/// Backend model identifier used for engine-core loading.
pub model: String,
/// Model name(s) exposed to clients via the OpenAI API. When non-empty,
/// the first entry is used as the primary ID in responses and all entries
/// are accepted in requests. When empty, falls back to `model`.
pub served_model_name: Vec<String>,
/// HTTP listener setup.
pub listener_mode: HttpListenerMode,
/// Tool-call parser selection.
pub tool_call_parser: ParserSelection,
/// Reasoning parser selection.
pub reasoning_parser: ParserSelection,
/// Chat renderer selection.
pub renderer: RendererSelection,
/// Server-default chat template override, as a file path or inline
/// template.
pub chat_template: Option<String>,
/// Server-default keyword arguments merged into every chat-template render.
pub default_chat_template_kwargs: Option<HashMap<String, Value>>,
/// How to serialize `message.content` for chat-template rendering.
pub chat_template_content_format: ChatTemplateContentFormatOption,
/// Log a summary line for each completed request.
pub enable_log_requests: bool,
/// When `true`, suppress periodic stats logging (throughput, queue depth,
/// cache usage).
pub disable_log_stats: bool,
/// TCP port for the gRPC Generate service. When `None`, no gRPC server is
/// started.
pub grpc_port: Option<u16>,
/// Maximum time to wait for active HTTP/gRPC requests to drain on shutdown.
pub shutdown_timeout: Duration,
}
impl Config {
/// Validate frontend configuration that can be checked before engine
/// startup.
pub fn validate(&self) -> Result<()> {
vllm_chat::validate_parser_overrides(&self.tool_call_parser, &self.reasoning_parser)?;
Ok(())
}
/// Return the number of engines implied by the configured transport mode.
pub fn engine_count(&self) -> usize {
match &self.transport_mode {
TransportMode::HandshakeOwner { engine_count, .. }
| TransportMode::Bootstrapped { engine_count, .. } => *engine_count,
}
}
/// Resolve the effective coordinator mode.
pub fn effective_coordinator_mode(
&self,
model_is_moe: bool,
) -> Option<EngineCoreCoordinatorMode> {
match &self.coordinator_mode {
CoordinatorMode::None => None,
CoordinatorMode::MaybeInProc => {
if model_is_moe && self.engine_count() > 1 {
Some(EngineCoreCoordinatorMode::InProc)
} else {
None
}
}
CoordinatorMode::External { address } => Some(EngineCoreCoordinatorMode::External {
address: address.clone(),
}),
}
}
}
+74
View File
@@ -0,0 +1,74 @@
use axum::Json;
use axum::http::StatusCode;
use axum::response::{IntoResponse, Response};
use thiserror_ext::{Construct, Macro};
use crate::routes::openai::utils::types::{ErrorDetail, ErrorResponse};
/// Small OpenAI-style error family used by the minimal HTTP layer.
#[derive(Debug, Construct, Macro)]
pub enum ApiError {
/// The request is syntactically valid OpenAI JSON but asks for unsupported
/// behavior.
InvalidRequest {
message: String,
param: Option<&'static str>,
},
/// The requested model name does not match the single configured model.
ModelNotFound { model: String },
/// The request body could not be parsed as valid JSON.
JsonParseError { message: String },
/// An unexpected internal failure happened before streaming started.
ServerError { message: String },
}
impl ApiError {
/// Return the HTTP status code associated with this API error.
pub fn status_code(&self) -> StatusCode {
match self {
Self::InvalidRequest { .. } => StatusCode::BAD_REQUEST,
Self::ModelNotFound { .. } => StatusCode::NOT_FOUND,
Self::ServerError { .. } => StatusCode::INTERNAL_SERVER_ERROR,
Self::JsonParseError { .. } => StatusCode::BAD_REQUEST,
}
}
/// Convert this error into the standard OpenAI-compatible JSON error
/// payload.
pub fn to_error_response(&self) -> ErrorResponse {
let error = match self {
Self::InvalidRequest { message, param } => ErrorDetail {
message: message.clone(),
error_type: "invalid_request_error".to_string(),
param: param.map(|p| p.to_string()),
code: Some("invalid_request_error".to_string()),
},
Self::ModelNotFound { model } => ErrorDetail {
message: format!("The model `{model}` does not exist."),
error_type: "invalid_request_error".to_string(),
param: Some("model".to_string()),
code: Some("model_not_found".to_string()),
},
Self::ServerError { message } => ErrorDetail {
message: message.clone(),
error_type: "server_error".to_string(),
param: None,
code: Some("server_error".to_string()),
},
Self::JsonParseError { message } => ErrorDetail {
message: message.clone(),
error_type: "invalid_request_error".to_string(),
param: None,
code: Some("json_parse_error".to_string()),
},
};
ErrorResponse { error }
}
}
impl IntoResponse for ApiError {
fn into_response(self) -> Response {
(self.status_code(), Json(self.to_error_response())).into_response()
}
}
+679
View File
@@ -0,0 +1,679 @@
//! Conversion between gRPC protobuf types and internal `vllm-text`
//! request/response types.
use tonic::Status;
use uuid::Uuid;
use vllm_engine_core_client::protocol::{StopReason, StructuredOutputsParams};
use vllm_text::{
DecodedLogprobs, DecodedPromptLogprobs, FinishReason, Finished, Prompt, SamplingParams,
TextDecodeOptions, TextRequest,
};
use super::pb;
// ========================================================================================
// Request conversion
// ========================================================================================
/// Convert a gRPC `GenerateRequest` into the internal `TextRequest`.
///
/// If `req.model` is non-empty, it must match one of `served_model_names`;
/// otherwise the request is rejected with `NotFound`. An empty string is
/// treated as "unset" (proto3 default) and accepted.
pub fn to_text_request(
req: pb::GenerateRequest,
stream: bool,
served_model_names: &[String],
) -> Result<TextRequest, Status> {
if !req.model.is_empty() && !served_model_names.iter().any(|n| n == &req.model) {
return Err(Status::not_found(format!(
"model `{}` not found",
req.model
)));
}
if req.truncate_prompt_tokens != 0 {
return Err(Status::invalid_argument(
"truncate_prompt_tokens is not supported",
));
}
let prompt = match req.prompt {
Some(pb::generate_request::Prompt::Text(text)) => Prompt::Text(text),
Some(pb::generate_request::Prompt::TokenIds(ids)) => Prompt::TokenIds(ids.ids),
None => return Err(Status::invalid_argument("prompt is required")),
};
let request_id = if req.request_id.is_empty() {
Uuid::new_v4().to_string()
} else {
req.request_id
};
let sampling = req.sampling.as_ref();
let decoding = req.decoding.as_ref();
let stopping = req.stopping.as_ref();
let response = req.response.as_ref();
let kv = req.kv.as_ref();
let mut sampling_params =
build_sampling_params(req.temperature, sampling, decoding, stopping, response)?;
// Thread KVCacheParameters → SamplingParams fields.
if let Some(kv) = kv {
// Thread kv_transfer_params through vllm_xargs, matching the HTTP route
// convention.
if let Some(kv_struct) = kv.kv_transfer_params.as_ref() {
let kv_json = proto_struct_to_json(kv_struct);
let map = sampling_params.vllm_xargs.get_or_insert_with(Default::default);
map.insert("kv_transfer_params".to_string(), kv_json);
}
if kv.bypass_prefix_cache {
sampling_params.skip_reading_prefix_cache = Some(true);
}
}
let decode_options = TextDecodeOptions {
skip_special_tokens: true,
include_stop_str_in_output: stopping.is_some_and(|s| s.include_stop_strings),
stop_strings: stopping.map(|s| &s.stop_strings).filter(|ss| !ss.is_empty()).cloned(),
min_tokens: stopping.map_or(0, |s| s.min_new_tokens),
};
Ok(TextRequest {
request_id,
prompt,
mm_features: None,
sampling_params,
decode_options,
intermediate: stream,
priority: req.priority,
cache_salt: kv.map(|k| &k.cache_salt).filter(|s| !s.is_empty()).cloned(),
add_special_tokens: true,
data_parallel_rank: None,
})
}
fn build_sampling_params(
temperature: Option<f32>,
sampling: Option<&pb::RandomSampling>,
decoding: Option<&pb::DecodingParameters>,
stopping: Option<&pb::StoppingCriteria>,
response: Option<&pb::ResponseOptions>,
) -> Result<SamplingParams, Status> {
// Temperature is a top-level GenerateRequest field. Default to greedy (0.0) for
// the gRPC API when the caller does not specify a value. This differs from
// the HTTP/OpenAI API (which defaults to 1.0) and matches the convention of
// programmatic generation APIs.
let temperature = temperature.or(Some(0.0));
let mut params = SamplingParams {
temperature,
..SamplingParams::default()
};
// RandomSampling: for every remaining sampling field the protobuf default (`0`)
// is treated as "unset" and leaves the resolved value to the lowering
// stage, which falls back to the model-provided default or a
// neutral/disabled value otherwise.
if let Some(s) = sampling {
// num_sequences (n > 1) is not supported yet by the TextLlm layer; the response
// path also hardcodes SequenceOutput.index = 0, so accepting >1 would silently
// truncate output cardinality. Reject explicitly.
if s.num_sequences > 1 {
return Err(Status::invalid_argument(
"num_sequences > 1 is not supported",
));
}
if s.top_k != 0 {
params.top_k = Some(s.top_k);
}
if s.top_p != 0.0 {
params.top_p = Some(s.top_p);
}
if s.min_p != 0.0 {
params.min_p = Some(s.min_p);
}
params.seed = s.seed;
}
// DecodingParameters
if let Some(d) = decoding {
if d.presence_penalty != 0.0 {
params.presence_penalty = Some(d.presence_penalty);
}
if d.frequency_penalty != 0.0 {
params.frequency_penalty = Some(d.frequency_penalty);
}
if d.repetition_penalty != 0.0 {
params.repetition_penalty = Some(d.repetition_penalty);
}
if !d.logit_bias.is_empty() {
params.logit_bias = Some(d.logit_bias.clone());
}
if !d.allowed_token_ids.is_empty() {
params.allowed_token_ids = Some(d.allowed_token_ids.clone());
}
params.structured_outputs = convert_structured_output(d)?;
}
// StoppingCriteria
if let Some(s) = stopping {
if s.max_new_tokens != 0 {
params.max_tokens = Some(s.max_new_tokens);
}
if s.min_new_tokens != 0 {
params.min_tokens = Some(s.min_new_tokens);
}
if !s.stop_token_ids.is_empty() {
params.stop_token_ids = Some(s.stop_token_ids.clone());
}
params.ignore_eos = s.ignore_eos;
}
// ResponseOptions → logprobs
if let Some(r) = response {
if r.output_logprobs {
let (count, token_ids) = candidate_logprob_spec(r.output_candidates.as_ref());
params.logprobs = Some(count);
params.logprob_token_ids = token_ids;
}
if r.prompt_logprobs {
// The engine-core protocol has only one shared `logprob_token_ids` field
// for output and prompt logprobs, so a per-token-id selector for prompt
// candidates can't be honored independently. Reject it instead of silently
// dropping the list.
if matches!(
r.prompt_candidates.as_ref().and_then(|c| c.select.as_ref()),
Some(pb::candidate_tokens::Select::TokenIds(_))
) {
return Err(Status::invalid_argument(
"prompt_candidates token_ids selector is not supported",
));
}
let (count, _) = candidate_logprob_spec(r.prompt_candidates.as_ref());
params.prompt_logprobs = Some(count);
}
}
Ok(params)
}
/// Map the proto `CandidateTokens` selector to a `(logprobs_count,
/// logprob_token_ids)` pair.
///
/// - `top_n(k)` → `(k, None)` — return top-k candidates by probability
/// - `all` → `(-1, None)` — return the full vocabulary
/// - `token_ids(n)` → `(1, Some(vec of n token ids))` — return logprobs for specific tokens (the
/// count `n` is stored in the proto as the number of token IDs that follow, but the actual IDs
/// are carried via `logprob_token_ids` on `SamplingParams`)
/// - absent → `(1, None)` — just the sampled/scored token
fn candidate_logprob_spec(candidates: Option<&pb::CandidateTokens>) -> (i32, Option<Vec<u32>>) {
match candidates.and_then(|c| c.select.as_ref()) {
Some(pb::candidate_tokens::Select::TopN(n)) => (*n as i32, None),
Some(pb::candidate_tokens::Select::All(true)) => (-1, None),
Some(pb::candidate_tokens::Select::TokenIds(ids)) => (1, Some(ids.ids.clone())),
_ => (1, None),
}
}
fn convert_structured_output(
d: &pb::DecodingParameters,
) -> Result<Option<StructuredOutputsParams>, Status> {
let so = match d.structured_output.as_ref() {
None => return Ok(None),
Some(so) => so,
};
use pb::decoding_parameters::StructuredOutput;
let params = match so {
StructuredOutput::Json(schema) => {
let json: serde_json::Value = serde_json::from_str(schema)
.map_err(|e| Status::invalid_argument(format!("invalid json schema: {e}")))?;
StructuredOutputsParams {
json: Some(json),
..Default::default()
}
}
StructuredOutput::Regex(regex) => StructuredOutputsParams {
regex: Some(regex.clone()),
..Default::default()
},
StructuredOutput::Choice(choices) => StructuredOutputsParams {
choice: Some(choices.choices.clone()),
..Default::default()
},
StructuredOutput::Grammar(grammar) => StructuredOutputsParams {
grammar: Some(grammar.clone()),
..Default::default()
},
StructuredOutput::JsonObject(true) => StructuredOutputsParams {
json_object: Some(true),
..Default::default()
},
StructuredOutput::JsonObject(false) => return Ok(None),
StructuredOutput::StructuralTag(tag) => StructuredOutputsParams {
structural_tag: Some(tag.clone()),
..Default::default()
},
};
Ok(Some(params))
}
// ========================================================================================
// Response conversion
// ========================================================================================
/// Convert a `DecodedTextEvent::Start` into the prompt info portion of a gRPC
/// response.
pub fn to_prompt_info(
prompt_token_ids: &[u32],
prompt_logprobs: Option<&DecodedPromptLogprobs>,
opts: &ResponseOpts,
) -> pb::PromptInfo {
let token_ids = if opts.prompt_token_ids {
prompt_token_ids.to_vec()
} else {
vec![]
};
let (logprobs, ranks, candidate_tokens) = match prompt_logprobs {
Some(plp) if opts.prompt_logprobs => prompt_logprobs_to_proto(plp),
_ => (vec![], vec![], vec![]),
};
pb::PromptInfo {
num_prompt_tokens: prompt_token_ids.len() as u32,
token_ids,
logprobs,
ranks,
candidate_tokens,
}
}
/// Convert a `DecodedTextEvent::TextDelta` into a gRPC `SequenceOutput`.
pub fn to_sequence_output(
delta: &str,
token_ids: &[u32],
logprobs: Option<&DecodedLogprobs>,
finished: Option<&Finished>,
opts: &ResponseOpts,
) -> pb::SequenceOutput {
let (lp_values, rank_values, candidates) = match logprobs {
Some(lp) if opts.output_logprobs => output_logprobs_to_proto(lp),
_ => (vec![], vec![], vec![]),
};
pb::SequenceOutput {
index: 0, // TODO: multi-sequence (n > 1) not supported
text: if opts.output_text {
delta.to_string()
} else {
String::new()
},
num_tokens: token_ids.len() as u32,
token_ids: if opts.output_token_ids {
token_ids.to_vec()
} else {
vec![]
},
logprobs: lp_values,
ranks: rank_values,
candidate_tokens: candidates,
finish_info: finished.map(|f| to_finish_info(f, token_ids)),
}
}
fn to_finish_info(finished: &Finished, token_ids: &[u32]) -> pb::FinishInfo {
use pb::finish_info::FinishReason as PbFinishReason;
let (finish_reason, stop_reason) = match &finished.finish_reason {
FinishReason::Stop(reason) => {
let sr = match reason {
Some(StopReason::TokenId(id)) => {
Some(pb::finish_info::StopReason::StopTokenId(*id))
}
Some(StopReason::Text(s)) => {
Some(pb::finish_info::StopReason::StopString(s.clone()))
}
// EOS-driven stop: engine-core matched the primary EOS token id but did not
// echo it back as a `stop_reason`. The matched token is, by construction, the
// last token of the terminal output batch (see vllm's `check_stop` in
// vllm/v1/core/sched/utils.py), so we recover it from there.
None => token_ids.last().copied().map(pb::finish_info::StopReason::EosTokenId),
};
(PbFinishReason::Stop as i32, sr)
}
FinishReason::Length => (PbFinishReason::Length as i32, None),
FinishReason::Abort | FinishReason::Error | FinishReason::Repetition => {
(PbFinishReason::Aborted as i32, None)
}
};
pb::FinishInfo {
num_output_tokens: finished.output_token_count as u32,
finish_reason,
stop_reason,
kv_transfer_params: finished.kv_transfer_params.as_ref().and_then(json_to_proto_struct),
}
}
// ========================================================================================
// Logprobs helpers
// ========================================================================================
/// Convert output logprobs to the flat proto representation.
///
/// Returns (logprob_values, ranks, candidate_tokens) — all parallel arrays
/// indexed by position.
fn output_logprobs_to_proto(
lp: &DecodedLogprobs,
) -> (Vec<f32>, Vec<u32>, Vec<pb::CandidateTokenInfo>) {
positions_to_proto(&lp.positions)
}
/// Convert prompt logprobs to the flat proto representation.
fn prompt_logprobs_to_proto(
plp: &DecodedPromptLogprobs,
) -> (Vec<f32>, Vec<u32>, Vec<pb::CandidateTokenInfo>) {
// The proto PromptInfo has flat parallel arrays covering all prompt positions.
// DecodedPromptLogprobs has first_token separately + scored_positions for the
// rest. The first prompt position has no scores, so we emit zeros for it.
let (mut logprobs, mut ranks, mut candidates) = positions_to_proto(&plp.scored_positions);
logprobs.insert(0, 0.0);
ranks.insert(0, 0);
candidates.insert(0, pb::CandidateTokenInfo { tokens: vec![] });
(logprobs, ranks, candidates)
}
/// Shared helper: convert a slice of decoded position logprobs to flat proto
/// arrays.
fn positions_to_proto(
positions: &[vllm_text::DecodedPositionLogprobs],
) -> (Vec<f32>, Vec<u32>, Vec<pb::CandidateTokenInfo>) {
let mut logprobs = Vec::with_capacity(positions.len());
let mut ranks = Vec::with_capacity(positions.len());
let mut candidates = Vec::with_capacity(positions.len());
for pos in positions {
// First entry is the sampled/scored token.
if let Some(first) = pos.entries.first() {
logprobs.push(first.logprob);
ranks.push(first.rank);
}
// Extra candidates beyond the first.
let entries = pos.entries.iter().skip(1);
candidates.push(pb::CandidateTokenInfo {
tokens: entries
.map(|e| pb::candidate_token_info::TokenInfo {
id: e.token_id,
logprob: e.logprob,
rank: e.rank,
})
.collect(),
});
}
(logprobs, ranks, candidates)
}
// ========================================================================================
// KV transfer params conversion (serde_json::Value ↔ prost_types::Struct)
// ========================================================================================
fn proto_struct_to_json(s: &prost_types::Struct) -> serde_json::Value {
serde_json::Value::Object(
s.fields.iter().map(|(k, v)| (k.clone(), proto_value_to_json(v))).collect(),
)
}
fn proto_value_to_json(v: &prost_types::Value) -> serde_json::Value {
use prost_types::value::Kind;
match v.kind.as_ref() {
None | Some(Kind::NullValue(_)) => serde_json::Value::Null,
Some(Kind::BoolValue(b)) => serde_json::Value::Bool(*b),
Some(Kind::NumberValue(n)) => serde_json::json!(*n),
Some(Kind::StringValue(s)) => serde_json::Value::String(s.clone()),
Some(Kind::ListValue(list)) => {
serde_json::Value::Array(list.values.iter().map(proto_value_to_json).collect())
}
Some(Kind::StructValue(s)) => proto_struct_to_json(s),
}
}
fn json_to_proto_struct(value: &serde_json::Value) -> Option<prost_types::Struct> {
match value {
serde_json::Value::Object(map) => Some(prost_types::Struct {
fields: map.iter().map(|(k, v)| (k.clone(), json_to_proto_value(v))).collect(),
}),
_ => None,
}
}
fn json_to_proto_value(v: &serde_json::Value) -> prost_types::Value {
use prost_types::value::Kind;
let kind = match v {
serde_json::Value::Null => Kind::NullValue(0),
serde_json::Value::Bool(b) => Kind::BoolValue(*b),
serde_json::Value::Number(n) => Kind::NumberValue(n.as_f64().unwrap_or(0.0)),
serde_json::Value::String(s) => Kind::StringValue(s.clone()),
serde_json::Value::Array(arr) => Kind::ListValue(prost_types::ListValue {
values: arr.iter().map(json_to_proto_value).collect(),
}),
serde_json::Value::Object(map) => Kind::StructValue(prost_types::Struct {
fields: map.iter().map(|(k, v)| (k.clone(), json_to_proto_value(v))).collect(),
}),
};
prost_types::Value { kind: Some(kind) }
}
// ========================================================================================
// Options extracted from the request for response building
// ========================================================================================
/// Response-shaping options extracted from the proto `ResponseOptions`.
#[derive(Default)]
pub struct ResponseOpts {
pub prompt_token_ids: bool,
pub prompt_logprobs: bool,
pub output_text: bool,
pub output_token_ids: bool,
pub output_logprobs: bool,
}
impl ResponseOpts {
pub fn from_proto(r: Option<&pb::ResponseOptions>) -> Self {
match r {
Some(r) => Self {
prompt_token_ids: r.prompt_token_ids,
prompt_logprobs: r.prompt_logprobs,
output_text: r.output_text.unwrap_or(true),
output_token_ids: r.output_token_ids,
output_logprobs: r.output_logprobs,
},
None => Self {
output_text: true,
..Default::default()
},
}
}
}
#[cfg(test)]
mod tests {
use vllm_engine_core_client::protocol::StopReason;
use vllm_text::{FinishReason, Finished, Prompt};
use super::pb::finish_info::{FinishReason as PbFinishReason, StopReason as PbStopReason};
use super::{ResponseOpts, pb, to_finish_info, to_sequence_output, to_text_request};
fn base_request() -> pb::GenerateRequest {
pb::GenerateRequest {
request_id: "req".to_string(),
model: "test-model".to_string(),
prompt: Some(pb::generate_request::Prompt::Text("hi".to_string())),
..Default::default()
}
}
#[test]
fn temperature_propagates_from_top_level_request_field() {
let req = pb::GenerateRequest {
temperature: Some(0.7),
..base_request()
};
let text = to_text_request(req, false, &["test-model".to_string()]).expect("convert ok");
assert_eq!(text.sampling_params.temperature, Some(0.7));
}
#[test]
fn unset_temperature_defaults_to_greedy() {
let text = to_text_request(base_request(), false, &["test-model".to_string()])
.expect("convert ok");
// The gRPC API defaults to greedy (0.0) when temperature is not specified.
assert_eq!(text.sampling_params.temperature, Some(0.0));
}
#[test]
fn absent_seed_is_none() {
let req = pb::GenerateRequest {
sampling: Some(pb::RandomSampling {
seed: None,
..Default::default()
}),
..base_request()
};
let text = to_text_request(req, false, &["test-model".to_string()]).expect("convert ok");
assert_eq!(text.sampling_params.seed, None);
}
#[test]
fn zero_seed_is_valid() {
let req = pb::GenerateRequest {
sampling: Some(pb::RandomSampling {
seed: Some(0),
..Default::default()
}),
..base_request()
};
let text = to_text_request(req, false, &["test-model".to_string()]).expect("convert ok");
assert_eq!(text.sampling_params.seed, Some(0));
}
#[test]
fn bypass_prefix_cache_maps_to_skip_reading_prefix_cache() {
let req = pb::GenerateRequest {
kv: Some(pb::KvCacheParameters {
bypass_prefix_cache: true,
..Default::default()
}),
..base_request()
};
let text = to_text_request(req, false, &["test-model".to_string()]).expect("convert ok");
assert_eq!(text.sampling_params.skip_reading_prefix_cache, Some(true));
}
#[test]
fn bypass_prefix_cache_false_leaves_field_unset() {
let req = pb::GenerateRequest {
kv: Some(pb::KvCacheParameters {
bypass_prefix_cache: false,
..Default::default()
}),
..base_request()
};
let text = to_text_request(req, false, &["test-model".to_string()]).expect("convert ok");
assert_eq!(text.sampling_params.skip_reading_prefix_cache, None);
// Prompt conversion still succeeds and reaches the expected variant.
assert!(matches!(text.prompt, Prompt::Text(s) if s == "hi"));
}
fn finished(reason: FinishReason) -> Finished {
Finished {
prompt_token_count: 0,
output_token_count: 0,
finish_reason: reason,
kv_transfer_params: None,
}
}
#[test]
fn eos_stop_reports_last_output_token_as_eos_id() {
let fin = finished(FinishReason::Stop(None));
let token_ids = [1_u32, 2, 3, 151643];
let info = to_finish_info(&fin, &token_ids);
assert_eq!(info.finish_reason, PbFinishReason::Stop as i32);
assert_eq!(info.stop_reason, Some(PbStopReason::EosTokenId(151643)));
}
#[test]
fn eos_stop_with_empty_token_ids_leaves_stop_reason_unset() {
let fin = finished(FinishReason::Stop(None));
let info = to_finish_info(&fin, &[]);
assert_eq!(info.finish_reason, PbFinishReason::Stop as i32);
assert_eq!(info.stop_reason, None);
}
#[test]
fn explicit_stop_token_id_is_preserved() {
let fin = finished(FinishReason::Stop(Some(StopReason::TokenId(42))));
// Terminal token list should be ignored when an explicit stop reason is
// present.
let info = to_finish_info(&fin, &[7, 42]);
assert_eq!(info.finish_reason, PbFinishReason::Stop as i32);
assert_eq!(info.stop_reason, Some(PbStopReason::StopTokenId(42)));
}
#[test]
fn explicit_stop_string_is_preserved() {
let fin = finished(FinishReason::Stop(Some(StopReason::Text("</stop>".into()))));
let info = to_finish_info(&fin, &[1, 2, 3]);
assert_eq!(info.finish_reason, PbFinishReason::Stop as i32);
assert_eq!(
info.stop_reason,
Some(PbStopReason::StopString("</stop>".into()))
);
}
#[test]
fn length_finish_has_no_stop_reason() {
let fin = finished(FinishReason::Length);
let info = to_finish_info(&fin, &[1, 2, 3]);
assert_eq!(info.finish_reason, PbFinishReason::Length as i32);
assert_eq!(info.stop_reason, None);
}
#[test]
fn abort_finish_is_mapped_to_aborted() {
let fin = finished(FinishReason::Abort);
let info = to_finish_info(&fin, &[]);
assert_eq!(info.finish_reason, PbFinishReason::Aborted as i32);
assert_eq!(info.stop_reason, None);
}
#[test]
fn to_sequence_output_threads_token_ids_into_eos_id() {
let fin = finished(FinishReason::Stop(None));
let opts = ResponseOpts {
output_text: true,
output_token_ids: true,
..Default::default()
};
let out = to_sequence_output("hello", &[10, 20, 30], None, Some(&fin), &opts);
let finish = out.finish_info.expect("finish_info should be present");
assert_eq!(finish.finish_reason, PbFinishReason::Stop as i32);
assert_eq!(finish.stop_reason, Some(PbStopReason::EosTokenId(30)));
}
}
+158
View File
@@ -0,0 +1,158 @@
//! gRPC Generate service backed by the shared [`vllm_text::TextLlm`] facade.
mod convert;
use std::pin::Pin;
use std::sync::Arc;
use futures::{Stream, StreamExt as _};
use thiserror_ext::AsReport as _;
use tokio::sync::mpsc;
use tokio_stream::wrappers::ReceiverStream;
use tonic::{Request, Response, Status};
use tracing::info;
use vllm_text::{DecodedTextEvent, TextOutputStreamExt as _};
use self::convert::ResponseOpts;
use crate::state::AppState;
/// Generated protobuf/gRPC types for the `vllm` package.
pub mod pb {
tonic::include_proto!("vllm");
}
pub use pb::generate_server::GenerateServer;
#[cfg(test)]
mod tests;
/// gRPC Generate service implementation backed by the shared application state.
pub struct GenerateServiceImpl {
state: Arc<AppState>,
}
impl GenerateServiceImpl {
pub fn new(state: Arc<AppState>) -> Self {
Self { state }
}
}
#[tonic::async_trait]
impl pb::generate_server::Generate for GenerateServiceImpl {
type GenerateStreamStream =
Pin<Box<dyn Stream<Item = Result<pb::GenerateResponse, Status>> + Send>>;
/// Unary generate: collect all output and return a single response.
async fn generate(
&self,
request: Request<pb::GenerateRequest>,
) -> Result<Response<pb::GenerateResponse>, Status> {
let proto_req = request.into_inner();
let response_opts = ResponseOpts::from_proto(proto_req.response.as_ref());
let text_request =
convert::to_text_request(proto_req, false, self.state.served_model_names())?;
let request_id = text_request.request_id.clone();
info!(%request_id, "grpc generate (unary)");
let stream = self.state.chat.text().generate(text_request).await;
let stream = stream.map_err(|e| Status::internal(e.to_report_string()))?;
let collected = stream
.collect_output()
.await
.map_err(|e| Status::internal(e.to_report_string()))?;
// Build the single aggregated response.
let prompt_info = convert::to_prompt_info(
&collected.prompt_token_ids,
collected.prompt_logprobs.as_ref(),
&response_opts,
);
let finish_info = vllm_text::Finished {
prompt_token_count: collected.prompt_token_ids.len(),
output_token_count: collected.token_ids.len(),
finish_reason: collected.finish_reason,
kv_transfer_params: collected.kv_transfer_params,
};
let outputs = convert::to_sequence_output(
&collected.text,
&collected.token_ids,
collected.logprobs.as_ref(),
Some(&finish_info),
&response_opts,
);
Ok(Response::new(pb::GenerateResponse {
prompt_info: Some(prompt_info),
outputs: Some(outputs),
}))
}
/// Streaming generate: yield incremental responses as tokens are produced.
async fn generate_stream(
&self,
request: Request<pb::GenerateRequest>,
) -> Result<Response<Self::GenerateStreamStream>, Status> {
let proto_req = request.into_inner();
let response_opts = ResponseOpts::from_proto(proto_req.response.as_ref());
let text_request =
convert::to_text_request(proto_req, true, self.state.served_model_names())?;
let request_id = text_request.request_id.clone();
info!(%request_id, "grpc generate (stream)");
let stream = self.state.chat.text().generate(text_request).await;
let stream = stream.map_err(|e| Status::internal(e.to_report_string()))?;
let (tx, rx) = mpsc::channel(32);
tokio::spawn(async move {
futures::pin_mut!(stream);
while let Some(event) = stream.next().await {
let response = match event {
Err(e) => Err(Status::internal(e.to_report_string())),
Ok(DecodedTextEvent::Start {
prompt_token_ids,
prompt_logprobs,
}) => {
let prompt_info = convert::to_prompt_info(
&prompt_token_ids,
prompt_logprobs.as_ref(),
&response_opts,
);
Ok(pb::GenerateResponse {
prompt_info: Some(prompt_info),
outputs: None,
})
}
Ok(DecodedTextEvent::TextDelta {
delta,
token_ids,
logprobs,
finished,
}) => Ok(pb::GenerateResponse {
prompt_info: None,
outputs: Some(convert::to_sequence_output(
&delta,
&token_ids,
logprobs.as_ref(),
finished.as_ref(),
&response_opts,
)),
}),
};
if tx.send(response).await.is_err() {
// Client disconnected.
break;
}
}
});
let response_stream = ReceiverStream::new(rx);
Ok(Response::new(Box::pin(response_stream)))
}
}
+662
View File
@@ -0,0 +1,662 @@
use std::future::Future;
use std::pin::Pin;
use std::sync::Arc;
use std::task::{Context, Poll};
use futures::StreamExt as _;
use serial_test::serial;
use tonic::transport::Server as TonicServer;
use vllm_chat::{
ChatBackend, ChatLlm, ChatRenderer, ChatRequest, ChatTextBackend, DefaultChatOutputProcessor,
DynChatOutputProcessor, DynChatRenderer, NewChatOutputProcessorOptions, RenderedPrompt,
};
use vllm_engine_core_client::protocol::{
EngineCoreFinishReason, EngineCoreOutput, EngineCoreOutputs, EngineCoreRequest,
};
use vllm_engine_core_client::test_utils::{IpcNamespace, spawn_mock_engine_task};
use vllm_engine_core_client::{EngineCoreClient, EngineCoreClientConfig, EngineId};
use vllm_llm::Llm;
use vllm_text::tokenizer::{DynTokenizer, Tokenizer};
use vllm_text::{Prompt, TextBackend};
use zeromq::prelude::{SocketRecv, SocketSend};
use zeromq::{DealerSocket, PushSocket, ZmqMessage};
use super::pb::generate_client::GenerateClient;
use super::{GenerateServer, GenerateServiceImpl, pb};
use crate::state::AppState;
// ========================================================================================
// Helpers (mirrors the patterns in routes/tests.rs)
// ========================================================================================
type TestFuture<'a> = Pin<Box<dyn Future<Output = ()> + Send + 'a>>;
fn boxed_test_future<'a>(future: impl Future<Output = ()> + Send + 'a) -> TestFuture<'a> {
Box::pin(future)
}
struct MockEngineTask {
shutdown_tx: Option<tokio::sync::oneshot::Sender<()>>,
join_handle: Option<tokio::task::JoinHandle<()>>,
}
impl MockEngineTask {
fn new(
(shutdown_tx, join_handle): (
tokio::sync::oneshot::Sender<()>,
tokio::task::JoinHandle<()>,
),
) -> Self {
Self {
shutdown_tx: Some(shutdown_tx),
join_handle: Some(join_handle),
}
}
}
impl Future for MockEngineTask {
type Output = Result<(), tokio::task::JoinError>;
fn poll(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Self::Output> {
if let Some(shutdown_tx) = self.shutdown_tx.take() {
let _ = shutdown_tx.send(());
}
match self.join_handle.as_mut() {
Some(join_handle) => Pin::new(join_handle).poll(cx),
None => Poll::Ready(Ok(())),
}
}
}
impl Drop for MockEngineTask {
fn drop(&mut self) {
if let Some(join_handle) = &self.join_handle {
join_handle.abort();
}
}
}
fn request_output(
request_id: &str,
new_token_ids: Vec<u32>,
finish_reason: Option<EngineCoreFinishReason>,
) -> EngineCoreOutput {
EngineCoreOutput {
request_id: request_id.to_string(),
new_token_ids,
new_logprobs: None,
new_prompt_logprobs_tensors: None,
pooling_output: None,
finish_reason,
stop_reason: None,
events: None,
kv_transfer_params: None,
trace_headers: None,
prefill_stats: None,
routed_experts: None,
num_nans_in_logits: 0,
}
}
fn engine_outputs_for_request(
request_id: &str,
output_specs: Vec<(Vec<u32>, Option<EngineCoreFinishReason>)>,
) -> EngineCoreOutputs {
EngineCoreOutputs {
engine_index: 0,
outputs: output_specs
.into_iter()
.map(|(token_ids, finish_reason)| request_output(request_id, token_ids, finish_reason))
.collect(),
scheduler_stats: None,
timestamp: 0.0,
utility_output: None,
finished_requests: None,
wave_complete: None,
start_wave: None,
}
}
fn default_stream_output_specs() -> Vec<(Vec<u32>, Option<EngineCoreFinishReason>)> {
vec![
(vec![b'h' as u32], None),
(vec![b'i' as u32], None),
(vec![b'!' as u32], Some(EngineCoreFinishReason::Stop)),
]
}
async fn send_outputs(push: &mut PushSocket, outputs: EngineCoreOutputs) {
push.send(ZmqMessage::from(
rmp_serde::to_vec_named(&outputs).expect("encode outputs"),
))
.await
.expect("send outputs");
}
async fn recv_engine_message(dealer: &mut DealerSocket) -> Vec<bytes::Bytes> {
dealer.recv().await.expect("recv engine message").into_vec()
}
fn test_llm(client: EngineCoreClient) -> Llm {
Llm::new(client).with_request_id_randomization(false)
}
#[derive(Clone, Debug)]
struct FakeTextBackend;
#[derive(Debug)]
struct FakeTokenizer;
impl Tokenizer for FakeTokenizer {
fn encode(
&self,
text: &str,
_add_special_tokens: bool,
) -> vllm_text::tokenizer::Result<Vec<u32>> {
Ok(text.bytes().map(u32::from).collect())
}
fn decode(
&self,
token_ids: &[u32],
_skip_special_tokens: bool,
) -> vllm_text::tokenizer::Result<String> {
Ok(
String::from_utf8_lossy(&token_ids.iter().map(|id| *id as u8).collect::<Vec<_>>())
.into_owned(),
)
}
fn token_to_id(&self, token: &str) -> Option<u32> {
token.bytes().next().map(u32::from)
}
}
impl TextBackend for FakeTextBackend {
fn tokenizer(&self) -> DynTokenizer {
Arc::new(FakeTokenizer)
}
fn model_id(&self) -> &str {
"test-model"
}
}
impl ChatBackend for FakeTextBackend {
fn chat_renderer(&self) -> DynChatRenderer {
Arc::new(self.clone())
}
fn new_chat_output_processor(
&self,
request: &mut ChatRequest,
options: NewChatOutputProcessorOptions<'_>,
) -> vllm_chat::Result<DynChatOutputProcessor> {
Ok(Box::new(DefaultChatOutputProcessor::new(
request,
self.model_id(),
self.tokenizer(),
options.tool_call_parser,
options.reasoning_parser,
)?))
}
}
impl ChatRenderer for FakeTextBackend {
fn render(&self, _request: &ChatRequest) -> vllm_chat::Result<RenderedPrompt> {
Ok(RenderedPrompt {
prompt: Prompt::Text(String::new()),
})
}
}
/// Spin up a gRPC server backed by a mock engine that serves a single request
/// with the given output specs. Returns the client, the gRPC server task, and
/// the mock engine task.
async fn grpc_test_server(
engine_id: impl Into<EngineId>,
output_specs: Vec<(Vec<u32>, Option<EngineCoreFinishReason>)>,
) -> (
GenerateClient<tonic::transport::Channel>,
tokio::task::JoinHandle<()>,
MockEngineTask,
) {
let ipc = IpcNamespace::new().expect("create ipc namespace");
let handshake_address = ipc.handshake_endpoint();
let engine_id = engine_id.into();
let engine_task = MockEngineTask::new(spawn_mock_engine_task(
handshake_address.clone(),
engine_id.clone(),
move |dealer, push| {
boxed_test_future(async move {
let add = recv_engine_message(dealer).await;
let request: EngineCoreRequest =
rmp_serde::from_slice(&add[1]).expect("decode request");
send_outputs(
push,
engine_outputs_for_request(&request.request_id, output_specs),
)
.await;
})
},
));
let client = EngineCoreClient::connect(
EngineCoreClientConfig::new_single(handshake_address)
.with_model_name("test-model")
.with_local_input_output_addresses(
Some(ipc.input_endpoint()),
Some(ipc.output_endpoint()),
),
)
.await
.expect("connect client");
let chat = ChatLlm::from_shared_backend(
test_llm(client),
Arc::new(FakeTextBackend) as Arc<dyn ChatTextBackend>,
);
let state = Arc::new(AppState::new(vec!["test-model".to_string()], chat));
let svc = GenerateServer::new(GenerateServiceImpl::new(state));
// Bind to an OS-assigned port.
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.expect("bind grpc listener");
let addr = listener.local_addr().expect("local addr");
let server_task = tokio::spawn(async move {
let incoming = tokio_stream::wrappers::TcpListenerStream::new(listener);
TonicServer::builder()
.add_service(svc)
.serve_with_incoming(incoming)
.await
.expect("grpc server");
});
// Connect the client.
let grpc_client = GenerateClient::connect(format!("http://{addr}"))
.await
.expect("connect grpc client");
(grpc_client, server_task, engine_task)
}
// ========================================================================================
// Tests
// ========================================================================================
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
#[serial]
async fn unary_generate_returns_collected_text() {
let (mut client, server_task, engine_task) =
grpc_test_server(b"engine-grpc-unary", default_stream_output_specs()).await;
let response = client
.generate(pb::GenerateRequest {
request_id: "test-unary-1".to_string(),
model: "test-model".to_string(),
prompt: Some(pb::generate_request::Prompt::Text("hello".to_string())),
stopping: Some(pb::StoppingCriteria {
max_new_tokens: 10,
..Default::default()
}),
response: Some(pb::ResponseOptions {
output_text: Some(true),
..Default::default()
}),
..Default::default()
})
.await
.expect("unary generate")
.into_inner();
// Unary collects all tokens into one response.
let outputs = response.outputs.expect("outputs present");
assert_eq!(outputs.text, "hi");
let finish = outputs.finish_info.expect("finish_info present");
assert_eq!(
finish.finish_reason,
pb::finish_info::FinishReason::Stop as i32
);
assert_eq!(finish.num_output_tokens, 3);
let prompt = response.prompt_info.expect("prompt_info present");
assert_eq!(prompt.num_prompt_tokens, 5); // "hello" = 5 bytes
engine_task.await.expect("mock engine task");
server_task.abort();
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
#[serial]
async fn unary_generate_with_token_ids_prompt() {
let (mut client, server_task, engine_task) =
grpc_test_server(b"engine-grpc-token-ids", default_stream_output_specs()).await;
let response = client
.generate(pb::GenerateRequest {
request_id: "test-token-ids".to_string(),
model: "test-model".to_string(),
prompt: Some(pb::generate_request::Prompt::TokenIds(pb::TokenIds {
ids: vec![1, 2, 3],
})),
stopping: Some(pb::StoppingCriteria {
max_new_tokens: 10,
..Default::default()
}),
..Default::default()
})
.await
.expect("unary generate with token ids")
.into_inner();
let outputs = response.outputs.expect("outputs present");
assert_eq!(outputs.text, "hi");
assert_eq!(
response.prompt_info.expect("prompt_info").num_prompt_tokens,
3
);
engine_task.await.expect("mock engine task");
server_task.abort();
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
#[serial]
async fn unary_generate_returns_token_ids_when_requested() {
let (mut client, server_task, engine_task) =
grpc_test_server(b"engine-grpc-tok-resp", default_stream_output_specs()).await;
let response = client
.generate(pb::GenerateRequest {
request_id: "test-tok-resp".to_string(),
model: "test-model".to_string(),
prompt: Some(pb::generate_request::Prompt::Text("hi".to_string())),
stopping: Some(pb::StoppingCriteria {
max_new_tokens: 10,
..Default::default()
}),
response: Some(pb::ResponseOptions {
output_text: Some(true),
output_token_ids: true,
prompt_token_ids: true,
..Default::default()
}),
..Default::default()
})
.await
.expect("unary generate")
.into_inner();
let outputs = response.outputs.expect("outputs present");
assert_eq!(
outputs.token_ids,
vec![b'h' as u32, b'i' as u32, b'!' as u32]
);
let prompt = response.prompt_info.expect("prompt_info present");
assert_eq!(prompt.token_ids, vec![b'h' as u32, b'i' as u32]);
engine_task.await.expect("mock engine task");
server_task.abort();
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
#[serial]
async fn unary_generate_missing_prompt_returns_invalid_argument() {
let (mut client, server_task, _engine_task) =
grpc_test_server(b"engine-grpc-no-prompt", default_stream_output_specs()).await;
let status = client
.generate(pb::GenerateRequest {
request_id: "test-no-prompt".to_string(),
model: "test-model".to_string(),
prompt: None,
..Default::default()
})
.await
.expect_err("should fail without prompt");
assert_eq!(status.code(), tonic::Code::InvalidArgument);
assert!(status.message().contains("prompt"));
server_task.abort();
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
#[serial]
async fn streaming_generate_yields_incremental_responses() {
let (mut client, server_task, engine_task) =
grpc_test_server(b"engine-grpc-stream", default_stream_output_specs()).await;
let stream = client
.generate_stream(pb::GenerateRequest {
request_id: "test-stream-1".to_string(),
model: "test-model".to_string(),
prompt: Some(pb::generate_request::Prompt::Text("hello".to_string())),
stopping: Some(pb::StoppingCriteria {
max_new_tokens: 10,
..Default::default()
}),
response: Some(pb::ResponseOptions {
output_text: Some(true),
..Default::default()
}),
..Default::default()
})
.await
.expect("streaming generate")
.into_inner();
let responses: Vec<pb::GenerateResponse> =
stream.map(|r| r.expect("stream item")).collect().await;
// First response carries prompt info, subsequent ones carry output deltas.
assert!(
responses.len() >= 2,
"expected at least 2 streamed responses, got {}",
responses.len()
);
// First message should have prompt info.
let first = &responses[0];
let prompt_info = first.prompt_info.as_ref().expect("first response has prompt_info");
assert_eq!(prompt_info.num_prompt_tokens, 5); // "hello"
// Collect all text deltas.
let full_text: String = responses
.iter()
.filter_map(|r| r.outputs.as_ref())
.map(|o| o.text.as_str())
.collect();
assert_eq!(full_text, "hi");
// Last output response should have finish info.
let last_output = responses
.iter()
.rev()
.find_map(|r| r.outputs.as_ref())
.expect("at least one output");
let finish = last_output.finish_info.as_ref().expect("finish_info on last output");
assert_eq!(
finish.finish_reason,
pb::finish_info::FinishReason::Stop as i32
);
engine_task.await.expect("mock engine task");
server_task.abort();
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
#[serial]
async fn streaming_generate_missing_prompt_returns_invalid_argument() {
let (mut client, server_task, _engine_task) = grpc_test_server(
b"engine-grpc-stream-no-prompt",
default_stream_output_specs(),
)
.await;
let status = client
.generate_stream(pb::GenerateRequest {
request_id: "test-stream-no-prompt".to_string(),
model: "test-model".to_string(),
prompt: None,
..Default::default()
})
.await
.expect_err("should fail without prompt");
assert_eq!(status.code(), tonic::Code::InvalidArgument);
server_task.abort();
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
#[serial]
async fn unary_generate_with_sampling_params() {
let (mut client, server_task, engine_task) =
grpc_test_server(b"engine-grpc-sampling", default_stream_output_specs()).await;
let response = client
.generate(pb::GenerateRequest {
request_id: "test-sampling".to_string(),
model: "test-model".to_string(),
prompt: Some(pb::generate_request::Prompt::Text("test".to_string())),
temperature: Some(0.7),
sampling: Some(pb::RandomSampling {
top_k: 50,
top_p: 0.9,
seed: Some(42),
..Default::default()
}),
stopping: Some(pb::StoppingCriteria {
max_new_tokens: 5,
..Default::default()
}),
..Default::default()
})
.await
.expect("generate with sampling params")
.into_inner();
// Verify the request was accepted and produced output.
let outputs = response.outputs.expect("outputs present");
assert_eq!(outputs.text, "hi");
engine_task.await.expect("mock engine task");
server_task.abort();
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
#[serial]
async fn unary_generate_rejects_wrong_model() {
let (mut client, server_task, _engine_task) =
grpc_test_server(b"engine-grpc-wrong-model", default_stream_output_specs()).await;
let status = client
.generate(pb::GenerateRequest {
request_id: "test-wrong-model".to_string(),
model: "other-model".to_string(),
prompt: Some(pb::generate_request::Prompt::Text("hi".to_string())),
stopping: Some(pb::StoppingCriteria {
max_new_tokens: 10,
..Default::default()
}),
..Default::default()
})
.await
.expect_err("should fail with wrong model");
assert_eq!(status.code(), tonic::Code::NotFound);
assert!(status.message().contains("other-model"));
server_task.abort();
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
#[serial]
async fn streaming_generate_rejects_wrong_model() {
let (mut client, server_task, _engine_task) = grpc_test_server(
b"engine-grpc-stream-wrong-model",
default_stream_output_specs(),
)
.await;
let status = client
.generate_stream(pb::GenerateRequest {
request_id: "test-stream-wrong-model".to_string(),
model: "other-model".to_string(),
prompt: Some(pb::generate_request::Prompt::Text("hi".to_string())),
stopping: Some(pb::StoppingCriteria {
max_new_tokens: 10,
..Default::default()
}),
..Default::default()
})
.await
.expect_err("should fail with wrong model");
assert_eq!(status.code(), tonic::Code::NotFound);
assert!(status.message().contains("other-model"));
server_task.abort();
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
#[serial]
async fn unary_generate_accepts_empty_model() {
let (mut client, server_task, engine_task) =
grpc_test_server(b"engine-grpc-empty-model", default_stream_output_specs()).await;
// Empty `model` (proto3 default) is treated as "unset" and should be accepted.
let response = client
.generate(pb::GenerateRequest {
request_id: "test-empty-model".to_string(),
model: String::new(),
prompt: Some(pb::generate_request::Prompt::Text("hi".to_string())),
stopping: Some(pb::StoppingCriteria {
max_new_tokens: 10,
..Default::default()
}),
..Default::default()
})
.await
.expect("unary generate with empty model")
.into_inner();
let outputs = response.outputs.expect("outputs present");
assert_eq!(outputs.text, "hi");
engine_task.await.expect("mock engine task");
server_task.abort();
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
#[serial]
async fn unary_generate_output_text_defaults_to_true() {
let (mut client, server_task, engine_task) =
grpc_test_server(b"engine-grpc-default-text", default_stream_output_specs()).await;
// No response options at all — output_text should default to true.
let response = client
.generate(pb::GenerateRequest {
request_id: "test-default-text".to_string(),
model: "test-model".to_string(),
prompt: Some(pb::generate_request::Prompt::Text("x".to_string())),
stopping: Some(pb::StoppingCriteria {
max_new_tokens: 10,
..Default::default()
}),
..Default::default()
})
.await
.expect("unary generate")
.into_inner();
let outputs = response.outputs.expect("outputs present");
assert_eq!(outputs.text, "hi");
engine_task.await.expect("mock engine task");
server_task.abort();
}
+237
View File
@@ -0,0 +1,237 @@
//! Minimal OpenAI-compatible HTTP server above [`vllm_chat`].
mod config;
mod error;
mod grpc;
mod listener;
mod middleware;
mod routes;
mod state;
mod utils;
use std::sync::{Arc, OnceLock};
use anyhow::{Context as _, Result};
use axum::serve::ListenerExt as _;
pub use config::{Config, CoordinatorMode, HttpListenerMode};
use tokio::net::TcpListener;
use tokio::time::{Instant, sleep_until};
use tokio_stream::wrappers::TcpListenerStream;
use tokio_util::either::Either;
use tokio_util::sync::CancellationToken;
use tonic::transport::Server as TonicServer;
use tracing::{info, trace, warn};
use vllm_chat::{ChatLlm, LoadModelBackendsOptions, load_model_backends};
pub use vllm_chat::{ChatTemplateContentFormatOption, ParserSelection, RendererSelection};
use vllm_engine_core_client::{EngineCoreClient, EngineCoreClientConfig};
use vllm_llm::Llm;
use vllm_text::TextLlm;
use crate::listener::Listener;
use crate::routes::build_router;
use crate::state::AppState;
/// Build the shared application state for one configured model and one engine
/// client.
async fn build_state(config: &Config) -> Result<Arc<AppState>> {
// Load both backends from the same model metadata so they stay in sync.
let loaded = load_model_backends(
&config.model,
LoadModelBackendsOptions {
renderer: config.renderer,
chat_template: config.chat_template.clone(),
chat_template_content_format: config.chat_template_content_format,
default_chat_template_kwargs: config
.default_chat_template_kwargs
.clone()
.unwrap_or_default(),
},
)
.await
.context("failed to create chat/text backends")?;
let text_backend = loaded.text_backend;
let chat_backend = loaded.chat_backend;
let coordinator_mode = config.effective_coordinator_mode(text_backend.is_moe());
info!(
engine_count = config.engine_count(),
model_is_moe = text_backend.is_moe(),
?coordinator_mode,
"resolved coordinator mode"
);
let client = EngineCoreClient::connect(EngineCoreClientConfig {
transport_mode: config.transport_mode.clone(),
coordinator_mode,
model_name: config.model.clone(),
client_index: 0,
})
.await
.context("failed to connect to engine core")?;
let llm = Llm::new(client).with_log_stats(!config.disable_log_stats);
let text = TextLlm::new(llm, text_backend);
let chat = ChatLlm::new(text, chat_backend)
.with_tool_call_parser(config.tool_call_parser.clone())
.with_reasoning_parser(config.reasoning_parser.clone());
// If no served names are specified, fall back to the backend model path so
// that the API always has at least one valid model ID.
let served_model_names = if config.served_model_name.is_empty() {
vec![config.model.clone()]
} else {
config.served_model_name.clone()
};
Ok(Arc::new(
AppState::new(served_model_names, chat).with_log_requests(config.enable_log_requests),
))
}
/// Run the OpenAI-compatible HTTP server until the supplied shutdown token is
/// cancelled.
///
/// The server owns one `vllm-chat` facade, which in turn owns the lower
/// `vllm-text` and `vllm-llm` layers, and shuts them down before returning.
pub async fn serve(config: Config, shutdown: CancellationToken) -> Result<()> {
config.validate().context("invalid OpenAI frontend configuration")?;
// Also check shutdown during the (potentially long) startup handshake.
let state = tokio::select! {
result = build_state(&config) => result?,
_ = shutdown.cancelled() => return Ok(()),
};
let listener = Listener::bind(&config.listener_mode)
.await
.context("failed to bind listener for OpenAI server")?;
let bind_address = listener.local_addr()?;
let model = state.primary_model_name().to_owned();
let app = build_router(state.clone());
// Optionally bind the gRPC Generate server on a separate port. Bind
// synchronously here so bind errors (port in use, permission denied, ...)
// surface before we start serving, rather than being deferred until
// shutdown. The gRPC listener follows the same host as the HTTP listener so
// that enabling --grpc-port does not accidentally expose the service on all
// interfaces when HTTP is intentionally local-only.
let grpc_setup = if let Some(grpc_port) = config.grpc_port {
let grpc_host = match &config.listener_mode {
HttpListenerMode::BindTcp { host, .. } => host.as_str(),
HttpListenerMode::BindUnix { .. } | HttpListenerMode::InheritedFd { .. } => "0.0.0.0",
};
let grpc_listener = TcpListener::bind((grpc_host, grpc_port))
.await
.with_context(|| format!("failed to bind gRPC listener on {grpc_host}:{grpc_port}"))?;
let addr = grpc_listener.local_addr()?;
let svc = grpc::GenerateServer::new(grpc::GenerateServiceImpl::new(state.clone()));
info!(%addr, "starting gRPC server");
Some((grpc_listener, svc))
} else {
None
};
info!(%bind_address, %model, "starting OpenAI server");
// Set TCP_NODELAY on accepted connections to reduce latency.
// By `tap_io` we will do this on every accepted connection.
let listener = listener.tap_io(|io| {
if let Either::Left(tcp_stream) = io
&& let Err(err) = tcp_stream.set_nodelay(true)
{
trace!(error = %err, "failed to enable TCP_NODELAY on accepted HTTP connection");
}
});
// Run HTTP and gRPC concurrently under a child token of the caller's shutdown
// token. Caller cancellation propagates into both protocols; if either
// protocol exits first, we cancel this child token so its sibling also
// begins a graceful drain.
let server_shutdown = shutdown.child_token();
let force_shutdown = CancellationToken::new();
let shutdown_deadline = Arc::new(OnceLock::new());
// Spawn a task to trigger `force_shutdown` after shutdown deadline elapses.
tokio::spawn({
let shutdown = server_shutdown.clone();
let force_shutdown = force_shutdown.clone();
let shutdown_deadline = shutdown_deadline.clone();
let shutdown_timeout = config.shutdown_timeout;
async move {
shutdown.cancelled().await;
let deadline = Instant::now() + shutdown_timeout;
let _ = shutdown_deadline.set(deadline);
if shutdown_timeout.is_zero() {
force_shutdown.cancel();
} else {
sleep_until(deadline).await;
force_shutdown.cancel();
}
}
});
let http_fut = {
let shutdown = server_shutdown.child_token();
let server_shutdown = server_shutdown.clone();
let force_shutdown = force_shutdown.clone();
async move {
let server =
axum::serve(listener, app).with_graceful_shutdown(shutdown.cancelled_owned());
let result = tokio::select! {
result = server => {
result.context("HTTP server failed")
}
_ = force_shutdown.cancelled() => {
warn!("HTTP graceful shutdown deadline elapsed; aborting server");
Ok(())
}
};
server_shutdown.cancel();
result
}
};
let grpc_fut = {
let shutdown = server_shutdown.child_token();
let server_shutdown = server_shutdown.clone();
let force_shutdown = force_shutdown.clone();
async move {
let Some((grpc_listener, svc)) = grpc_setup else {
// No gRPC configured: just wait for shutdown so we do not race the
// join! by resolving early and tripping the cancellation token.
shutdown.cancelled().await;
return Ok(());
};
let server = TonicServer::builder().add_service(svc).serve_with_incoming_shutdown(
TcpListenerStream::new(grpc_listener),
shutdown.cancelled_owned(),
);
let result = tokio::select! {
result = server => {
result.context("gRPC server failed")
}
_ = force_shutdown.cancelled() => {
warn!("gRPC graceful shutdown deadline elapsed; aborting server");
Ok(())
}
};
server_shutdown.cancel();
result
}
};
let (http_res, grpc_res) = tokio::join!(http_fut, grpc_fut);
http_res.and(grpc_res)?;
let shutdown_deadline = shutdown_deadline
.get()
.copied()
.unwrap_or_else(|| Instant::now() + config.shutdown_timeout);
state.shutdown(shutdown_deadline).await
}
+135
View File
@@ -0,0 +1,135 @@
//! Unified HTTP listener wrapper for the Rust frontend.
//!
//! This module hides the difference between TCP and Unix-domain listeners so
//! the rest of the server can bind or inherit one socket and pass it to
//! `axum::serve(...)` through a single type.
use std::io::Result;
use std::net::TcpListener as StdTcpListener;
use std::os::fd::{FromRawFd, IntoRawFd, OwnedFd};
use std::os::unix::net::UnixListener as StdUnixListener;
use socket2::Socket;
use tokio::net::{TcpListener, TcpStream, UnixListener, UnixStream};
use tokio_util::either::Either;
use crate::HttpListenerMode;
/// Runtime listener type used by the OpenAI-compatible HTTP server, which is
/// either a TCP listener or a Unix-domain listener.
#[derive(Debug)]
pub enum Listener {
Tcp(TcpListener),
Unix(UnixListener),
}
impl Listener {
/// Bind or adopt the listener described by the frontend configuration.
///
/// For inherited sockets, the concrete listener kind is detected from the
/// socket family of the supplied file descriptor.
pub async fn bind(mode: &HttpListenerMode) -> Result<Self> {
match mode {
HttpListenerMode::BindTcp { host, port } => {
Ok(Self::Tcp(TcpListener::bind((host.as_str(), *port)).await?))
}
HttpListenerMode::BindUnix { path } => Ok(Self::Unix(UnixListener::bind(path)?)),
HttpListenerMode::InheritedFd { fd } => Self::from_inherited_fd(*fd),
}
}
/// Return a log-friendly local address string for either TCP or Unix
/// sockets.
pub fn local_addr(&self) -> Result<String> {
match self {
Self::Tcp(listener) => Ok(listener.local_addr()?.to_string()),
Self::Unix(listener) => Ok(match listener.local_addr()?.as_pathname() {
Some(path) => format!("unix:{}", path.display()),
None => "unix:<unnamed>".to_string(),
}),
}
}
fn from_inherited_fd(fd: i32) -> Result<Self> {
// SAFETY: We trust the caller to only pass valid listener fds, and we only use
// this fd once to create a single listener.
let owned_fd = unsafe { OwnedFd::from_raw_fd(fd) };
let socket = Socket::from(owned_fd);
// The Python supervisor pre-binds the socket to reserve the endpoint early, but
// Rust is responsible for transitioning inherited stream sockets into
// the listening state before accepting connections.
socket.listen(libc::SOMAXCONN)?;
socket.set_nonblocking(true)?;
if socket.local_addr()?.is_unix() {
let std_listener = unsafe { StdUnixListener::from_raw_fd(socket.into_raw_fd()) };
Ok(Self::Unix(UnixListener::from_std(std_listener)?))
} else {
let std_listener = unsafe { StdTcpListener::from_raw_fd(socket.into_raw_fd()) };
Ok(Self::Tcp(TcpListener::from_std(std_listener)?))
}
}
}
/// Allow the unified listener to plug directly into `axum::serve(...)`.
impl axum::serve::Listener for Listener {
type Addr = Either<std::net::SocketAddr, tokio::net::unix::SocketAddr>;
type Io = Either<TcpStream, UnixStream>;
async fn accept(&mut self) -> (Self::Io, Self::Addr) {
match self {
Self::Tcp(listener) => {
let (io, addr) = listener.accept().await;
(Either::Left(io), Either::Left(addr))
}
Self::Unix(listener) => {
let (io, addr) = listener.accept().await;
(Either::Right(io), Either::Right(addr))
}
}
}
fn local_addr(&self) -> Result<Self::Addr> {
match self {
Self::Tcp(listener) => listener.local_addr().map(Either::Left),
Self::Unix(listener) => listener.local_addr().map(Either::Right),
}
}
}
#[cfg(test)]
mod tests {
use std::net::{Ipv4Addr, SocketAddrV4};
use std::os::fd::IntoRawFd;
use socket2::{Domain, SockAddr, Socket, Type};
use uuid::Uuid;
use super::Listener;
use crate::HttpListenerMode;
#[tokio::test(flavor = "current_thread")]
async fn inherited_fd_detects_tcp_listener_without_uds_hint() {
let socket = Socket::new(Domain::IPV4, Type::STREAM, None).unwrap();
socket.bind(&SockAddr::from(SocketAddrV4::new(Ipv4Addr::LOCALHOST, 0))).unwrap();
let fd = socket.into_raw_fd();
let listener = Listener::bind(&HttpListenerMode::InheritedFd { fd }).await.unwrap();
assert!(matches!(listener, Listener::Tcp(_)));
}
#[tokio::test(flavor = "current_thread")]
async fn inherited_fd_detects_unix_listener_from_fd() {
let path = std::env::temp_dir().join(format!("vllm-rs-{}.sock", Uuid::new_v4()));
let socket = Socket::new(Domain::UNIX, Type::STREAM, None).unwrap();
socket.bind(&SockAddr::unix(&path).unwrap()).unwrap();
let fd = socket.into_raw_fd();
let listener = Listener::bind(&HttpListenerMode::InheritedFd { fd }).await.unwrap();
assert!(matches!(listener, Listener::Unix(_)));
let _ = std::fs::remove_file(path);
}
}
+110
View File
@@ -0,0 +1,110 @@
use std::pin::Pin;
use std::sync::{Arc, Weak};
use std::task::{Context, Poll};
use axum::body::{Body, Bytes, HttpBody};
use axum::extract::{MatchedPath, Request, State};
use axum::middleware::Next;
use axum::response::Response;
use http_body::{Frame, SizeHint};
use crate::state::AppState;
/// Endpoints that will be tracked for server load.
///
/// Derived from the Python frontend's actual `@load_aware_call` coverage. This
/// includes alias paths that delegate into decorated handlers, such as
/// `/v1/rerank` and `/v2/rerank`.
const TRACKED_HANDLERS: &[&str] = &[
"/v1/responses",
"/v1/responses/{response_id}",
"/v1/responses/{response_id}/cancel",
"/v1/messages",
"/v1/messages/count_tokens",
"/v1/chat/completions",
"/v1/completions",
"/v1/audio/transcriptions",
"/v1/audio/translations",
"/v1/embeddings",
"/pooling",
"/classify",
"/score",
"/v1/score",
"/rerank",
"/v1/rerank",
"/v2/rerank",
"/inference/v1/generate",
];
/// Track frontend-local in-flight inference requests for the `/load` endpoint.
pub async fn track_server_load(
State(state): State<Arc<AppState>>,
req: Request,
next: Next,
) -> Response {
let handler = req
.extensions()
.get::<MatchedPath>()
.map_or_else(|| "none", |path| path.as_str());
if !TRACKED_HANDLERS.contains(&handler) {
return next.run(req).await;
}
state.increment_server_load();
let guard = ServerLoadGuard {
state: Arc::downgrade(&state),
};
let response = next.run(req).await;
let (parts, body) = response.into_parts();
Response::from_parts(
parts,
Body::new(LoadTrackedBody {
inner: body,
_guard: guard,
}),
)
}
/// A guard that decrements the server load when dropped.
struct ServerLoadGuard {
state: Weak<AppState>,
}
impl Drop for ServerLoadGuard {
fn drop(&mut self) {
if let Some(state) = self.state.upgrade() {
state.decrement_server_load();
}
}
}
/// A wrapper around response bodies that tracks server load by holding a
/// `ServerLoadGuard`, which will decrement the load when the body is fully
/// consumed and dropped.
struct LoadTrackedBody {
inner: Body,
_guard: ServerLoadGuard,
}
// Simply delegate all `HttpBody` methods to the inner body.
impl HttpBody for LoadTrackedBody {
type Data = Bytes;
type Error = axum::Error;
fn poll_frame(
mut self: Pin<&mut Self>,
cx: &mut Context<'_>,
) -> Poll<Option<Result<Frame<Self::Data>, Self::Error>>> {
Pin::new(&mut self.inner).poll_frame(cx)
}
fn is_end_stream(&self) -> bool {
self.inner.is_end_stream()
}
fn size_hint(&self) -> SizeHint {
self.inner.size_hint()
}
}
+79
View File
@@ -0,0 +1,79 @@
use std::time::Instant;
use axum::extract::{MatchedPath, Request};
use axum::middleware::Next;
use axum::response::Response;
use vllm_metrics::{HttpHandlerLabels, HttpRequestLabels, METRICS};
/// Endpoints that will be excluded from HTTP metrics tracking.
///
/// Original Python definition:
/// <https://github.com/vllm-project/vllm/blob/bc2c0c86efb28e77677a3cfb8687e976914a313a/vllm/entrypoints/serve/instrumentator/metrics.py#L28-L38>
const EXCLUDED_HANDLERS: &[&str] = &[
"/metrics",
"/health",
"/load",
"/ping",
"/version",
"/server_info",
// Rust frontend extra:
"/reset_prefix_cache",
"/reset_mm_cache",
"/reset_encoder_cache",
"/collective_rpc",
"/sleep",
"/wake_up",
"/is_sleeping",
];
/// Record API-server HTTP metrics with Python-compatible
/// (`PrometheusFastApiInstrumentator` style) family names and labels.
pub async fn track_http_metrics(req: Request, next: Next) -> Response {
let method = req.method().as_str().to_string();
let handler = req
.extensions()
.get::<MatchedPath>()
.map_or_else(|| "none".to_string(), |path| path.as_str().to_string());
let excluded = EXCLUDED_HANDLERS.contains(&handler.as_str());
let started_at = Instant::now();
let response = next.run(req).await;
if excluded {
return response;
}
let elapsed = started_at.elapsed().as_secs_f64();
let status = status_group(response.status().as_u16());
let metrics = &METRICS.api_server;
metrics
.http_requests
.get_or_create(&HttpRequestLabels {
method: method.clone(),
status,
handler: handler.clone(),
})
.inc();
metrics
.http_request_duration_seconds
.get_or_create(&HttpHandlerLabels { method, handler })
.observe(elapsed);
metrics.http_request_duration_highr_seconds.observe(elapsed);
response
}
fn status_group(status: u16) -> &'static str {
match status / 100 {
1 => "1xx",
2 => "2xx",
3 => "3xx",
4 => "4xx",
5 => "5xx",
_ => "unknown",
}
}
+5
View File
@@ -0,0 +1,5 @@
mod load;
mod metrics;
pub use load::track_server_load;
pub use metrics::track_http_metrics;
+68
View File
@@ -0,0 +1,68 @@
mod cache;
mod collective_rpc;
mod health;
mod inference;
mod load;
mod metrics;
pub(crate) mod openai;
mod sleep;
use std::sync::Arc;
use axum::Router;
use axum::middleware::{from_fn, from_fn_with_state};
use axum::routing::{get, post};
use tower_http::trace::TraceLayer;
use crate::middleware;
use crate::state::AppState;
fn server_dev_mode_enabled() -> bool {
std::env::var("VLLM_SERVER_DEV_MODE")
.ok()
.and_then(|value| value.parse::<i64>().ok())
.is_some_and(|value| value != 0)
}
/// Build the minimal OpenAI-compatible router for one configured model.
pub fn build_router(state: Arc<AppState>) -> Router {
build_router_with_dev_mode(state, server_dev_mode_enabled())
}
fn build_router_with_dev_mode(state: Arc<AppState>, dev_mode_enabled: bool) -> Router {
let mut router = Router::new()
// Health & monitoring
.route("/health", get(health::health))
.route("/metrics", get(metrics::scrape))
.route("/load", get(load::load))
// OpenAI-compatible endpoints
.route("/v1/models", get(openai::list_models))
.route("/v1/completions", post(openai::completions))
.route("/v1/chat/completions", post(openai::chat_completions))
// vLLM specific inference endpoints
.route("/inference/v1/generate", post(inference::generate));
if dev_mode_enabled {
// Development-only
router = router
.route("/reset_prefix_cache", post(cache::reset_prefix_cache))
.route("/reset_mm_cache", post(cache::reset_mm_cache))
.route("/reset_encoder_cache", post(cache::reset_encoder_cache))
.route("/collective_rpc", post(collective_rpc::collective_rpc))
.route("/sleep", post(sleep::sleep))
.route("/wake_up", post(sleep::wake_up))
.route("/is_sleeping", get(sleep::is_sleeping))
}
router
.with_state(state.clone())
.layer(from_fn_with_state(state, middleware::track_server_load))
.layer(from_fn(middleware::track_http_metrics))
.layer(TraceLayer::new_for_http())
}
#[cfg(test)]
mod tests;
#[cfg(test)]
mod http_client_tests;
+56
View File
@@ -0,0 +1,56 @@
use std::sync::Arc;
use axum::extract::{Query, State};
use axum::http::StatusCode;
use serde::Deserialize;
use crate::error::ApiError;
use crate::state::AppState;
use crate::utils::utility_call_error;
#[derive(Debug, Default, Deserialize)]
pub(crate) struct ResetPrefixCacheParams {
#[serde(default)]
reset_running_requests: bool,
#[serde(default)]
reset_external: bool,
}
/// Reset the local prefix cache and optionally the connector-managed external
/// cache.
pub async fn reset_prefix_cache(
State(state): State<Arc<AppState>>,
Query(params): Query<ResetPrefixCacheParams>,
) -> Result<StatusCode, ApiError> {
state
.engine_core_client()
.reset_prefix_cache(params.reset_running_requests, params.reset_external)
.await
.map_err(|error| utility_call_error("reset_prefix_cache", error))?;
Ok(StatusCode::OK)
}
/// Reset the multi-modal cache.
pub async fn reset_mm_cache(State(state): State<Arc<AppState>>) -> Result<StatusCode, ApiError> {
state
.engine_core_client()
.reset_mm_cache()
.await
.map_err(|error| utility_call_error("reset_mm_cache", error))?;
Ok(StatusCode::OK)
}
/// Reset the encoder cache.
pub async fn reset_encoder_cache(
State(state): State<Arc<AppState>>,
) -> Result<StatusCode, ApiError> {
state
.engine_core_client()
.reset_encoder_cache()
.await
.map_err(|error| utility_call_error("reset_encoder_cache", error))?;
Ok(StatusCode::OK)
}
@@ -0,0 +1,51 @@
use std::collections::BTreeMap;
use std::sync::Arc;
use axum::Json;
use axum::extract::State;
use axum::extract::rejection::JsonRejection;
use rmpv::Value as MsgpackValue;
use serde::{Deserialize, Serialize};
use serde_json::Value as JsonValue;
use crate::error::ApiError;
use crate::state::AppState;
use crate::utils::utility_call_error;
#[derive(Debug, Deserialize)]
pub(crate) struct CollectiveRpcRequest {
method: Option<String>,
#[serde(default)]
timeout: Option<f64>,
#[serde(default)]
args: Vec<JsonValue>,
#[serde(default)]
kwargs: BTreeMap<String, JsonValue>,
}
#[derive(Debug, Serialize)]
pub(crate) struct CollectiveRpcResponse {
results: Vec<MsgpackValue>,
}
/// Execute a development-only collective RPC on the connected engine(s).
pub async fn collective_rpc(
State(state): State<Arc<AppState>>,
body: Result<Json<CollectiveRpcRequest>, JsonRejection>,
) -> Result<Json<CollectiveRpcResponse>, ApiError> {
let Json(body) = body.map_err(|error| ApiError::json_parse_error(error.body_text()))?;
let method = body.method.ok_or_else(|| {
ApiError::invalid_request(
"Missing 'method' in request body".to_string(),
Some("method"),
)
})?;
let results = state
.engine_core_client()
.collective_rpc(&method, body.timeout, body.args, body.kwargs)
.await
.map_err(|error| utility_call_error("collective_rpc", error))?;
Ok(Json(CollectiveRpcResponse { results }))
}
+14
View File
@@ -0,0 +1,14 @@
use std::sync::Arc;
use axum::extract::State;
use axum::http::StatusCode;
use crate::state::AppState;
pub async fn health(State(state): State<Arc<AppState>>) -> StatusCode {
if state.chat.engine_core_client().is_healthy() {
StatusCode::OK
} else {
StatusCode::SERVICE_UNAVAILABLE
}
}
@@ -0,0 +1,392 @@
//! Integration tests that exercise the OpenAI-compatible HTTP API through a
//! real TCP connection using the `async-openai` client library, backed by a
//! mock engine.
use std::future::Future;
use std::pin::Pin;
use std::sync::Arc;
use std::task::{Context, Poll};
use async_openai::Client;
use async_openai::config::OpenAIConfig;
use async_openai::types::chat::{
ChatCompletionRequestUserMessageArgs, CreateChatCompletionRequestArgs,
};
use futures::StreamExt as _;
use serial_test::serial;
use vllm_chat::{
ChatBackend, ChatLlm, ChatRenderer, ChatRequest, ChatTextBackend, DefaultChatOutputProcessor,
DynChatOutputProcessor, DynChatRenderer, NewChatOutputProcessorOptions, RenderedPrompt,
};
use vllm_engine_core_client::protocol::{
EngineCoreFinishReason, EngineCoreOutput, EngineCoreOutputs, EngineCoreRequest,
};
use vllm_engine_core_client::test_utils::{IpcNamespace, spawn_mock_engine_task};
use vllm_engine_core_client::{EngineCoreClient, EngineCoreClientConfig, EngineId};
use vllm_llm::Llm;
use vllm_text::tokenizer::{DynTokenizer, Tokenizer};
use vllm_text::{Prompt, TextBackend};
use zeromq::prelude::{SocketRecv, SocketSend};
use zeromq::{DealerSocket, PushSocket, ZmqMessage};
use crate::routes::build_router;
use crate::state::AppState;
// ========================================================================================
// Test infrastructure (mirrors routes/tests.rs helpers)
// ========================================================================================
type TestFuture<'a> = Pin<Box<dyn Future<Output = ()> + Send + 'a>>;
fn boxed_test_future<'a>(future: impl Future<Output = ()> + Send + 'a) -> TestFuture<'a> {
Box::pin(future)
}
struct MockEngineTask {
shutdown_tx: Option<tokio::sync::oneshot::Sender<()>>,
join_handle: Option<tokio::task::JoinHandle<()>>,
}
impl MockEngineTask {
fn new(
(shutdown_tx, join_handle): (
tokio::sync::oneshot::Sender<()>,
tokio::task::JoinHandle<()>,
),
) -> Self {
Self {
shutdown_tx: Some(shutdown_tx),
join_handle: Some(join_handle),
}
}
}
impl Future for MockEngineTask {
type Output = Result<(), tokio::task::JoinError>;
fn poll(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Self::Output> {
if let Some(shutdown_tx) = self.shutdown_tx.take() {
let _ = shutdown_tx.send(());
}
match self.join_handle.as_mut() {
Some(join_handle) => Pin::new(join_handle).poll(cx),
None => Poll::Ready(Ok(())),
}
}
}
impl Drop for MockEngineTask {
fn drop(&mut self) {
if let Some(join_handle) = &self.join_handle {
join_handle.abort();
}
}
}
fn request_output(
request_id: &str,
new_token_ids: Vec<u32>,
finish_reason: Option<EngineCoreFinishReason>,
) -> EngineCoreOutput {
EngineCoreOutput {
request_id: request_id.to_string(),
new_token_ids,
new_logprobs: None,
new_prompt_logprobs_tensors: None,
pooling_output: None,
finish_reason,
stop_reason: None,
events: None,
kv_transfer_params: None,
trace_headers: None,
prefill_stats: None,
routed_experts: None,
num_nans_in_logits: 0,
}
}
fn engine_outputs_for_request(
request_id: &str,
output_specs: Vec<(Vec<u32>, Option<EngineCoreFinishReason>)>,
) -> EngineCoreOutputs {
EngineCoreOutputs {
engine_index: 0,
outputs: output_specs
.into_iter()
.map(|(token_ids, finish_reason)| request_output(request_id, token_ids, finish_reason))
.collect(),
scheduler_stats: None,
timestamp: 0.0,
utility_output: None,
finished_requests: None,
wave_complete: None,
start_wave: None,
}
}
fn default_stream_output_specs() -> Vec<(Vec<u32>, Option<EngineCoreFinishReason>)> {
vec![
(vec![b'h' as u32], None),
(vec![b'i' as u32], None),
(vec![b'!' as u32], Some(EngineCoreFinishReason::Stop)),
]
}
async fn send_outputs(push: &mut PushSocket, outputs: EngineCoreOutputs) {
push.send(ZmqMessage::from(
rmp_serde::to_vec_named(&outputs).expect("encode outputs"),
))
.await
.expect("send outputs");
}
async fn recv_engine_message(dealer: &mut DealerSocket) -> Vec<bytes::Bytes> {
dealer.recv().await.expect("recv engine message").into_vec()
}
fn test_llm(client: EngineCoreClient) -> Llm {
Llm::new(client).with_request_id_randomization(false)
}
#[derive(Clone, Debug)]
struct FakeChatBackend;
#[derive(Debug)]
struct FakeChatTokenizer;
impl Tokenizer for FakeChatTokenizer {
fn encode(
&self,
text: &str,
_add_special_tokens: bool,
) -> vllm_text::tokenizer::Result<Vec<u32>> {
Ok(text.bytes().map(u32::from).collect())
}
fn decode(
&self,
token_ids: &[u32],
_skip_special_tokens: bool,
) -> vllm_text::tokenizer::Result<String> {
Ok(
String::from_utf8_lossy(&token_ids.iter().map(|id| *id as u8).collect::<Vec<_>>())
.into_owned(),
)
}
fn token_to_id(&self, token: &str) -> Option<u32> {
token.bytes().next().map(u32::from)
}
}
impl TextBackend for FakeChatBackend {
fn tokenizer(&self) -> DynTokenizer {
Arc::new(FakeChatTokenizer)
}
fn model_id(&self) -> &str {
"test-model"
}
}
impl ChatBackend for FakeChatBackend {
fn chat_renderer(&self) -> DynChatRenderer {
Arc::new(self.clone())
}
fn new_chat_output_processor(
&self,
request: &mut ChatRequest,
options: NewChatOutputProcessorOptions<'_>,
) -> vllm_chat::Result<DynChatOutputProcessor> {
Ok(Box::new(DefaultChatOutputProcessor::new(
request,
self.model_id(),
self.tokenizer(),
options.tool_call_parser,
options.reasoning_parser,
)?))
}
}
impl ChatRenderer for FakeChatBackend {
fn render(&self, request: &ChatRequest) -> vllm_chat::Result<RenderedPrompt> {
let mut prompt = String::new();
for message in &request.messages {
prompt.push_str(message.role().as_str());
prompt.push_str(": ");
prompt.push_str(&message.text_content()?);
prompt.push('\n');
}
if request.chat_options.add_generation_prompt() {
prompt.push_str("assistant:");
}
Ok(RenderedPrompt {
prompt: Prompt::Text(prompt),
})
}
}
/// Spin up an HTTP server on a random port backed by a mock engine.
/// Returns the `async-openai` client, the HTTP server task, and the mock engine
/// task.
async fn http_test_server(
engine_id: impl Into<EngineId>,
output_specs: Vec<(Vec<u32>, Option<EngineCoreFinishReason>)>,
) -> (
Client<OpenAIConfig>,
tokio::task::JoinHandle<()>,
MockEngineTask,
) {
let ipc = IpcNamespace::new().expect("create ipc namespace");
let handshake_address = ipc.handshake_endpoint();
let engine_id = engine_id.into();
let engine_task = MockEngineTask::new(spawn_mock_engine_task(
handshake_address.clone(),
engine_id.clone(),
move |dealer, push| {
boxed_test_future(async move {
let add = recv_engine_message(dealer).await;
let request: EngineCoreRequest =
rmp_serde::from_slice(&add[1]).expect("decode request");
send_outputs(
push,
engine_outputs_for_request(&request.request_id, output_specs),
)
.await;
})
},
));
let client = EngineCoreClient::connect(
EngineCoreClientConfig::new_single(handshake_address)
.with_model_name("test-model")
.with_local_input_output_addresses(
Some(ipc.input_endpoint()),
Some(ipc.output_endpoint()),
),
)
.await
.expect("connect client");
let chat = ChatLlm::from_shared_backend(
test_llm(client),
Arc::new(FakeChatBackend) as Arc<dyn ChatTextBackend>,
);
let state = Arc::new(AppState::new(vec!["test-model".to_string()], chat));
let app = build_router(state);
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.expect("bind http listener");
let addr = listener.local_addr().expect("local addr");
let server_task = tokio::spawn(async move {
axum::serve(listener, app).await.expect("http server");
});
let openai_client = Client::with_config(
OpenAIConfig::new()
.with_api_key("unused")
.with_api_base(format!("http://{addr}/v1")),
);
(openai_client, server_task, engine_task)
}
// ========================================================================================
// Tests
// ========================================================================================
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
#[serial]
async fn list_models_via_http_client() {
let (client, server_task, _engine_task) =
http_test_server(b"engine-http-models", default_stream_output_specs()).await;
let models = client.models().list().await.expect("list models");
let model_ids: Vec<&str> = models.data.iter().map(|m| m.id.as_str()).collect();
assert_eq!(model_ids, vec!["test-model"]);
server_task.abort();
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
#[serial]
async fn non_streaming_chat_via_http_client() {
let (client, server_task, engine_task) =
http_test_server(b"engine-http-chat", default_stream_output_specs()).await;
let request = CreateChatCompletionRequestArgs::default()
.model("test-model")
.stream(false)
.max_completion_tokens(10u32)
.messages([ChatCompletionRequestUserMessageArgs::default()
.content("hello")
.build()
.expect("build user message")
.into()])
.build()
.expect("build request");
let response = client.chat().create(request).await.expect("chat completion");
assert_eq!(response.model, "test-model");
assert_eq!(response.choices.len(), 1);
let choice = &response.choices[0];
// The stop token `!` is suppressed from text.
assert_eq!(choice.message.content.as_deref(), Some("hi"));
assert_eq!(
choice.finish_reason,
Some(async_openai::types::chat::FinishReason::Stop)
);
engine_task.await.expect("mock engine task");
server_task.abort();
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
#[serial]
async fn streaming_chat_via_http_client() {
let (client, server_task, engine_task) =
http_test_server(b"engine-http-stream", default_stream_output_specs()).await;
let request = CreateChatCompletionRequestArgs::default()
.model("test-model")
.stream(true)
.max_completion_tokens(10u32)
.messages([ChatCompletionRequestUserMessageArgs::default()
.content("hello")
.build()
.expect("build user message")
.into()])
.build()
.expect("build request");
let mut stream = client.chat().create_stream(request).await.expect("streaming chat completion");
let mut full_text = String::new();
let mut saw_role = false;
let mut saw_finish_reason = false;
while let Some(chunk) = stream.next().await {
let chunk = chunk.expect("stream chunk");
for choice in &chunk.choices {
if choice.delta.role.is_some() {
saw_role = true;
}
if let Some(ref delta) = choice.delta.content {
full_text.push_str(delta);
}
if choice.finish_reason.is_some() {
saw_finish_reason = true;
}
}
}
assert!(saw_role, "expected an assistant role chunk");
assert!(saw_finish_reason, "expected a terminal finish reason");
assert_eq!(full_text, "hi");
engine_task.await.expect("mock engine task");
server_task.abort();
}
@@ -0,0 +1,215 @@
mod convert;
mod types;
mod validate;
use std::collections::HashMap;
use std::sync::Arc;
use axum::Json;
use axum::extract::State;
use axum::http::HeaderMap;
use axum::response::{IntoResponse, Response};
use thiserror_ext::AsReport as _;
use tracing::info;
use tracing_futures::Instrument as _;
use vllm_engine_core_client::protocol::logprobs::{Logprobs, PositionLogprobs};
use vllm_llm::{CollectedGenerateOutput, GenerateOutputStreamExt as _};
use self::convert::prepare_generate_request;
use self::types::{GenerateLogprob, GenerateRequest, GenerateResponse, GenerateResponseChoice};
use crate::error::{ApiError, server_error};
use crate::routes::openai::utils::logprobs::clamp_logprob;
use crate::routes::openai::utils::types::{ChatLogProbs, ChatLogProbsContent, TopLogProb};
use crate::routes::openai::utils::validated_json::ValidatedJson;
use crate::state::AppState;
use crate::utils::resolve_request_context;
/// Validate one token-in/token-out request and proxy it into the shared
/// `vllm-text` stack.
pub async fn generate(
State(state): State<Arc<AppState>>,
headers: HeaderMap,
ValidatedJson(body): ValidatedJson<GenerateRequest>,
) -> Response {
let request_context = resolve_request_context(&headers, body.request_id.as_deref());
let prepared = match prepare_generate_request(body, state.served_model_names(), request_context)
{
Ok(prepared) => prepared,
Err(error) => return error.into_response(),
};
let request_span = tracing::info_span!(
"generate",
request_id = %prepared.request_id,
engine_request_id = tracing::field::Empty,
);
let log_request = state.enable_log_requests;
let include_logprobs = prepared.include_logprobs;
let include_prompt_logprobs = prepared.include_prompt_logprobs;
let raw_stream = match state
.chat
.text()
.generate_raw(prepared.text_request)
.instrument(request_span.clone())
.await
{
Ok(stream) => stream,
Err(error) => {
return server_error!(
"failed to submit raw generate request: {}",
error.to_report_string()
)
.into_response();
}
};
let collected = match raw_stream.collect_output().instrument(request_span.clone()).await {
Ok(collected) => collected,
Err(error) => {
return server_error!(
"failed to collect raw generate response: {}",
error.to_report_string()
)
.into_response();
}
};
if log_request {
info!(
parent: &request_span,
prompt_tokens = collected.prompt_token_ids.len(),
output_tokens = collected.token_ids.len(),
finish_reason = collected.finish_reason.as_str(),
"generate finished"
);
}
let response = match collect_generate(
collected,
prepared.request_id,
include_logprobs,
include_prompt_logprobs,
) {
Ok(response) => response,
Err(error) => return error.into_response(),
};
Json(response).into_response()
}
fn collect_generate(
collected: CollectedGenerateOutput,
request_id: String,
include_logprobs: bool,
include_prompt_logprobs: bool,
) -> Result<GenerateResponse, ApiError> {
let logprobs = if include_logprobs {
let logprobs = collected.logprobs.as_ref().ok_or_else(|| {
ApiError::server_error(
"raw generate response requested logprobs but generation returned none".to_string(),
)
})?;
Some(raw_logprobs_to_openai_chat(logprobs)?)
} else {
None
};
let prompt_logprobs = if include_prompt_logprobs {
let prompt_logprobs = collected.prompt_logprobs.as_ref().ok_or_else(|| {
ApiError::server_error(
"raw generate response requested prompt_logprobs but generation returned none"
.to_string(),
)
})?;
Some(raw_prompt_logprobs_to_maps(prompt_logprobs))
} else {
None
};
Ok(GenerateResponse {
request_id,
choices: vec![GenerateResponseChoice {
index: 0,
logprobs,
finish_reason: Some(collected.finish_reason.as_str().to_string()),
token_ids: collected.token_ids,
}],
prompt_logprobs,
kv_transfer_params: collected.kv_transfer_params,
})
}
fn raw_logprobs_to_openai_chat(logprobs: &Logprobs) -> Result<ChatLogProbs, ApiError> {
let content = logprobs
.positions
.iter()
.map(position_to_chat_logprobs_content)
.collect::<Result<Vec<_>, _>>()?;
Ok(ChatLogProbs {
content: Some(content),
})
}
fn raw_prompt_logprobs_to_maps(
prompt_logprobs: &Logprobs,
) -> Vec<Option<HashMap<u32, GenerateLogprob>>> {
std::iter::once(None)
.chain(
prompt_logprobs
.positions
.iter()
.map(|position| Some(position_to_logprob_map(position))),
)
.collect()
}
fn position_to_chat_logprobs_content(
position: &PositionLogprobs,
) -> Result<ChatLogProbsContent, ApiError> {
let chosen = position.entries.first().ok_or_else(|| {
ApiError::server_error(
"raw generate logprobs position unexpectedly had no token candidates".to_string(),
)
})?;
let token = format_token_id(chosen.token_id);
Ok(ChatLogProbsContent {
token: token.clone(),
logprob: clamp_logprob(chosen.logprob),
bytes: Some(token.as_bytes().to_vec()),
top_logprobs: position
.entries
.iter()
.map(|entry| {
let token = format_token_id(entry.token_id);
TopLogProb {
token: token.clone(),
logprob: clamp_logprob(entry.logprob),
bytes: Some(token.into_bytes()),
}
})
.collect(),
})
}
fn position_to_logprob_map(position: &PositionLogprobs) -> HashMap<u32, GenerateLogprob> {
position
.entries
.iter()
.map(|entry| {
(
entry.token_id,
GenerateLogprob {
logprob: clamp_logprob(entry.logprob),
rank: Some(entry.rank),
decoded_token: Some(format_token_id(entry.token_id)),
},
)
})
.collect()
}
fn format_token_id(token_id: u32) -> String {
format!("token_id:{token_id}")
}
@@ -0,0 +1,112 @@
use vllm_text::{Prompt, TextDecodeOptions, TextRequest};
use super::types::GenerateRequest;
use super::validate;
use crate::error::ApiError;
use crate::utils::{ResolvedRequestContext, merge_kv_transfer_params};
/// Lowered generate request plus the response request ID.
#[derive(Debug, Clone, PartialEq)]
pub struct PreparedRequest {
pub request_id: String,
pub text_request: TextRequest,
pub include_logprobs: bool,
pub include_prompt_logprobs: bool,
}
/// Validate and lower one raw generate request into the internal
/// text-generation format.
pub fn prepare_generate_request(
request: GenerateRequest,
served_model_names: &[String],
ctx: ResolvedRequestContext,
) -> Result<PreparedRequest, ApiError> {
validate::validate_request_compat(&request, served_model_names)?;
let include_logprobs = request.sampling_params.logprobs.is_some();
let include_prompt_logprobs = request.sampling_params.prompt_logprobs.is_some();
let mut sampling_params = request.sampling_params;
sampling_params.vllm_xargs = merge_kv_transfer_params(
sampling_params.vllm_xargs,
request.kv_transfer_params.as_ref(),
);
let text_request = TextRequest {
request_id: ctx.request_id.clone(),
prompt: Prompt::TokenIds(request.token_ids),
mm_features: None,
sampling_params,
decode_options: TextDecodeOptions::default(),
intermediate: false,
priority: request.priority,
cache_salt: request.cache_salt,
add_special_tokens: false,
data_parallel_rank: ctx.data_parallel_rank,
};
Ok(PreparedRequest {
request_id: ctx.request_id,
text_request,
include_logprobs,
include_prompt_logprobs,
})
}
#[cfg(test)]
mod tests {
use serde_json::json;
use vllm_text::Prompt;
use super::prepare_generate_request;
use crate::routes::inference::generate::types::GenerateRequest;
use crate::utils::ResolvedRequestContext;
#[test]
fn prepare_generate_request_maps_token_prompt_and_sampling_params() {
let request: GenerateRequest = serde_json::from_value(json!({
"model": "Qwen/Qwen1.5-0.5B-Chat",
"token_ids": [11, 22, 33],
"priority": -3,
"cache_salt": "salt",
"sampling_params": {
"max_tokens": 7,
"logprobs": 2,
"prompt_logprobs": 1,
"ignore_eos": true
},
"kv_transfer_params": {
"connector": "x"
}
}))
.expect("parse request");
let prepared = prepare_generate_request(
request,
&["Qwen/Qwen1.5-0.5B-Chat".to_string()],
ResolvedRequestContext::default(),
)
.expect("prepare");
assert_eq!(
prepared.text_request.prompt,
Prompt::TokenIds(vec![11, 22, 33])
);
assert_eq!(prepared.text_request.sampling_params.max_tokens, Some(7));
assert_eq!(prepared.text_request.sampling_params.logprobs, Some(2));
assert_eq!(
prepared.text_request.sampling_params.prompt_logprobs,
Some(1)
);
assert!(prepared.text_request.sampling_params.ignore_eos);
assert_eq!(prepared.text_request.priority, -3);
assert_eq!(prepared.text_request.cache_salt.as_deref(), Some("salt"));
assert_eq!(
prepared
.text_request
.sampling_params
.vllm_xargs
.and_then(|mut xargs| xargs.remove("kv_transfer_params")),
Some(json!({"connector": "x"}))
);
}
}
@@ -0,0 +1,57 @@
use std::collections::HashMap;
use serde::{Deserialize, Serialize};
use serde_json::{Map, Value};
use validator::Validate;
use vllm_text::SamplingParams;
use crate::routes::openai::utils::types::{ChatLogProbs, Normalizable};
/// vLLM-compatible request type for the token-in/token-out generate API.
#[serde_with::skip_serializing_none]
#[derive(Debug, Clone, Deserialize, Serialize, Validate)]
pub struct GenerateRequest {
pub request_id: Option<String>,
pub model: Option<String>,
pub token_ids: Vec<u32>,
pub sampling_params: SamplingParams,
#[serde(default)]
pub stream: bool,
pub cache_salt: Option<String>,
#[serde(default)]
pub priority: i32,
pub kv_transfer_params: Option<HashMap<String, Value>>,
#[serde(flatten)]
pub other: Map<String, Value>,
}
impl Normalizable for GenerateRequest {}
/// Mirrors the Python vLLM `GenerateResponseChoice` class.
#[serde_with::skip_serializing_none]
#[derive(Debug, Clone, Serialize)]
pub(super) struct GenerateResponseChoice {
pub index: u32,
pub logprobs: Option<ChatLogProbs>,
pub finish_reason: Option<String>,
pub token_ids: Vec<u32>,
}
/// Mirrors the Python vLLM `GenerateResponse` class.
#[serde_with::skip_serializing_none]
#[derive(Debug, Clone, Serialize)]
pub(super) struct GenerateResponse {
pub request_id: String,
pub choices: Vec<GenerateResponseChoice>,
pub prompt_logprobs: Option<Vec<Option<HashMap<u32, GenerateLogprob>>>>,
pub kv_transfer_params: Option<Value>,
}
/// Mirrors the Python vLLM `Logprob` class used in prompt-logprobs payloads.
#[serde_with::skip_serializing_none]
#[derive(Debug, Clone, Serialize)]
pub(super) struct GenerateLogprob {
pub logprob: f32,
pub rank: Option<u32>,
pub decoded_token: Option<String>,
}
@@ -0,0 +1,84 @@
use super::types::GenerateRequest;
use crate::error::{ApiError, bail_invalid_request};
/// Enforce the minimal compatibility contract for the Rust token generate
/// route.
pub(super) fn validate_request_compat(
request: &GenerateRequest,
served_model_names: &[String],
) -> Result<(), ApiError> {
if let Some(model) = request.model.as_ref()
&& !served_model_names.iter().any(|n| n == model)
{
return Err(ApiError::model_not_found(model.clone()));
}
if request.stream {
bail_invalid_request!(param = "stream", "stream=true is not supported.");
}
if request.token_ids.is_empty() {
bail_invalid_request!(
param = "token_ids",
"token_ids must contain at least one token ID."
);
}
if request.sampling_params.max_tokens == Some(0) {
bail_invalid_request!(
param = "sampling_params",
"max_tokens must be greater than 0."
);
}
if let Some(prompt_logprobs) = request.sampling_params.prompt_logprobs
&& prompt_logprobs < 0
&& prompt_logprobs != -1
{
bail_invalid_request!(
param = "sampling_params",
"`prompt_logprobs` must be a non-negative value or -1."
);
}
Ok(())
}
#[cfg(test)]
mod tests {
use serde_json::json;
use super::validate_request_compat;
use crate::routes::inference::generate::types::GenerateRequest;
fn base_request() -> GenerateRequest {
serde_json::from_value(json!({
"model": "Qwen/Qwen1.5-0.5B-Chat",
"token_ids": [11, 22],
"sampling_params": {}
}))
.expect("parse request")
}
fn served(names: &[&str]) -> Vec<String> {
names.iter().map(|s| s.to_string()).collect()
}
#[test]
fn validate_request_compat_rejects_streaming() {
let request = GenerateRequest {
stream: true,
..base_request()
};
assert!(validate_request_compat(&request, &served(&["Qwen/Qwen1.5-0.5B-Chat"])).is_err());
}
#[test]
fn validate_request_compat_rejects_empty_token_ids() {
let request = GenerateRequest {
token_ids: Vec::new(),
..base_request()
};
assert!(validate_request_compat(&request, &served(&["Qwen/Qwen1.5-0.5B-Chat"])).is_err());
}
}
@@ -0,0 +1,3 @@
pub mod generate;
pub use generate::generate;
+18
View File
@@ -0,0 +1,18 @@
use std::sync::Arc;
use axum::Json;
use axum::extract::State;
use serde::Serialize;
use crate::state::AppState;
#[derive(Serialize)]
pub(crate) struct ServerLoadResponse {
server_load: u64,
}
pub async fn load(State(state): State<Arc<AppState>>) -> Json<ServerLoadResponse> {
Json(ServerLoadResponse {
server_load: state.server_load(),
})
}
+26
View File
@@ -0,0 +1,26 @@
use axum::http::header::CONTENT_TYPE;
use axum::http::{HeaderValue, StatusCode};
use axum::response::{IntoResponse, Response};
use thiserror_ext::AsReport;
use vllm_metrics::METRICS;
const OPENMETRICS_CONTENT_TYPE: &str = "application/openmetrics-text; version=1.0.0; charset=utf-8";
pub async fn scrape() -> Response {
match METRICS.render() {
Ok(body) => (
[(
CONTENT_TYPE,
HeaderValue::from_static(OPENMETRICS_CONTENT_TYPE),
)],
body,
)
.into_response(),
Err(error) => (
StatusCode::INTERNAL_SERVER_ERROR,
format!("failed to render metrics: {}", error.as_report()),
)
.into_response(),
}
}
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,992 @@
use itertools::Itertools as _;
use vllm_chat::{
AssistantContentBlock, AssistantToolCall, ChatContent, ChatContentPart,
ChatMessage as VllmChatMessage, ChatOptions, ChatRequest, ChatTool, ChatToolChoice,
GenerationPromptMode, SamplingParams,
};
use super::types::ChatCompletionRequest;
use super::validate;
use crate::error::{ApiError, bail_invalid_request};
use crate::routes::openai::utils::structured_outputs::convert_from_response_format;
use crate::routes::openai::utils::types::{
ChatMessage, ContentPart, MessageContent, Tool, ToolChoice, ToolChoiceValue,
};
use crate::utils::{ResolvedRequestContext, convert_logit_bias, merge_kv_transfer_params};
/// Lowered chat request plus the public response metadata carried by every SSE
/// chunk.
#[derive(Debug, Clone, PartialEq)]
pub struct PreparedRequest {
/// Stable OpenAI-style request ID, reused as the external chat request ID.
pub request_id: String,
/// Public model ID echoed back to the client.
pub response_model: String,
/// Whether the caller asked for the final streamed usage chunk.
pub include_usage: bool,
/// Whether the caller requested output logprobs on chat choices.
pub requested_logprobs: bool,
/// Whether the caller requested top-level prompt logprobs.
pub include_prompt_logprobs: bool,
/// Lowered chat request for `vllm-chat`.
pub chat_request: ChatRequest,
/// Last assistant-role message content to echo back when `echo=true`.
pub echo: Option<String>,
/// Whether to include token IDs alongside generated text.
pub return_token_ids: bool,
/// Whether to format logprob tokens as `token_id:{id}`.
pub return_tokens_as_token_ids: bool,
}
/// Validate and lower one OpenAI chat completion request into the internal chat
/// format.
///
/// `served_model_names` must be non-empty; the first entry is used as the
/// `model` field in responses.
pub(crate) fn prepare_chat_request(
request: ChatCompletionRequest,
served_model_names: &[String],
ctx: ResolvedRequestContext,
) -> Result<PreparedRequest, ApiError> {
validate::validate_request_compat(&request, served_model_names)?;
let request_id = format!("chatcmpl-{}", ctx.request_id);
let echo = request
.echo
.then(|| extract_last_assistant_content(&request.messages))
.flatten();
let messages: Vec<_> = request.messages.into_iter().map(convert_message).try_collect()?;
let generation_prompt_mode = normalize_generation_prompt_mode(
request.add_generation_prompt,
request.continue_final_message,
&messages,
)?;
let template_kwargs = request.chat_template_kwargs.unwrap_or_default();
let include_usage = (request.stream_options.as_ref())
.and_then(|options| options.include_usage)
.unwrap_or(false);
let requested_logprobs = request.logprobs;
// Auto-enable prompt logprobs for non-streaming echo, matching Python vLLM's
// behavior.
let top_logprobs = request.top_logprobs.unwrap_or(0);
let prompt_logprobs = request
.prompt_logprobs
.or((request.echo && !request.stream).then_some(top_logprobs));
let include_prompt_logprobs = prompt_logprobs.is_some();
let structured_outputs = convert_from_response_format(
request.response_format.as_ref(),
&request.structured_outputs,
)?;
let chat_request = ChatRequest {
request_id: request_id.clone(),
messages,
sampling_params: SamplingParams {
temperature: request.temperature,
top_p: request.top_p,
top_k: request.top_k,
seed: request.seed,
max_tokens: request.max_completion_tokens,
min_tokens: request.min_tokens,
logprobs: request.logprobs.then_some(top_logprobs),
prompt_logprobs,
min_p: request.min_p,
frequency_penalty: request.frequency_penalty,
presence_penalty: request.presence_penalty,
repetition_penalty: request.repetition_penalty,
stop_token_ids: request.stop_token_ids,
ignore_eos: request.ignore_eos,
logit_bias: convert_logit_bias(request.logit_bias)?,
allowed_token_ids: request.allowed_token_ids,
bad_words: request.bad_words,
logprob_token_ids: None,
structured_outputs,
skip_reading_prefix_cache: None,
vllm_xargs: merge_kv_transfer_params(
request.vllm_xargs,
request.kv_transfer_params.as_ref(),
),
},
chat_options: ChatOptions {
generation_prompt_mode,
chat_template: request.chat_template,
reasoning_effort: request.reasoning_effort,
template_kwargs,
},
tools: convert_tools(request.tools)?,
tool_choice: convert_tool_choice(request.tool_choice.as_ref())?,
decode_options: vllm_text::output::TextDecodeOptions {
skip_special_tokens: request.skip_special_tokens,
include_stop_str_in_output: request.include_stop_str_in_output,
stop_strings: request.stop.map(|stop| stop.into_vec()),
min_tokens: request.min_tokens.unwrap_or(0),
},
intermediate: request.stream,
priority: request.priority.unwrap_or(0),
documents: request.documents,
cache_salt: request.cache_salt,
add_special_tokens: request.add_special_tokens,
data_parallel_rank: ctx.data_parallel_rank,
};
Ok(PreparedRequest {
request_id,
response_model: served_model_names.first().cloned().unwrap_or_default(),
include_usage,
requested_logprobs,
include_prompt_logprobs,
chat_request,
echo,
return_token_ids: request.return_token_ids.unwrap_or(false),
return_tokens_as_token_ids: request.return_tokens_as_token_ids.unwrap_or(false),
})
}
fn normalize_generation_prompt_mode(
add_generation_prompt: Option<bool>,
continue_final_message: bool,
messages: &[VllmChatMessage],
) -> Result<GenerationPromptMode, ApiError> {
if add_generation_prompt == Some(true) && continue_final_message {
bail_invalid_request!(
"Cannot set both `continue_final_message` and `add_generation_prompt` to True."
);
}
let last_role = messages.last().map(VllmChatMessage::role);
match (add_generation_prompt, continue_final_message, last_role) {
(Some(true), true, _) => unreachable!("rejected above"),
(_, true, Some(vllm_chat::ChatRole::Assistant)) => {
Ok(GenerationPromptMode::ContinueFinalAssistant)
}
(_, true, _) => {
bail_invalid_request!(
"Cannot set `continue_final_message` to True when the last message is not from the assistant."
);
}
(Some(false), false, _) => Ok(GenerationPromptMode::NoGenerationPrompt),
(None | Some(true), false, _) => Ok(GenerationPromptMode::StartNewAssistant),
}
}
/// Extract the text content of the last message if it has the assistant role.
fn extract_last_assistant_content(messages: &[ChatMessage]) -> Option<String> {
let ChatMessage::Assistant { content, .. } = messages.last()? else {
return None;
};
let text = match content.as_ref()? {
MessageContent::Text(text) => text.clone(),
MessageContent::Parts(parts) => parts
.iter()
.filter_map(|p| match p {
ContentPart::Text { text } => Some(text.as_str()),
_ => None,
})
.collect::<Vec<_>>()
.join("\n"),
};
(!text.is_empty()).then_some(text)
}
/// Lower one OpenAI chat message into the `vllm-chat` message shape.
fn convert_message(message: ChatMessage) -> Result<VllmChatMessage, ApiError> {
match message {
ChatMessage::System { content, .. } => {
Ok(VllmChatMessage::system(convert_content(content)?))
}
ChatMessage::User { content, .. } => Ok(VllmChatMessage::user(convert_content(content)?)),
ChatMessage::Assistant {
content,
tool_calls,
reasoning,
name: _,
} => {
let mut blocks = Vec::new();
if let Some(reasoning) = reasoning
&& !reasoning.is_empty()
{
blocks.push(AssistantContentBlock::Reasoning { text: reasoning });
}
if let Some(content) = content {
blocks.extend(convert_assistant_text_blocks(content)?);
}
if let Some(tool_calls) = tool_calls {
blocks.extend(convert_assistant_tool_calls(tool_calls)?);
}
if blocks.is_empty() {
bail_invalid_request!(
"Assistant messages must contain text, reasoning content, or tool_calls."
);
}
Ok(VllmChatMessage::assistant_blocks(blocks))
}
ChatMessage::Tool {
content,
tool_call_id,
} => Ok(VllmChatMessage::tool_response(
convert_content(content)?,
tool_call_id,
)),
ChatMessage::Function { .. } => {
bail_invalid_request!("Function messages are not supported.")
}
ChatMessage::Developer {
content,
tools,
name: _,
} => Ok(VllmChatMessage::developer(
convert_content(content)?,
convert_message_tools(tools)?,
)),
}
}
/// Convert the given OpenAI message content value into the internal format in
/// `vllm-chat`.
fn convert_content(content: MessageContent) -> Result<ChatContent, ApiError> {
match content {
MessageContent::Text(text) => Ok(ChatContent::Text(text)),
MessageContent::Parts(parts) => parts
.into_iter()
.map(|part| match part {
ContentPart::Text { text } => Ok(ChatContentPart::text(text)),
ContentPart::ImageUrl { image_url, uuid } => Ok(ChatContentPart::ImageUrl {
image_url: image_url.url,
detail: image_url.detail,
uuid,
}),
_ => bail_invalid_request!("Only text and image_url content parts are supported."),
})
.try_collect()
.map(ChatContent::Parts),
}
}
/// Convert the given OpenAI assistant message content into the internal format
/// in `vllm-chat`.
fn convert_assistant_text_blocks(
content: MessageContent,
) -> Result<Vec<AssistantContentBlock>, ApiError> {
match content {
MessageContent::Text(text) => Ok(vec![AssistantContentBlock::Text { text }]),
MessageContent::Parts(parts) => parts
.into_iter()
.map(|part| match part {
ContentPart::Text { text } => Ok(AssistantContentBlock::Text { text }),
_ => bail_invalid_request!(
"Only text content parts are supported for assistant messages."
),
})
.try_collect(),
}
}
fn convert_assistant_tool_calls(
tool_calls: Vec<crate::routes::openai::utils::types::ToolCall>,
) -> Result<Vec<AssistantContentBlock>, ApiError> {
tool_calls
.into_iter()
.map(|tool_call| {
if tool_call.tool_type != "function" {
bail_invalid_request!("Only function tool calls are supported.");
}
Ok(AssistantContentBlock::ToolCall(AssistantToolCall {
id: tool_call.id,
name: tool_call.function.name,
arguments: tool_call.function.arguments.unwrap_or_else(|| "{}".to_string()),
}))
})
.collect()
}
fn convert_tools(tools: Option<Vec<Tool>>) -> Result<Vec<ChatTool>, ApiError> {
tools
.unwrap_or_default()
.into_iter()
.map(|tool| {
if tool.tool_type != "function" {
bail_invalid_request!("Only function tools are supported.");
}
Ok(ChatTool {
name: tool.function.name,
description: tool.function.description,
parameters: tool.function.parameters,
strict: tool.function.strict,
})
})
.collect()
}
fn convert_message_tools(tools: Option<Vec<Tool>>) -> Result<Option<Vec<ChatTool>>, ApiError> {
let tools = convert_tools(tools)?;
Ok((!tools.is_empty()).then_some(tools))
}
fn convert_tool_choice(tool_choice: Option<&ToolChoice>) -> Result<ChatToolChoice, ApiError> {
match tool_choice {
None | Some(ToolChoice::Value(ToolChoiceValue::Auto)) => Ok(ChatToolChoice::Auto),
Some(ToolChoice::Value(ToolChoiceValue::None)) => Ok(ChatToolChoice::None),
_ => bail_invalid_request!("tool_choice={:?} is not supported yet.", tool_choice),
}
}
#[cfg(test)]
mod tests {
use std::collections::HashMap;
use axum::http::HeaderMap;
use expect_test::expect;
use llm_multimodal::ImageDetail;
use serde_json::json;
use vllm_chat::{
AssistantContentBlock, AssistantToolCall, ChatContentPart, ChatMessage as VllmChatMessage,
ChatTool as VllmChatTool, ChatToolChoice, GenerationPromptMode,
SamplingParams as VllmSamplingParams,
};
use vllm_text::output::TextDecodeOptions;
use super::prepare_chat_request;
use crate::routes::openai::chat_completions::types::{
AssistantRole, ChatCompletionMessage, ChatCompletionRequest,
};
use crate::routes::openai::utils::types::{
ChatMessage, ContentPart, Function, FunctionCallResponse, ImageUrl, MessageContent, Tool,
ToolCall, ToolChoice, ToolChoiceValue, VideoUrl,
};
use crate::utils::{ResolvedRequestContext, resolve_request_context};
fn request_context(headers: &HeaderMap, request_id: Option<&str>) -> ResolvedRequestContext {
resolve_request_context(headers, request_id)
}
fn served(names: &[&str]) -> Vec<String> {
names.iter().map(|s| s.to_string()).collect()
}
fn base_request() -> ChatCompletionRequest {
ChatCompletionRequest {
model: "Qwen/Qwen1.5-0.5B-Chat".to_string(),
messages: vec![ChatMessage::User {
content: MessageContent::Text("hello".to_string()),
name: None,
}],
stream: true,
..Default::default()
}
}
#[test]
fn prepare_chat_request_maps_text_parts() {
let mut request = base_request();
request.messages = vec![ChatMessage::Assistant {
content: Some(MessageContent::Parts(vec![ContentPart::Text {
text: "hello".to_string(),
}])),
name: None,
tool_calls: None,
reasoning: None,
}];
request.add_generation_prompt = Some(false);
request.continue_final_message = true;
request.skip_special_tokens = false;
request.chat_template_kwargs = Some(HashMap::from([("foo".to_string(), json!("bar"))]));
let prepared = prepare_chat_request(
request,
&served(&["Qwen/Qwen1.5-0.5B-Chat"]),
ResolvedRequestContext::default(),
)
.expect("request is valid");
assert!(prepared.request_id.starts_with("chatcmpl-"));
assert_eq!(
prepared.chat_request.messages,
vec![VllmChatMessage::assistant_text("hello")]
);
assert_eq!(
prepared.chat_request.sampling_params,
VllmSamplingParams::default()
);
assert_eq!(
prepared.chat_request.chat_options.generation_prompt_mode,
GenerationPromptMode::ContinueFinalAssistant
);
assert_eq!(
prepared.chat_request.chat_options.template_kwargs,
HashMap::from([("foo".to_string(), json!("bar"))])
);
assert_eq!(
prepared.chat_request.decode_options,
TextDecodeOptions {
skip_special_tokens: false,
include_stop_str_in_output: false,
stop_strings: None,
min_tokens: 0,
}
);
assert!(prepared.chat_request.tools.is_empty());
assert_eq!(prepared.chat_request.tool_choice, ChatToolChoice::Auto);
}
#[test]
fn prepare_chat_request_keeps_optional_sampling_fields_unset() {
let prepared = prepare_chat_request(
base_request(),
&served(&["Qwen/Qwen1.5-0.5B-Chat"]),
ResolvedRequestContext::default(),
)
.expect("request is valid");
assert!(prepared.request_id.starts_with("chatcmpl-"));
assert_eq!(
prepared.chat_request.messages,
vec![VllmChatMessage::user("hello")]
);
assert_eq!(
prepared.chat_request.sampling_params,
VllmSamplingParams::default()
);
assert_eq!(
prepared.chat_request.chat_options.generation_prompt_mode,
GenerationPromptMode::StartNewAssistant
);
assert_eq!(
prepared.chat_request.decode_options,
TextDecodeOptions {
skip_special_tokens: true,
include_stop_str_in_output: false,
stop_strings: None,
min_tokens: 0,
}
);
assert!(prepared.chat_request.tools.is_empty());
assert_eq!(prepared.chat_request.tool_choice, ChatToolChoice::Auto);
}
#[test]
fn prepare_chat_request_preserves_sampling_passthrough_fields() {
let request = ChatCompletionRequest {
seed: Some(42),
min_p: Some(0.2),
frequency_penalty: Some(0.3),
presence_penalty: Some(0.4),
repetition_penalty: Some(1.1),
..base_request()
};
let prepared = prepare_chat_request(
request,
&served(&["Qwen/Qwen1.5-0.5B-Chat"]),
ResolvedRequestContext::default(),
)
.expect("request is valid");
let expected = VllmSamplingParams {
seed: Some(42),
min_p: Some(0.2),
frequency_penalty: Some(0.3),
presence_penalty: Some(0.4),
repetition_penalty: Some(1.1),
..VllmSamplingParams::default()
};
assert_eq!(prepared.chat_request.sampling_params, expected);
}
#[test]
fn prepare_chat_request_accepts_developer_messages() {
let request = ChatCompletionRequest {
messages: vec![ChatMessage::Developer {
content: MessageContent::Text("hello".to_string()),
tools: Some(vec![Tool {
tool_type: "function".to_string(),
function: Function {
name: "get_weather".to_string(),
description: Some("Get weather".to_string()),
parameters: json!({
"type": "object",
"properties": {"city": {"type": "string"}},
}),
strict: Some(true),
},
}]),
name: None,
}],
..base_request()
};
let prepared = prepare_chat_request(
request,
&served(&["Qwen/Qwen1.5-0.5B-Chat"]),
ResolvedRequestContext::default(),
)
.expect("request is valid");
assert_eq!(
prepared.chat_request.messages,
vec![VllmChatMessage::developer(
"hello",
Some(vec![VllmChatTool {
name: "get_weather".to_string(),
description: Some("Get weather".to_string()),
parameters: json!({
"type": "object",
"properties": {"city": {"type": "string"}},
}),
strict: Some(true),
}]),
)]
);
}
#[test]
fn prepare_chat_request_maps_image_url_content_parts() {
let request = ChatCompletionRequest {
messages: vec![ChatMessage::User {
content: MessageContent::Parts(vec![
ContentPart::Text {
text: "describe ".to_string(),
},
ContentPart::ImageUrl {
image_url: ImageUrl {
url: "https://example.com/image.png".to_string(),
detail: Some(ImageDetail::Low),
},
uuid: Some("image-1".to_string()),
},
ContentPart::Text {
text: " briefly".to_string(),
},
]),
name: None,
}],
..base_request()
};
let prepared = prepare_chat_request(
request,
&served(&["Qwen/Qwen1.5-0.5B-Chat"]),
ResolvedRequestContext::default(),
)
.expect("request is valid");
assert_eq!(
prepared.chat_request.messages,
vec![VllmChatMessage::user(vec![
ChatContentPart::text("describe "),
ChatContentPart::ImageUrl {
image_url: "https://example.com/image.png".to_string(),
detail: Some(ImageDetail::Low),
uuid: Some("image-1".to_string()),
},
ChatContentPart::text(" briefly"),
])]
);
}
#[test]
fn prepare_chat_request_maps_developer_image_url_content_parts() {
let request = ChatCompletionRequest {
messages: vec![ChatMessage::Developer {
content: MessageContent::Parts(vec![ContentPart::ImageUrl {
image_url: ImageUrl {
url: "https://example.com/image.png".to_string(),
detail: None,
},
uuid: None,
}]),
tools: None,
name: None,
}],
..base_request()
};
let prepared = prepare_chat_request(
request,
&served(&["Qwen/Qwen1.5-0.5B-Chat"]),
ResolvedRequestContext::default(),
)
.expect("request is valid");
assert_eq!(
prepared.chat_request.messages,
vec![VllmChatMessage::developer(
vec![ChatContentPart::image_url("https://example.com/image.png")],
None,
)]
);
}
#[test]
fn prepare_chat_request_rejects_video_content_parts() {
let request = ChatCompletionRequest {
messages: vec![ChatMessage::User {
content: MessageContent::Parts(vec![ContentPart::VideoUrl {
video_url: VideoUrl {
url: "https://example.com/video.mp4".to_string(),
},
}]),
name: None,
}],
..base_request()
};
let error = prepare_chat_request(
request,
&served(&["Qwen/Qwen1.5-0.5B-Chat"]),
ResolvedRequestContext::default(),
)
.unwrap_err();
expect!["Only text and image_url content parts are supported."]
.assert_eq(&error.to_error_response().error.message);
}
#[test]
fn prepare_chat_request_rejects_assistant_image_url_content_parts() {
let request = ChatCompletionRequest {
messages: vec![ChatMessage::Assistant {
content: Some(MessageContent::Parts(vec![ContentPart::ImageUrl {
image_url: ImageUrl {
url: "https://example.com/image.png".to_string(),
detail: None,
},
uuid: None,
}])),
name: None,
tool_calls: None,
reasoning: None,
}],
..base_request()
};
let error = prepare_chat_request(
request,
&served(&["Qwen/Qwen1.5-0.5B-Chat"]),
ResolvedRequestContext::default(),
)
.unwrap_err();
expect!["Only text content parts are supported for assistant messages."]
.assert_eq(&error.to_error_response().error.message);
}
#[test]
fn prepare_chat_request_accepts_assistant_reasoning_history() {
let message = ChatCompletionMessage {
role: AssistantRole,
content: Some("answer".to_string()),
tool_calls: None,
reasoning: Some("inner".to_string()),
};
let message_json = serde_json::to_value(message).expect("message serializes");
let request = ChatCompletionRequest {
messages: vec![
serde_json::from_value(message_json).expect("response message is valid history"),
],
add_generation_prompt: Some(false),
..base_request()
};
let prepared = prepare_chat_request(
request,
&served(&["Qwen/Qwen1.5-0.5B-Chat"]),
ResolvedRequestContext::default(),
)
.expect("request is valid");
assert_eq!(
prepared.chat_request.messages,
vec![VllmChatMessage::assistant_blocks(vec![
AssistantContentBlock::Reasoning {
text: "inner".to_string(),
},
AssistantContentBlock::Text {
text: "answer".to_string(),
},
])]
);
assert!(prepared.chat_request.tools.is_empty());
assert_eq!(prepared.chat_request.tool_choice, ChatToolChoice::Auto);
}
#[test]
fn prepare_chat_request_accepts_legacy_reasoning_content_alias() {
let request = ChatCompletionRequest {
messages: vec![
serde_json::from_value(json!({
"role": "assistant",
"content": "answer",
"reasoning_content": "inner",
}))
.expect("legacy reasoning_content alias is accepted"),
],
add_generation_prompt: Some(false),
..base_request()
};
let prepared = prepare_chat_request(
request,
&served(&["Qwen/Qwen1.5-0.5B-Chat"]),
ResolvedRequestContext::default(),
)
.expect("request is valid");
assert_eq!(
prepared.chat_request.messages,
vec![VllmChatMessage::assistant_blocks(vec![
AssistantContentBlock::Reasoning {
text: "inner".to_string(),
},
AssistantContentBlock::Text {
text: "answer".to_string(),
},
])]
);
}
#[test]
fn prepare_chat_request_accepts_tools_and_tool_history() {
let request = ChatCompletionRequest {
messages: vec![
ChatMessage::Assistant {
content: None,
name: None,
tool_calls: Some(vec![ToolCall {
id: "call_1".to_string(),
tool_type: "function".to_string(),
function: FunctionCallResponse {
name: "get_weather".to_string(),
arguments: Some(r#"{"city":"Paris"}"#.to_string()),
},
}]),
reasoning: None,
},
ChatMessage::Tool {
content: MessageContent::Text("Sunny".to_string()),
tool_call_id: "call_1".to_string(),
},
],
tools: Some(vec![Tool {
tool_type: "function".to_string(),
function: Function {
name: "get_weather".to_string(),
description: Some("Get weather".to_string()),
parameters: json!({
"type": "object",
"properties": {"city": {"type": "string"}},
}),
strict: None,
},
}]),
tool_choice: Some(ToolChoice::Value(ToolChoiceValue::None)),
..base_request()
};
let prepared = prepare_chat_request(
request,
&served(&["Qwen/Qwen1.5-0.5B-Chat"]),
ResolvedRequestContext::default(),
)
.expect("request is valid");
assert_eq!(
prepared.chat_request.messages,
vec![
VllmChatMessage::assistant_blocks(vec![AssistantContentBlock::ToolCall(
AssistantToolCall {
id: "call_1".to_string(),
name: "get_weather".to_string(),
arguments: r#"{"city":"Paris"}"#.to_string(),
},
)]),
VllmChatMessage::tool_response("Sunny", "call_1"),
]
);
assert_eq!(
prepared.chat_request.tools,
vec![VllmChatTool {
name: "get_weather".to_string(),
description: Some("Get weather".to_string()),
parameters: json!({
"type": "object",
"properties": {"city": {"type": "string"}},
}),
strict: None,
}]
);
assert_eq!(prepared.chat_request.tool_choice, ChatToolChoice::None);
}
#[test]
fn prepare_chat_request_lowers_logprobs_fields() {
let request = ChatCompletionRequest {
stream: false,
logprobs: true,
prompt_logprobs: Some(2),
..base_request()
};
let prepared = prepare_chat_request(
request,
&served(&["Qwen/Qwen1.5-0.5B-Chat"]),
ResolvedRequestContext::default(),
)
.expect("request is valid");
assert!(prepared.requested_logprobs);
assert!(prepared.include_prompt_logprobs);
assert_eq!(prepared.chat_request.sampling_params.logprobs, Some(0));
assert_eq!(
prepared.chat_request.sampling_params.prompt_logprobs,
Some(2)
);
}
#[test]
fn prepare_chat_request_keeps_prompt_logprobs_independent_from_echo() {
let request = ChatCompletionRequest {
logprobs: true,
top_logprobs: Some(3),
echo: true,
..base_request()
};
let prepared = prepare_chat_request(
request,
&served(&["Qwen/Qwen1.5-0.5B-Chat"]),
ResolvedRequestContext::default(),
)
.expect("request is valid");
assert_eq!(prepared.chat_request.sampling_params.logprobs, Some(3));
assert_eq!(prepared.chat_request.sampling_params.prompt_logprobs, None);
assert!(!prepared.include_prompt_logprobs);
}
#[test]
fn prepare_chat_request_threads_data_parallel_rank() {
let mut headers = HeaderMap::new();
headers.insert("X-data-parallel-rank", "7".parse().unwrap());
let prepared = prepare_chat_request(
base_request(),
&served(&["Qwen/Qwen1.5-0.5B-Chat"]),
request_context(&headers, None),
)
.expect("request is valid");
assert_eq!(prepared.chat_request.data_parallel_rank, Some(7));
}
#[test]
fn prepare_chat_request_leaves_data_parallel_rank_none_when_absent() {
let prepared = prepare_chat_request(
base_request(),
&served(&["Qwen/Qwen1.5-0.5B-Chat"]),
ResolvedRequestContext::default(),
)
.expect("request is valid");
assert_eq!(prepared.chat_request.data_parallel_rank, None);
}
#[test]
fn prepare_chat_request_maps_no_generation_prompt_mode() {
let mut request = base_request();
request.add_generation_prompt = Some(false);
let prepared = prepare_chat_request(
request,
&served(&["Qwen/Qwen1.5-0.5B-Chat"]),
ResolvedRequestContext::default(),
)
.expect("request is valid");
assert_eq!(
prepared.chat_request.chat_options.generation_prompt_mode,
GenerationPromptMode::NoGenerationPrompt
);
}
#[test]
fn prepare_chat_request_rejects_conflicting_explicit_generation_prompt_flags() {
let mut request = base_request();
request.add_generation_prompt = Some(true);
request.continue_final_message = true;
let error = prepare_chat_request(
request,
&served(&["Qwen/Qwen1.5-0.5B-Chat"]),
ResolvedRequestContext::default(),
)
.unwrap_err();
expect!["Cannot set both `continue_final_message` and `add_generation_prompt` to True."]
.assert_eq(&error.to_error_response().error.message);
}
#[test]
fn prepare_chat_request_accepts_continue_final_message_with_implicit_add_generation_prompt() {
let mut request = base_request();
request.messages = vec![ChatMessage::Assistant {
content: Some(MessageContent::Text("hello".to_string())),
name: None,
tool_calls: None,
reasoning: None,
}];
request.continue_final_message = true;
let prepared = prepare_chat_request(
request,
&served(&["Qwen/Qwen1.5-0.5B-Chat"]),
ResolvedRequestContext::default(),
)
.expect("request is valid");
assert_eq!(
prepared.chat_request.chat_options.generation_prompt_mode,
GenerationPromptMode::ContinueFinalAssistant
);
}
#[test]
fn prepare_chat_request_rejects_continue_final_message_without_final_assistant() {
let mut request = base_request();
request.continue_final_message = true;
let error = prepare_chat_request(
request,
&served(&["Qwen/Qwen1.5-0.5B-Chat"]),
ResolvedRequestContext::default(),
)
.unwrap_err();
expect!["Cannot set `continue_final_message` to True when the last message is not from the assistant."]
.assert_eq(&error.to_error_response().error.message);
}
#[test]
fn prepare_chat_request_allows_new_assistant_mode_after_final_assistant() {
let request = ChatCompletionRequest {
messages: vec![ChatMessage::Assistant {
content: Some(MessageContent::Text("hello".to_string())),
name: None,
tool_calls: None,
reasoning: None,
}],
..base_request()
};
let prepared = prepare_chat_request(
request,
&served(&["Qwen/Qwen1.5-0.5B-Chat"]),
ResolvedRequestContext::default(),
)
.expect("request is valid");
assert_eq!(
prepared.chat_request.chat_options.generation_prompt_mode,
GenerationPromptMode::StartNewAssistant
);
}
}
@@ -0,0 +1,600 @@
use std::collections::HashMap;
use std::fmt;
use serde::{Deserialize, Serialize};
use serde_json::Value;
use serde_with::SerializeDisplay;
use validator::Validate;
use vllm_chat::ReasoningEffort;
use crate::routes::openai::utils::structured_outputs::ResponseFormat;
use crate::routes::openai::utils::types::{
ChatLogProbs, ChatMessage, MessageContent, Normalizable, StreamOptions, StringOrArray, Tool,
ToolCall, ToolCallDelta, ToolChoice, ToolChoiceValue, ToolReference, UNKNOWN_MODEL_ID, Usage,
default_true, validate_stop, validate_top_p_value,
};
/// vLLM-compatible request type for the Chat Completions API.
///
/// Mirrors the Python vLLM `ChatCompletionRequest` class. The local copy keeps
/// the request type route-owned so we can add vLLM-only fields directly instead
/// of layering wrapper deserializers on top.
#[serde_with::skip_serializing_none]
#[derive(Debug, Clone, Deserialize, Serialize, Validate)]
#[validate(schema(function = "validate_chat_cross_parameters"))]
pub struct ChatCompletionRequest {
// -------- Standard OpenAI API Parameters --------
/// A list of messages comprising the conversation so far
#[validate(custom(function = "validate_messages"))]
pub messages: Vec<ChatMessage>,
/// ID of the model to use
#[serde(default = "default_model")]
pub model: String,
/// Number between -2.0 and 2.0. Positive values penalize new tokens based
/// on their existing frequency in the text so far
#[validate(range(min = -2.0, max = 2.0))]
pub frequency_penalty: Option<f32>,
/// Modify the likelihood of specified tokens appearing in the completion
pub logit_bias: Option<HashMap<String, f32>>,
/// Whether to return log probabilities of the output tokens
#[serde(default)]
pub logprobs: bool,
/// An integer specifying the number of most likely tokens to return
/// -1 means return all
#[validate(range(min = -1))]
pub top_logprobs: Option<i32>,
/// Deprecated: Replaced by max_completion_tokens
#[deprecated(note = "Use max_completion_tokens instead")]
#[validate(range(min = 1))]
pub max_tokens: Option<u32>,
/// An upper bound for the number of tokens that can be generated for a
/// completion
#[validate(range(min = 1))]
pub max_completion_tokens: Option<u32>,
/// How many chat completion choices to generate for each input message
#[validate(range(min = 1, max = 10))]
pub n: Option<u32>,
/// Number between -2.0 and 2.0. Positive values penalize new tokens based
/// on whether they appear in the text so far
#[validate(range(min = -2.0, max = 2.0))]
pub presence_penalty: Option<f32>,
/// An object specifying the format that the model must output
pub response_format: Option<ResponseFormat>,
/// If specified, our system will make a best effort to sample
/// deterministically
pub seed: Option<i64>,
/// Up to 4 sequences where the API will stop generating further tokens
#[validate(custom(function = "validate_stop"))]
pub stop: Option<StringOrArray>,
/// If set, partial message deltas will be sent
#[serde(default)]
pub stream: bool,
/// Options for streaming response
pub stream_options: Option<StreamOptions>,
/// What sampling temperature to use, between 0 and 2
#[validate(range(min = 0.0, max = 2.0))]
pub temperature: Option<f32>,
/// An alternative to sampling with temperature
#[validate(custom(function = "validate_top_p_value"))]
pub top_p: Option<f32>,
/// A list of tools the model may call
pub tools: Option<Vec<Tool>>,
/// Controls which (if any) tool is called by the model
pub tool_choice: Option<ToolChoice>,
/// Effort level for reasoning models (none, minimal, low, medium, high,
/// xhigh, max)
pub reasoning_effort: Option<ReasoningEffort>,
/// Whether to enable parallel function calling during tool use
pub parallel_tool_calls: Option<bool>,
/// A unique identifier representing your end-user
pub user: Option<String>,
// -------- vLLM Sampling Parameters --------
/// Use beam search instead of sampling
#[serde(default)]
pub use_beam_search: bool,
/// Top-k sampling parameter
pub top_k: Option<u32>,
/// Min-p nucleus sampling parameter
#[validate(range(min = 0.0, max = 1.0))]
pub min_p: Option<f32>,
/// Repetition penalty for reducing repetitive text
#[validate(range(min = 0.0, max = 2.0))]
pub repetition_penalty: Option<f32>,
/// Length penalty for beam search
pub length_penalty: Option<f32>,
/// Specific token IDs to use as stop conditions
pub stop_token_ids: Option<Vec<u32>>,
/// Include stop string in output
#[serde(default)]
pub include_stop_str_in_output: bool,
/// Ignore end-of-sequence tokens during generation
#[serde(default)]
pub ignore_eos: bool,
/// Minimum number of tokens to generate
#[validate(range(min = 1))]
pub min_tokens: Option<u32>,
/// Skip special tokens during detokenization
#[serde(default = "default_true")]
pub skip_special_tokens: bool,
/// Add spaces between special tokens during detokenization
#[serde(default = "default_true")]
pub spaces_between_special_tokens: bool,
/// Truncate prompt tokens to this length
pub truncate_prompt_tokens: Option<i64>,
/// Number of prompt logprobs to return
pub prompt_logprobs: Option<i32>,
/// Restrict output to these token IDs only
pub allowed_token_ids: Option<Vec<u32>>,
/// List of bad words to avoid during generation
pub bad_words: Option<Vec<String>>,
// -------- Extra vLLM Parameters --------
/// Token budget for reasoning/thinking
pub thinking_token_budget: Option<u32>,
/// Whether to include reasoning content in the response
#[serde(default = "default_true")]
pub include_reasoning: bool,
/// If true, the new message will be prepended with the last message if they
/// belong to the same role.
#[serde(default)]
pub echo: bool,
/// Whether to add the generation prompt to the chat template.
///
/// When omitted, the request follows the API default behavior, which is
/// equivalent to `true` unless `continue_final_message=true` selects
/// final assistant continuation instead.
pub add_generation_prompt: Option<bool>,
/// Continue generating from final assistant message
#[serde(default)]
pub continue_final_message: bool,
/// Whether to add special tokens (e.g. BOS) to the prompt
#[serde(default)]
pub add_special_tokens: bool,
/// Documents for RAG (retrieval-augmented generation)
pub documents: Option<Vec<Value>>,
/// Jinja chat template override
pub chat_template: Option<String>,
/// Additional keyword args passed to the chat template renderer
pub chat_template_kwargs: Option<HashMap<String, Value>>,
/// Additional kwargs for media IO connectors, keyed by modality
pub media_io_kwargs: Option<HashMap<String, Value>>,
/// Additional kwargs for the HF processor
pub mm_processor_kwargs: Option<HashMap<String, Value>>,
/// Additional kwargs for structured outputs
pub structured_outputs: Option<Value>,
/// Request scheduling priority (lower means earlier; default 0)
pub priority: Option<i32>,
/// External request ID used for response correlation.
pub request_id: Option<String>,
/// Tokens represented as strings of the form 'token_id:{token_id}' in
/// logprobs
pub return_tokens_as_token_ids: Option<bool>,
/// Include token IDs alongside generated text
pub return_token_ids: Option<bool>,
/// Salt for prefix cache isolation in multi-user environments
pub cache_salt: Option<String>,
/// KV transfer parameters for disaggregated serving
pub kv_transfer_params: Option<HashMap<String, Value>>,
/// Additional request parameters with string or numeric values for custom
/// extensions
pub vllm_xargs: Option<HashMap<String, Value>>,
/// Parameters for detecting repetitive N-gram patterns in output tokens
pub repetition_detection: Option<Value>,
}
impl Default for ChatCompletionRequest {
#[expect(deprecated)]
fn default() -> Self {
Self {
messages: Vec::new(),
model: default_model(),
frequency_penalty: None,
logit_bias: None,
logprobs: false,
top_logprobs: None,
max_tokens: None,
max_completion_tokens: None,
n: None,
presence_penalty: None,
response_format: None,
seed: None,
stop: None,
stream: false,
stream_options: None,
temperature: None,
top_p: None,
tools: None,
tool_choice: None,
reasoning_effort: None,
thinking_token_budget: None,
include_reasoning: true,
parallel_tool_calls: None,
user: None,
use_beam_search: false,
top_k: None,
min_p: None,
repetition_penalty: None,
length_penalty: None,
stop_token_ids: None,
include_stop_str_in_output: false,
ignore_eos: false,
min_tokens: None,
skip_special_tokens: true,
spaces_between_special_tokens: true,
truncate_prompt_tokens: None,
prompt_logprobs: None,
allowed_token_ids: None,
bad_words: None,
echo: false,
add_generation_prompt: None,
continue_final_message: false,
add_special_tokens: false,
documents: None,
chat_template: None,
chat_template_kwargs: None,
media_io_kwargs: None,
mm_processor_kwargs: None,
structured_outputs: None,
priority: None,
request_id: None,
return_tokens_as_token_ids: None,
return_token_ids: None,
cache_salt: None,
kv_transfer_params: None,
vllm_xargs: None,
repetition_detection: None,
}
}
}
impl Normalizable for ChatCompletionRequest {
/// Normalize the request by applying migrations and defaults.
fn normalize(&mut self) {
// Migrate deprecated max_tokens → max_completion_tokens
#[expect(deprecated)]
if self.max_completion_tokens.is_none() && self.max_tokens.is_some() {
self.max_completion_tokens = self.max_tokens;
self.max_tokens = None;
}
// Apply tool_choice defaults
// If tools is None, leave tool_choice as None (don't set it)
if self.tool_choice.is_none()
&& let Some(tools) = &self.tools
{
let choice_value = if tools.is_empty() {
ToolChoiceValue::None
} else {
ToolChoiceValue::Auto
};
self.tool_choice = Some(ToolChoice::Value(choice_value));
}
}
}
/// Mirrors the Python vLLM `ChatCompletionResponse` class.
#[serde_with::skip_serializing_none]
#[derive(Debug, Clone, Serialize)]
pub(super) struct ChatCompletionResponse {
pub id: String,
pub object: String,
pub created: u64,
pub model: String,
pub choices: Vec<ChatCompletionChoice>,
pub usage: Option<Usage>,
pub system_fingerprint: Option<String>,
pub prompt_logprobs: Option<Vec<Option<HashMap<String, f32>>>>,
pub prompt_token_ids: Option<Vec<u32>>,
pub kv_transfer_params: Option<Value>,
}
/// Mirrors the Python vLLM `ChatCompletionResponseChoice` class.
#[serde_with::skip_serializing_none]
#[derive(Debug, Clone, Serialize)]
pub(super) struct ChatCompletionChoice {
pub index: u32,
pub message: ChatCompletionMessage,
pub logprobs: Option<ChatLogProbs>,
pub finish_reason: Option<String>,
pub stop_reason: Option<Value>,
pub token_ids: Option<Vec<u32>>,
}
/// A literal type for the "assistant" role, since the API only allows that
/// specific value in responses.
#[derive(Debug, Clone, Copy, PartialEq, Eq, SerializeDisplay)]
pub(super) struct AssistantRole;
impl fmt::Display for AssistantRole {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.write_str("assistant")
}
}
/// Mirrors the Python vLLM response `ChatMessage` class.
#[serde_with::skip_serializing_none]
#[derive(Debug, Clone, Serialize)]
pub(super) struct ChatCompletionMessage {
pub role: AssistantRole,
pub content: Option<String>,
pub tool_calls: Option<Vec<ToolCall>>,
pub reasoning: Option<String>,
}
/// Mirrors the Python vLLM `ChatCompletionStreamResponse` class.
#[serde_with::skip_serializing_none]
#[derive(Debug, Clone, Serialize)]
pub(super) struct ChatCompletionStreamResponse {
pub id: String,
pub object: String,
pub created: u64,
pub model: String,
pub choices: Vec<ChatCompletionStreamChoice>,
pub usage: Option<Usage>,
pub prompt_token_ids: Option<Vec<u32>>,
}
impl ChatCompletionStreamResponse {
/// Create a stream response with the standard envelope fields pre-filled.
pub fn new(id: &str, model: &str, created: u64) -> Self {
Self {
id: id.to_string(),
object: "chat.completion.chunk".to_string(),
created,
model: model.to_string(),
choices: Vec::new(),
usage: None,
prompt_token_ids: None,
}
}
}
/// Mirrors the Python vLLM `ChatCompletionResponseStreamChoice` class.
#[serde_with::skip_serializing_none]
#[derive(Debug, Clone, Default, Serialize)]
pub(super) struct ChatCompletionStreamChoice {
pub index: u32,
pub delta: ChatMessageDelta,
pub logprobs: Option<ChatLogProbs>,
pub finish_reason: Option<String>,
pub stop_reason: Option<Value>,
pub token_ids: Option<Vec<u32>>,
}
/// Mirrors the Python vLLM `DeltaMessage` class.
#[serde_with::skip_serializing_none]
#[derive(Debug, Clone, Default, Serialize)]
pub(super) struct ChatMessageDelta {
pub role: Option<AssistantRole>,
pub content: Option<String>,
pub tool_calls: Option<Vec<ToolCallDelta>>,
pub reasoning: Option<String>,
}
fn default_model() -> String {
UNKNOWN_MODEL_ID.to_string()
}
/// Validates messages array is not empty and has valid content
fn validate_messages(messages: &[ChatMessage]) -> Result<(), validator::ValidationError> {
if messages.is_empty() {
return Err(validator::ValidationError::new("messages cannot be empty"));
}
for msg in messages {
if let ChatMessage::User { content, .. } = msg {
match content {
MessageContent::Text(text) if text.is_empty() => {
return Err(validator::ValidationError::new(
"message content cannot be empty",
));
}
MessageContent::Parts(parts) if parts.is_empty() => {
return Err(validator::ValidationError::new(
"message content parts cannot be empty",
));
}
_ => {}
}
}
}
Ok(())
}
/// Schema-level validation for cross-field dependencies
fn validate_chat_cross_parameters(
req: &ChatCompletionRequest,
) -> Result<(), validator::ValidationError> {
// 1. Validate logprobs dependency
if req.top_logprobs.is_some() && !req.logprobs {
let mut e = validator::ValidationError::new("top_logprobs_requires_logprobs");
e.message = Some("top_logprobs is only allowed when logprobs is enabled".into());
return Err(e);
}
// 2. Validate stream_options dependency
if req.stream_options.is_some() && !req.stream {
let mut e = validator::ValidationError::new("stream_options_requires_stream");
e.message = Some("stream_options can only be used when stream is true".into());
return Err(e);
}
// 3. Validate token limits - min <= max
if let (Some(min_tokens), Some(max_completion_tokens)) =
(req.min_tokens, req.max_completion_tokens)
&& min_tokens > max_completion_tokens
{
let mut e = validator::ValidationError::new("min_tokens_exceeds_max_completion_tokens");
e.message = Some("min_tokens cannot be greater than max_completion_tokens".into());
return Err(e);
}
#[expect(deprecated, reason = "Local type still mirrors legacy upstream field")]
if let (Some(min_tokens), Some(max_tokens)) = (req.min_tokens, req.max_tokens)
&& min_tokens > max_tokens
{
let mut e = validator::ValidationError::new("min_tokens_exceeds_max_tokens");
e.message = Some("min_tokens cannot be greater than max_tokens".into());
return Err(e);
}
// 4. Validate response format JSON schema name
if let Some(ResponseFormat::JsonSchema { json_schema }) = &req.response_format
&& json_schema.name.is_empty()
{
let mut e = validator::ValidationError::new("json_schema_name_empty");
e.message = Some("JSON schema name cannot be empty".into());
return Err(e);
}
// 5. Validate tool_choice requires tools (except for "none")
if let Some(ref tool_choice) = req.tool_choice {
let has_tools = req.tools.as_ref().is_some_and(|t| !t.is_empty());
// Check if tool_choice is anything other than "none"
let is_some_choice = !matches!(tool_choice, ToolChoice::Value(ToolChoiceValue::None));
if is_some_choice && !has_tools {
let mut e = validator::ValidationError::new("tool_choice_requires_tools");
e.message = Some("Invalid value for 'tool_choice': 'tool_choice' is only allowed when 'tools' are specified.".into());
return Err(e);
}
// Additional validation when tools are present
if let Some(tools) = req.tools.as_ref().filter(|t| !t.is_empty()) {
match tool_choice {
ToolChoice::Function { function, .. } => {
// Validate that the specified function name exists in tools
let function_exists = tools.iter().any(|tool| {
tool.tool_type == "function" && tool.function.name == function.name
});
if !function_exists {
let mut e =
validator::ValidationError::new("tool_choice_function_not_found");
e.message = Some(
format!(
"Invalid value for 'tool_choice': function '{}' not found in 'tools'.",
function.name
)
.into(),
);
return Err(e);
}
}
ToolChoice::AllowedTools {
mode,
tools: allowed_tools,
..
} => {
// Validate mode is "auto" or "required"
if mode != "auto" && mode != "required" {
let mut e = validator::ValidationError::new("tool_choice_invalid_mode");
e.message = Some(format!(
"Invalid value for 'tool_choice.mode': must be 'auto' or 'required', got '{mode}'."
).into());
return Err(e);
}
// Validate that all ToolReferences are Function type (Chat API only supports
// function tools)
for tool_ref in allowed_tools {
match tool_ref {
ToolReference::Function { name } => {
// Validate that the function exists in tools array
let tool_exists = tools.iter().any(|tool| {
tool.tool_type == "function" && tool.function.name == *name
});
if !tool_exists {
let mut e = validator::ValidationError::new(
"tool_choice_tool_not_found",
);
e.message = Some(
format!(
"Invalid value for 'tool_choice.tools': tool '{name}' not found in 'tools'."
)
.into(),
);
return Err(e);
}
}
_ => {
// Chat Completion API only supports function tools in tool_choice
let mut e = validator::ValidationError::new(
"tool_choice_invalid_tool_type",
);
e.message = Some(
format!(
"Invalid value for 'tool_choice.tools': Chat Completion API only supports function tools, got '{}'.",
tool_ref.identifier()
)
.into(),
);
return Err(e);
}
}
}
}
ToolChoice::Value(_) => {}
}
}
}
Ok(())
}
@@ -0,0 +1,410 @@
use super::types::ChatCompletionRequest;
use crate::error::{ApiError, bail_invalid_request};
use crate::routes::openai::utils::types::{ChatMessage, Tool, ToolChoice, ToolChoiceValue};
/// Enforce the minimal compatibility contract for the Rust OpenAI server.
pub(super) fn validate_request_compat(
request: &ChatCompletionRequest,
served_model_names: &[String],
) -> Result<(), ApiError> {
if !served_model_names.iter().any(|n| n == &request.model) {
return Err(ApiError::model_not_found(request.model.clone()));
}
if request.stream_options.is_some() && !request.stream {
bail_invalid_request!(
param = "stream_options",
"stream_options are only supported when stream=true."
);
}
if request.n.unwrap_or(1) > 1 {
bail_invalid_request!(param = "n", "Only n=1 is supported.");
}
if request.top_logprobs.is_some() && !request.logprobs {
bail_invalid_request!(
param = "top_logprobs",
"top_logprobs can only be used when logprobs=true."
);
}
if let Some(prompt_logprobs) = request.prompt_logprobs {
if prompt_logprobs < 0 && prompt_logprobs != -1 {
bail_invalid_request!(
param = "prompt_logprobs",
"prompt_logprobs must be a non-negative value or -1."
);
}
if request.stream && (prompt_logprobs > 0 || prompt_logprobs == -1) {
bail_invalid_request!(
param = "prompt_logprobs",
"prompt_logprobs are not available when stream=true."
);
}
}
if let Some(tools) = request.tools.as_ref() {
validate_function_tools(tools, "tools")?;
}
for message in &request.messages {
if let ChatMessage::Developer {
tools: Some(tools), ..
} = message
{
validate_function_tools(tools, "messages[].tools")?;
}
}
if let Some(tool_choice) = &request.tool_choice {
match tool_choice {
ToolChoice::Value(ToolChoiceValue::Auto | ToolChoiceValue::None) => {}
ToolChoice::Value(ToolChoiceValue::Required) => {
bail_invalid_request!(
param = "tool_choice",
"tool_choice=required is not supported yet."
);
}
ToolChoice::Function { .. } => {
bail_invalid_request!(
param = "tool_choice",
"Named function tool_choice is not supported yet."
);
}
ToolChoice::AllowedTools { .. } => {
bail_invalid_request!(
param = "tool_choice",
"allowed_tools tool_choice is not supported yet."
);
}
}
}
if request.use_beam_search {
bail_invalid_request!(
param = "use_beam_search",
"use_beam_search is not supported."
);
}
// ---- Reject parameters that are accepted for deserialization but not yet
// implemented ----
if request.parallel_tool_calls.is_some() {
bail_invalid_request!(
param = "parallel_tool_calls",
"parallel_tool_calls is not supported."
);
}
reject_non_default(
request.length_penalty.as_ref(),
"length_penalty",
"length_penalty is not supported.",
)?;
if !request.spaces_between_special_tokens {
bail_invalid_request!(
param = "spaces_between_special_tokens",
"spaces_between_special_tokens is not supported."
);
}
reject_non_default(
request.truncate_prompt_tokens.as_ref(),
"truncate_prompt_tokens",
"truncate_prompt_tokens is not supported.",
)?;
reject_non_default(
request.thinking_token_budget.as_ref(),
"thinking_token_budget",
"thinking_token_budget is not supported.",
)?;
if !request.include_reasoning {
bail_invalid_request!(
param = "include_reasoning",
"include_reasoning is not supported."
);
}
reject_non_default(
request.media_io_kwargs.as_ref(),
"media_io_kwargs",
"media_io_kwargs is not supported.",
)?;
reject_non_default(
request.mm_processor_kwargs.as_ref(),
"mm_processor_kwargs",
"mm_processor_kwargs is not supported.",
)?;
reject_non_default(
request.repetition_detection.as_ref(),
"repetition_detection",
"repetition_detection is not supported.",
)?;
if let Some(options) = &request.stream_options
&& options.continuous_usage_stats.is_some()
{
bail_invalid_request!(
param = "stream_options",
"continuous_usage_stats is not supported."
);
}
Ok(())
}
/// Reject one option unless it is entirely absent.
fn reject_non_default<T>(
value: Option<&T>,
param: &'static str,
message: &str,
) -> Result<(), ApiError> {
if value.is_some() {
bail_invalid_request!(param = param, "{}", message);
}
Ok(())
}
fn validate_function_tools(tools: &[Tool], param: &'static str) -> Result<(), ApiError> {
for tool in tools {
if tool.tool_type != "function" {
bail_invalid_request!(param = param, "Only function tools are supported.");
}
}
Ok(())
}
#[cfg(test)]
mod tests {
use std::collections::HashMap;
use serde_json::json;
use vllm_chat::ReasoningEffort;
use super::validate_request_compat;
use crate::routes::openai::chat_completions::types::ChatCompletionRequest;
use crate::routes::openai::utils::structured_outputs::ResponseFormat;
use crate::routes::openai::utils::types::{
ChatMessage, Function, FunctionChoice, MessageContent, StringOrArray, Tool, ToolChoice,
ToolChoiceValue, ToolReference,
};
fn served(names: &[&str]) -> Vec<String> {
names.iter().map(|s| s.to_string()).collect()
}
fn base_request() -> ChatCompletionRequest {
ChatCompletionRequest {
model: "Qwen/Qwen1.5-0.5B-Chat".to_string(),
messages: vec![ChatMessage::User {
content: MessageContent::Text("hello".to_string()),
name: None,
}],
stream: true,
..Default::default()
}
}
#[test]
fn validate_request_compat_accepts_stop() {
let request = ChatCompletionRequest {
stop: Some(StringOrArray::String("stop".to_string())),
..base_request()
};
validate_request_compat(&request, &served(&["Qwen/Qwen1.5-0.5B-Chat"]))
.expect("stop strings should be accepted");
}
#[test]
fn validate_request_compat_accepts_non_zero_penalties_and_function_tools() {
let request = ChatCompletionRequest {
frequency_penalty: Some(0.5),
presence_penalty: Some(0.25),
min_p: Some(0.2),
repetition_penalty: Some(1.1),
seed: Some(7),
..base_request()
};
validate_request_compat(&request, &served(&["Qwen/Qwen1.5-0.5B-Chat"]))
.expect("sampling fields should be accepted");
let request = ChatCompletionRequest {
tools: Some(vec![Tool {
tool_type: "function".to_string(),
function: Function {
name: "tool".to_string(),
description: None,
parameters: json!({}),
strict: None,
},
}]),
..base_request()
};
validate_request_compat(&request, &served(&["Qwen/Qwen1.5-0.5B-Chat"]))
.expect("function tools should be accepted");
let request = ChatCompletionRequest {
messages: vec![ChatMessage::Developer {
content: MessageContent::Text("policy".to_string()),
tools: Some(vec![Tool {
tool_type: "function".to_string(),
function: Function {
name: "tool".to_string(),
description: None,
parameters: json!({}),
strict: None,
},
}]),
name: None,
}],
..base_request()
};
validate_request_compat(&request, &served(&["Qwen/Qwen1.5-0.5B-Chat"]))
.expect("developer function tools should be accepted");
}
#[test]
fn validate_request_compat_rejects_non_function_developer_tools() {
let request = ChatCompletionRequest {
messages: vec![ChatMessage::Developer {
content: MessageContent::Text("policy".to_string()),
tools: Some(vec![Tool {
tool_type: "mcp".to_string(),
function: Function {
name: "tool".to_string(),
description: None,
parameters: json!({}),
strict: None,
},
}]),
name: None,
}],
..base_request()
};
assert!(validate_request_compat(&request, &served(&["Qwen/Qwen1.5-0.5B-Chat"])).is_err());
}
#[test]
fn validate_request_compat_accepts_output_logprobs() {
let request = ChatCompletionRequest {
logprobs: true,
..base_request()
};
validate_request_compat(&request, &served(&["Qwen/Qwen1.5-0.5B-Chat"]))
.expect("logprobs should be accepted");
}
#[test]
fn validate_request_compat_accepts_reasoning_effort() {
let request = ChatCompletionRequest {
reasoning_effort: Some(ReasoningEffort::Max),
chat_template_kwargs: Some(HashMap::from([(
"reasoning_effort".to_string(),
json!("low"),
)])),
..base_request()
};
validate_request_compat(&request, &served(&["Qwen/Qwen1.5-0.5B-Chat"]))
.expect("reasoning_effort should be accepted");
}
#[test]
fn validate_request_compat_rejects_top_logprobs_without_logprobs() {
let request = ChatCompletionRequest {
top_logprobs: Some(0),
..base_request()
};
assert!(validate_request_compat(&request, &served(&["Qwen/Qwen1.5-0.5B-Chat"])).is_err());
}
#[test]
fn validate_request_compat_rejects_streaming_prompt_logprobs_requests() {
let request = ChatCompletionRequest {
prompt_logprobs: Some(1),
..base_request()
};
assert!(validate_request_compat(&request, &served(&["Qwen/Qwen1.5-0.5B-Chat"])).is_err());
let request = ChatCompletionRequest {
prompt_logprobs: Some(-1),
..base_request()
};
assert!(validate_request_compat(&request, &served(&["Qwen/Qwen1.5-0.5B-Chat"])).is_err());
}
#[test]
fn validate_request_compat_rejects_invalid_prompt_logprobs_value() {
let request = ChatCompletionRequest {
stream: false,
prompt_logprobs: Some(-2),
..base_request()
};
assert!(validate_request_compat(&request, &served(&["Qwen/Qwen1.5-0.5B-Chat"])).is_err());
}
#[test]
fn validate_request_compat_accepts_response_format() {
let request = ChatCompletionRequest {
response_format: Some(ResponseFormat::Text),
..base_request()
};
validate_request_compat(&request, &served(&["Qwen/Qwen1.5-0.5B-Chat"]))
.expect("response_format=text should be accepted");
let request = ChatCompletionRequest {
response_format: Some(ResponseFormat::JsonObject),
..base_request()
};
validate_request_compat(&request, &served(&["Qwen/Qwen1.5-0.5B-Chat"]))
.expect("response_format=json_object should be accepted");
}
#[test]
fn validate_request_compat_accepts_noop_tool_choice_none() {
let request = ChatCompletionRequest {
tool_choice: Some(ToolChoice::Value(ToolChoiceValue::None)),
..base_request()
};
validate_request_compat(&request, &served(&["Qwen/Qwen1.5-0.5B-Chat"]))
.expect("tool_choice=none is ok");
}
#[test]
fn validate_request_compat_rejects_required_and_named_tool_choices() {
let required = ChatCompletionRequest {
tool_choice: Some(ToolChoice::Value(ToolChoiceValue::Required)),
..base_request()
};
assert!(validate_request_compat(&required, &served(&["Qwen/Qwen1.5-0.5B-Chat"])).is_err());
let named = ChatCompletionRequest {
tool_choice: Some(ToolChoice::Function {
tool_type: "function".to_string(),
function: FunctionChoice {
name: "tool".to_string(),
},
}),
..base_request()
};
assert!(validate_request_compat(&named, &served(&["Qwen/Qwen1.5-0.5B-Chat"])).is_err());
let allowed_tools = ChatCompletionRequest {
tool_choice: Some(ToolChoice::AllowedTools {
tool_type: "allowed_tools".to_string(),
mode: "auto".to_string(),
tools: vec![ToolReference::Function {
name: "tool".to_string(),
}],
}),
..base_request()
};
assert!(
validate_request_compat(&allowed_tools, &served(&["Qwen/Qwen1.5-0.5B-Chat"])).is_err()
);
}
}
@@ -0,0 +1,571 @@
mod convert;
mod types;
mod validate;
use std::convert::Infallible;
use std::result::Result;
use std::sync::Arc;
use asynk_strim_attr::{TryYielder, try_stream};
use axum::Json;
use axum::extract::State;
use axum::http::HeaderMap;
use axum::response::sse::{Event, Sse};
use axum::response::{IntoResponse, Response};
use futures::{Stream, StreamExt as _, pin_mut};
use thiserror_ext::AsReport as _;
use tracing::{debug, error, info, trace};
use tracing_futures::Instrument as _;
use vllm_text::{DecodedTextEvent, FinishReason, TextOutputStream, TextOutputStreamExt as _};
use super::utils::logprobs::{
collected_logprobs_to_openai, decoded_logprobs_to_openai, decoded_prompt_logprobs_to_maps,
text_len,
};
use super::utils::types::Usage;
use crate::error::{ApiError, bail_server_error, server_error};
use crate::routes::openai::completions::convert::prepare_completion_request;
use crate::routes::openai::completions::types::{
CompletionChoice, CompletionRequest, CompletionResponse, CompletionSseChunk,
CompletionStreamChoice, CompletionStreamResponse,
};
use crate::routes::openai::utils::types::LogProbs;
use crate::routes::openai::utils::validated_json::ValidatedJson;
use crate::state::AppState;
use crate::utils::{resolve_request_context, unix_timestamp};
/// Validate one completions request and proxy it into the shared `vllm-text`
/// stack.
pub async fn completions(
State(state): State<Arc<AppState>>,
headers: HeaderMap,
ValidatedJson(body): ValidatedJson<CompletionRequest>,
) -> Response {
let stream = body.stream;
let logprobs = body.logprobs;
let request_context = resolve_request_context(&headers, body.request_id.as_deref());
let prepared =
match prepare_completion_request(body, state.served_model_names(), request_context) {
Ok(prepared) => prepared,
Err(error) => return error.into_response(),
};
let request_span = tracing::info_span!(
"completions",
request_id = %prepared.request_id,
engine_request_id = tracing::field::Empty,
);
let created = unix_timestamp();
let include_prompt_logprobs = prepared.text_request.sampling_params.prompt_logprobs.is_some();
let log_request = state.enable_log_requests;
let text_stream = match state
.chat
.text()
.generate(prepared.text_request)
.instrument(request_span.clone())
.await
{
Ok(stream) => stream,
Err(error) => {
return server_error!(
"failed to submit completion request: {}",
error.to_report_string()
)
.into_response();
}
};
if stream {
let chunk_stream = completion_chunk_stream(
text_stream,
prepared.request_id,
prepared.response_model,
created,
log_request,
prepared.include_usage,
prepared.echo,
logprobs,
prepared.return_token_ids,
prepared.return_tokens_as_token_ids,
);
let sse_stream = completion_sse_stream(chunk_stream).instrument(request_span);
Sse::new(sse_stream).into_response()
} else {
let response = match collect_completion(
text_stream,
prepared.request_id,
prepared.response_model,
created,
prepared.echo,
logprobs,
include_prompt_logprobs,
prepared.return_token_ids,
prepared.return_tokens_as_token_ids,
)
.instrument(request_span.clone())
.await
{
Ok(response) => response,
Err(error) => return error.into_response(),
};
if log_request {
let usage = response.usage.as_ref();
info!(
parent: &request_span,
model = %response.model,
prompt_tokens = usage.map_or(0, |u| u.prompt_tokens),
output_tokens = usage.and_then(|u| u.completion_tokens).unwrap_or(0),
finish_reason = response.choices.first().and_then(|c| c.finish_reason.as_deref()).unwrap_or("unknown"),
"completion finished"
);
}
Json(response).into_response()
}
}
async fn collect_completion(
stream: impl TextOutputStream,
request_id: String,
response_model: String,
created: u64,
echo: Option<String>,
requested_logprobs: Option<u32>,
include_prompt_logprobs: bool,
return_token_ids: bool,
return_tokens_as_token_ids: bool,
) -> Result<CompletionResponse, ApiError> {
let collected = stream
.collect_output()
.await
.map_err(|error| server_error!("completion stream failed: {}", error.to_report_string()))?;
let finish_reason = collected.finish_reason.clone();
let stop_reason = finish_reason
.as_stop_reason()
.map(|sr| serde_json::to_value(sr).expect("StopReason must serialize to JSON"));
let prompt_char_count = echo.as_ref().map(|prompt| text_len(prompt)).unwrap_or_default();
let prompt_logprobs = if include_prompt_logprobs {
let prompt_logprobs = collected.prompt_logprobs.as_ref().ok_or_else(|| {
server_error!(
"completion response requested prompt_logprobs but generation returned none"
)
})?;
Some(prompt_logprobs)
} else {
None
};
let logprobs = if requested_logprobs.is_some() {
Some(collected_logprobs_to_openai(
&collected,
echo.is_some(),
prompt_char_count,
return_tokens_as_token_ids,
)?)
} else {
None
};
let prompt_logprobs =
prompt_logprobs.map(|lp| decoded_prompt_logprobs_to_maps(lp, return_tokens_as_token_ids));
let text = match &echo {
None => collected.text,
Some(prompt) => format!("{prompt}{}", collected.text),
};
Ok(CompletionResponse {
id: request_id,
object: "text_completion".to_string(),
created,
model: response_model,
choices: vec![CompletionChoice {
index: 0,
text,
logprobs,
finish_reason: Some(completion_finish_reason_to_openai(finish_reason)?.into()),
stop_reason,
prompt_logprobs,
token_ids: return_token_ids.then(|| collected.token_ids.clone()),
prompt_token_ids: return_token_ids.then(|| collected.prompt_token_ids.to_vec()),
}],
usage: Some(Usage::from_counts(
collected.prompt_token_ids.len() as u32,
collected.token_ids.len() as u32,
)),
system_fingerprint: None,
kv_transfer_params: collected.kv_transfer_params,
})
}
/// Convert one internal decoded-text stream into OpenAI completions chunks.
#[try_stream]
async fn completion_chunk_stream(
stream: impl TextOutputStream,
request_id: String,
response_model: String,
created: u64,
log_request: bool,
include_usage: bool,
echo: Option<String>,
requested_logprobs: Option<u32>,
return_token_ids: bool,
return_tokens_as_token_ids: bool,
mut y: TryYielder<CompletionSseChunk, ApiError>,
) -> Result<(), ApiError> {
pin_mut!(stream);
let mut visible_text_len = 0_u32;
let mut first_chunk = true;
while let Some(next) = stream.next().await {
match next {
Ok(DecodedTextEvent::Start {
prompt_token_ids, ..
}) => {
debug!("completion stream started");
if let Some(prompt) = echo.as_ref() {
visible_text_len = text_len(prompt);
let mut chunk =
delta_chunk(&request_id, &response_model, created, prompt.clone(), None);
if return_token_ids && first_chunk {
if let Some(choice) = chunk.choices.first_mut() {
choice.prompt_token_ids = Some(prompt_token_ids.to_vec());
}
first_chunk = false;
}
y.yield_ok(CompletionSseChunk::Chunk(chunk)).await;
} else if return_token_ids {
// Emit a chunk with prompt_token_ids in the first streaming response
let mut chunk =
delta_chunk(&request_id, &response_model, created, String::new(), None);
if let Some(choice) = chunk.choices.first_mut() {
choice.prompt_token_ids = Some(prompt_token_ids.to_vec());
}
first_chunk = false;
y.yield_ok(CompletionSseChunk::Chunk(chunk)).await;
}
}
Ok(DecodedTextEvent::TextDelta {
delta,
token_ids,
logprobs,
finished,
}) => {
let delta_text_len = text_len(&delta);
let logprobs = if requested_logprobs.is_some() {
let decoded_logprobs = logprobs.as_ref().ok_or_else(|| {
server_error!(
"completion stream requested logprobs but generation returned none"
)
})?;
Some(decoded_logprobs_to_openai(
decoded_logprobs,
visible_text_len,
return_tokens_as_token_ids,
)?)
} else {
None
};
let mut chunk = delta_chunk(&request_id, &response_model, created, delta, logprobs);
if return_token_ids && let Some(choice) = chunk.choices.first_mut() {
choice.token_ids = Some(token_ids);
}
y.yield_ok(CompletionSseChunk::Chunk(chunk)).await;
visible_text_len = visible_text_len.saturating_add(delta_text_len);
if let Some(finished) = finished {
if log_request {
info!(
stream = true,
model = %response_model,
prompt_tokens = finished.prompt_token_count,
output_tokens = finished.output_token_count,
finish_reason = finished.finish_reason.as_str(),
"completion finished"
);
}
y.yield_ok(CompletionSseChunk::Chunk(final_chunk(
&request_id,
&response_model,
created,
finished.finish_reason,
)?))
.await;
if include_usage {
y.yield_ok(CompletionSseChunk::Usage(usage_chunk(
&request_id,
&response_model,
created,
Usage::from_counts(
finished.prompt_token_count as u32,
finished.output_token_count as u32,
),
)))
.await;
}
}
}
Err(error) => {
error!(
error = %error.as_report(),
"completion stream failed"
);
bail_server_error!("{}", error.to_report_string());
}
}
}
Ok(())
}
fn delta_chunk(
request_id: &str,
response_model: &str,
created: u64,
text: String,
logprobs: Option<LogProbs>,
) -> CompletionStreamResponse {
let mut chunk = CompletionStreamResponse::new(request_id, response_model, created);
chunk.choices.push(CompletionStreamChoice {
text,
logprobs,
..Default::default()
});
chunk
}
fn final_chunk(
request_id: &str,
response_model: &str,
created: u64,
finish_reason: FinishReason,
) -> Result<CompletionStreamResponse, ApiError> {
let finish_reason = completion_finish_reason_to_openai(finish_reason)?;
let mut chunk = CompletionStreamResponse::new(request_id, response_model, created);
chunk.choices.push(CompletionStreamChoice {
finish_reason: Some(finish_reason.to_string()),
..Default::default()
});
Ok(chunk)
}
fn completion_finish_reason_to_openai(
finish_reason: FinishReason,
) -> Result<&'static str, ApiError> {
match finish_reason {
FinishReason::Stop(_) | FinishReason::Repetition => Ok("stop"),
FinishReason::Length => Ok("length"),
FinishReason::Abort => Ok("abort"),
FinishReason::Error => {
bail_server_error!("Internal server error");
}
}
}
fn usage_chunk(
request_id: &str,
response_model: &str,
created: u64,
usage: Usage,
) -> CompletionStreamResponse {
let mut chunk = CompletionStreamResponse::new(request_id, response_model, created);
chunk.usage = Some(usage);
chunk
}
/// Convert one chunk stream into OpenAI-style SSE events.
///
/// OpenAI-style streaming errors are encoded as ordinary `data: {"error": ...}`
/// events followed by `data: [DONE]`, so the transport stream itself stays
/// infallible even when generation fails after the HTTP response has started.
#[try_stream]
async fn completion_sse_stream(
stream: impl Stream<Item = Result<CompletionSseChunk, ApiError>>,
mut y: TryYielder<Event, Infallible>,
) -> Result<(), Infallible> {
pin_mut!(stream);
while let Some(next) = stream.next().await {
match next {
Ok(chunk) => y.yield_ok(to_sse_event(&chunk)).await,
Err(error) => {
y.yield_ok(to_error_sse_event(&error)).await;
break;
}
}
}
y.yield_ok(done_sse_event()).await;
Ok(())
}
/// Serialize one OpenAI chunk payload into one SSE `data:` event.
fn to_sse_event(chunk: &CompletionSseChunk) -> Event {
let payload = serde_json::to_string(chunk).expect("completion chunk must serialize to JSON");
trace!(payload, "completion emitting chunk");
Event::default().data(payload)
}
/// Serialize one OpenAI error payload into one SSE `data:` event.
fn to_error_sse_event(error: &ApiError) -> Event {
let payload = serde_json::to_string(&error.to_error_response())
.expect("ErrorResponse must serialize to JSON");
trace!(payload, "completion emitting error");
Event::default().data(payload)
}
/// Build the terminal OpenAI SSE sentinel event.
fn done_sse_event() -> Event {
trace!("completion emitting done");
Event::default().data("[DONE]")
}
#[cfg(test)]
mod tests {
use futures::{StreamExt as _, stream};
use itertools::Itertools as _;
use vllm_text::{
DecodedLogprobs, DecodedPositionLogprobs, DecodedTextEvent, DecodedTokenLogprob,
FinishReason, Finished,
};
use super::{CompletionSseChunk, completion_chunk_stream, final_chunk};
#[test]
fn final_chunk_maps_stop_finish_reason() {
let chunk = final_chunk("cmpl-1", "model", 1, FinishReason::stop_eos())
.expect("finish reason valid");
assert_eq!(chunk.choices[0].finish_reason.as_deref(), Some("stop"));
assert_eq!(chunk.choices[0].text, "");
}
#[test]
fn final_chunk_maps_length_finish_reason() {
let chunk =
final_chunk("cmpl-1", "model", 1, FinishReason::Length).expect("finish reason valid");
assert_eq!(chunk.choices[0].finish_reason.as_deref(), Some("length"));
}
#[test]
fn final_chunk_maps_abort_finish_reason() {
let chunk =
final_chunk("cmpl-1", "model", 1, FinishReason::Abort).expect("finish reason valid");
assert_eq!(chunk.choices[0].finish_reason.as_deref(), Some("abort"));
}
#[test]
fn final_chunk_rejects_error_finish_reason() {
assert!(final_chunk("cmpl-1", "model", 1, FinishReason::Error).is_err());
}
#[tokio::test]
async fn completion_chunk_stream_maps_streaming_logprobs() {
let stream = stream::iter(vec![
Ok(DecodedTextEvent::Start {
prompt_token_ids: vec![1, 2, 3, 4, 5].into(),
prompt_logprobs: None,
}),
Ok(DecodedTextEvent::TextDelta {
delta: "h".to_string(),
token_ids: vec![b'h' as u32],
logprobs: Some(DecodedLogprobs {
positions: vec![DecodedPositionLogprobs {
entries: vec![
DecodedTokenLogprob {
token_id: 0,
token: "h".to_string(),
logprob: -0.1,
rank: 1,
},
DecodedTokenLogprob {
token_id: 0,
token: "H".to_string(),
logprob: -0.2,
rank: 1,
},
],
}],
}),
finished: None,
}),
Ok(DecodedTextEvent::TextDelta {
delta: String::new(),
token_ids: vec![b'!' as u32],
logprobs: Some(DecodedLogprobs {
positions: vec![DecodedPositionLogprobs {
entries: vec![
DecodedTokenLogprob {
token_id: 0,
token: "!".to_string(),
logprob: -0.3,
rank: 1,
},
DecodedTokenLogprob {
token_id: 0,
token: "?".to_string(),
logprob: -0.4,
rank: 1,
},
],
}],
}),
finished: Some(Finished {
prompt_token_count: 5,
output_token_count: 2,
finish_reason: FinishReason::stop_eos(),
kv_transfer_params: None,
}),
}),
]);
let chunks = completion_chunk_stream(
stream,
"cmpl-1".to_string(),
"model".to_string(),
1,
false,
false,
None,
Some(1),
false,
false,
)
.collect::<Vec<_>>()
.await;
let chunks: Vec<_> = chunks.into_iter().try_collect().expect("stream should succeed");
match &chunks[0] {
CompletionSseChunk::Chunk(chunk) => {
assert_eq!(chunk.choices[0].text, "h");
assert_eq!(
chunk.choices[0].logprobs.as_ref().expect("logprobs").tokens,
vec!["h".to_string()]
);
assert_eq!(
chunk.choices[0].logprobs.as_ref().expect("logprobs").text_offset,
vec![0]
);
}
CompletionSseChunk::Usage(_) => panic!("expected regular chunk"),
}
match &chunks[1] {
CompletionSseChunk::Chunk(chunk) => {
assert_eq!(chunk.choices[0].text, "");
assert_eq!(
chunk.choices[0].logprobs.as_ref().expect("logprobs").tokens,
vec!["!".to_string()]
);
assert_eq!(
chunk.choices[0].logprobs.as_ref().expect("logprobs").text_offset,
vec![1]
);
}
CompletionSseChunk::Usage(_) => panic!("expected regular chunk"),
}
}
}
@@ -0,0 +1,352 @@
use vllm_text::{SamplingParams, TextDecodeOptions, TextRequest};
use super::types::CompletionRequest;
use crate::error::ApiError;
use crate::routes::openai::completions::validate;
use crate::routes::openai::utils::structured_outputs::convert_from_response_format_value;
use crate::utils::{ResolvedRequestContext, convert_logit_bias, merge_kv_transfer_params};
/// Lowered completion request plus the public response metadata carried by
/// every SSE chunk.
#[derive(Debug, Clone, PartialEq)]
pub struct PreparedRequest {
/// Stable OpenAI-style request ID, reused as the external text request ID.
pub request_id: String,
/// Public model ID echoed back to the client.
pub response_model: String,
/// Whether the caller asked for the final streamed usage chunk.
pub include_usage: bool,
/// Lowered text request for the shared `vllm-text` facade.
pub text_request: TextRequest,
/// Original text prompt that should be echoed back northbound when
/// `echo=true`.
pub echo: Option<String>,
/// Whether to include token IDs alongside generated text.
pub return_token_ids: bool,
/// Whether to format logprob tokens as `token_id:{id}`.
pub return_tokens_as_token_ids: bool,
}
/// Validate and lower one OpenAI completions request into the internal
/// text-generation format.
///
/// `served_model_names` must be non-empty; the first entry is used as the
/// `model` field in responses.
pub(crate) fn prepare_completion_request(
request: CompletionRequest,
served_model_names: &[String],
ctx: ResolvedRequestContext,
) -> Result<PreparedRequest, ApiError> {
validate::validate_request_compat(&request, served_model_names)?;
let request_id = format!("cmpl-{}", ctx.request_id);
let logprobs = match request.logprobs {
Some(logprobs) => Some(i32::try_from(logprobs).map_err(|_| {
ApiError::invalid_request(
"`logprobs` must fit within a signed 32-bit integer.".to_string(),
Some("logprobs"),
)
})?),
None => None,
};
let prompt_logprobs = request.prompt_logprobs.or(if request.echo && !request.stream {
logprobs
} else {
None
});
let include_usage = (request.stream_options.as_ref())
.and_then(|options| options.include_usage)
.unwrap_or(false);
let echo = request.echo.then(|| request.prompt.as_text().cloned()).flatten();
let structured_outputs =
convert_from_response_format_value(&request.response_format, &request.structured_outputs)?;
let text_request = TextRequest {
request_id: request_id.clone(),
prompt: request.prompt,
mm_features: None,
sampling_params: SamplingParams {
temperature: request.temperature,
top_p: request.top_p,
top_k: request.top_k,
seed: request.seed,
max_tokens: request.max_tokens,
min_tokens: request.min_tokens,
logprobs,
prompt_logprobs,
min_p: request.min_p,
frequency_penalty: request.frequency_penalty,
presence_penalty: request.presence_penalty,
repetition_penalty: request.repetition_penalty,
stop_token_ids: request.stop_token_ids,
ignore_eos: request.ignore_eos,
logit_bias: convert_logit_bias(request.logit_bias)?,
allowed_token_ids: request.allowed_token_ids,
bad_words: None,
logprob_token_ids: None,
structured_outputs,
skip_reading_prefix_cache: None,
vllm_xargs: merge_kv_transfer_params(
request.vllm_xargs,
request.kv_transfer_params.as_ref(),
),
},
decode_options: TextDecodeOptions {
skip_special_tokens: request.skip_special_tokens,
include_stop_str_in_output: request.include_stop_str_in_output,
stop_strings: request.stop.map(|stop| stop.into_vec()),
min_tokens: request.min_tokens.unwrap_or(0),
},
intermediate: request.stream,
priority: request.priority.unwrap_or(0),
cache_salt: request.cache_salt,
add_special_tokens: request.add_special_tokens,
data_parallel_rank: ctx.data_parallel_rank,
};
Ok(PreparedRequest {
request_id,
response_model: served_model_names.first().cloned().unwrap_or_default(),
include_usage,
text_request,
echo,
return_token_ids: request.return_token_ids.unwrap_or(false),
return_tokens_as_token_ids: request.return_tokens_as_token_ids.unwrap_or(false),
})
}
#[cfg(test)]
mod tests {
use axum::http::HeaderMap;
use serde_json::json;
use vllm_text::Prompt;
use super::prepare_completion_request;
use crate::routes::openai::completions::types::CompletionRequest;
use crate::utils::{ResolvedRequestContext, resolve_request_context};
fn request_context(headers: &HeaderMap, request_id: Option<&str>) -> ResolvedRequestContext {
resolve_request_context(headers, request_id)
}
fn served(names: &[&str]) -> Vec<String> {
names.iter().map(|s| s.to_string()).collect()
}
fn base_request_json() -> serde_json::Value {
json!({
"model": "Qwen/Qwen1.5-0.5B-Chat",
"prompt": "hello",
"stream": true
})
}
#[test]
fn completion_http_request_deserializes_text_prompt() {
let request: CompletionRequest =
serde_json::from_value(base_request_json()).expect("parse request");
assert_eq!(request.prompt, Prompt::Text("hello".to_string()));
assert_eq!(request.model, "Qwen/Qwen1.5-0.5B-Chat");
}
#[test]
fn completion_http_request_deserializes_token_id_prompt() {
let request: CompletionRequest = serde_json::from_value(json!({
"model": "Qwen/Qwen1.5-0.5B-Chat",
"prompt": [11, 22, 33],
"stream": true,
"ignore_eos": true,
"max_tokens": 7
}))
.expect("parse request");
assert_eq!(request.prompt, Prompt::TokenIds(vec![11, 22, 33]));
assert_eq!(request.max_tokens, Some(7));
assert!(request.ignore_eos);
}
#[test]
fn prepare_completion_request_maps_sampling_fields() {
let request: CompletionRequest = serde_json::from_value(json!({
"model": "Qwen/Qwen1.5-0.5B-Chat",
"prompt": [11, 22, 33],
"stream": true,
"stream_options": {"include_usage": true},
"max_tokens": 7,
"logprobs": 2,
"top_p": 0.9,
"top_k": 42,
"min_p": 0.1,
"frequency_penalty": 0.2,
"presence_penalty": 0.3,
"repetition_penalty": 1.1,
"ignore_eos": true,
"skip_special_tokens": false
}))
.expect("parse request");
let prepared = prepare_completion_request(
request,
&served(&["Qwen/Qwen1.5-0.5B-Chat"]),
ResolvedRequestContext::default(),
)
.expect("prepare");
assert!(prepared.include_usage);
assert_eq!(
prepared.text_request.prompt,
Prompt::TokenIds(vec![11, 22, 33])
);
assert_eq!(prepared.text_request.sampling_params.max_tokens, Some(7));
assert_eq!(prepared.text_request.sampling_params.logprobs, Some(2));
assert_eq!(prepared.text_request.sampling_params.top_p, Some(0.9));
assert_eq!(prepared.text_request.sampling_params.top_k, Some(42));
assert_eq!(prepared.text_request.sampling_params.min_p, Some(0.1));
assert_eq!(
prepared.text_request.sampling_params.frequency_penalty,
Some(0.2)
);
assert_eq!(
prepared.text_request.sampling_params.presence_penalty,
Some(0.3)
);
assert_eq!(
prepared.text_request.sampling_params.repetition_penalty,
Some(1.1)
);
assert!(prepared.text_request.sampling_params.ignore_eos);
assert!(!prepared.text_request.decode_options.skip_special_tokens);
}
#[test]
fn prepare_completion_request_accepts_text_echo() {
let request: CompletionRequest = serde_json::from_value(json!({
"model": "Qwen/Qwen1.5-0.5B-Chat",
"prompt": "hello",
"stream": true,
"echo": true,
"max_tokens": 7
}))
.expect("parse request");
let prepared = prepare_completion_request(
request,
&served(&["Qwen/Qwen1.5-0.5B-Chat"]),
ResolvedRequestContext::default(),
)
.expect("prepare");
assert_eq!(prepared.echo, Some("hello".to_string()));
assert_eq!(prepared.text_request.sampling_params.max_tokens, Some(7));
}
#[test]
fn prepare_completion_request_enables_prompt_logprobs_for_non_stream_echo() {
let request: CompletionRequest = serde_json::from_value(json!({
"model": "Qwen/Qwen1.5-0.5B-Chat",
"prompt": "hello",
"echo": true,
"stream": false,
"logprobs": 3
}))
.expect("parse request");
let prepared = prepare_completion_request(
request,
&served(&["Qwen/Qwen1.5-0.5B-Chat"]),
ResolvedRequestContext::default(),
)
.expect("prepare");
assert_eq!(prepared.text_request.sampling_params.logprobs, Some(3));
assert_eq!(
prepared.text_request.sampling_params.prompt_logprobs,
Some(3)
);
}
#[test]
fn prepare_completion_request_rejects_token_id_prompt_echo() {
let request: CompletionRequest = serde_json::from_value(json!({
"model": "Qwen/Qwen1.5-0.5B-Chat",
"prompt": [11, 22, 33],
"stream": true,
"echo": true
}))
.expect("parse request");
assert!(
prepare_completion_request(
request,
&served(&["Qwen/Qwen1.5-0.5B-Chat"]),
ResolvedRequestContext::default(),
)
.is_err()
);
}
#[test]
fn prepare_completion_request_accepts_logprobs_fields() {
let request: CompletionRequest = serde_json::from_value(json!({
"model": "Qwen/Qwen1.5-0.5B-Chat",
"prompt": "hello",
"stream": false,
"logprobs": 1,
"prompt_logprobs": 2
}))
.expect("parse request");
let prepared = prepare_completion_request(
request,
&served(&["Qwen/Qwen1.5-0.5B-Chat"]),
ResolvedRequestContext::default(),
)
.expect("prepare");
assert_eq!(prepared.text_request.sampling_params.logprobs, Some(1));
assert_eq!(
prepared.text_request.sampling_params.prompt_logprobs,
Some(2)
);
}
#[test]
fn prepare_completion_request_threads_data_parallel_rank() {
let request: CompletionRequest = serde_json::from_value(json!({
"model": "Qwen/Qwen1.5-0.5B-Chat",
"prompt": "hello",
"stream": false,
}))
.expect("parse request");
let mut headers = HeaderMap::new();
headers.insert("X-data-parallel-rank", "3".parse().unwrap());
let prepared = prepare_completion_request(
request,
&served(&["Qwen/Qwen1.5-0.5B-Chat"]),
request_context(&headers, None),
)
.expect("prepare");
assert_eq!(prepared.text_request.data_parallel_rank, Some(3));
}
#[test]
fn prepare_completion_request_leaves_data_parallel_rank_none_when_absent() {
let request: CompletionRequest = serde_json::from_value(json!({
"model": "Qwen/Qwen1.5-0.5B-Chat",
"prompt": "hello",
"stream": false,
}))
.expect("parse request");
let prepared = prepare_completion_request(
request,
&served(&["Qwen/Qwen1.5-0.5B-Chat"]),
ResolvedRequestContext::default(),
)
.expect("prepare");
assert_eq!(prepared.text_request.data_parallel_rank, None);
}
}
@@ -0,0 +1,253 @@
use std::collections::HashMap;
use serde::{Deserialize, Serialize};
use serde_json::{Map, Value};
use validator::Validate;
use vllm_text::Prompt;
use crate::routes::openai::utils::types::{
LogProbs, Normalizable, StreamOptions, StringOrArray, Usage, default_true, validate_stop,
};
/// Serde default for `CompletionRequest::max_tokens`, matching the Python vLLM
/// / OpenAI default.
fn default_completion_max_tokens() -> Option<u32> {
Some(16)
}
/// vLLM-compatible request type for the Completions API.
///
/// Mirrors the Python vLLM `CompletionRequest` class. The local copy keeps the
/// request type route-owned so we can accept token-id prompts via
/// [`vllm_text::Prompt`] and add vLLM-only fields directly instead of layering
/// wrapper deserializers on top.
#[serde_with::skip_serializing_none]
#[derive(Debug, Clone, Deserialize, Serialize, Validate)]
pub struct CompletionRequest {
// -------- Standard OpenAI API Parameters --------
/// ID of the model to use
pub model: String,
/// The prompt(s) to generate completions for.
///
/// We use [`vllm_text::Prompt`] here to support token-id input.
pub prompt: Prompt,
/// Echo back the prompt in addition to the completion
#[serde(default)]
pub echo: bool,
/// Number between -2.0 and 2.0. Positive values penalize new tokens based
/// on their existing frequency in the text so far
pub frequency_penalty: Option<f32>,
/// Modify the likelihood of specified tokens appearing in the completion
pub logit_bias: Option<HashMap<String, f32>>,
/// Include the log probabilities on the logprobs most likely tokens
pub logprobs: Option<u32>,
/// The maximum number of tokens to generate (defaults to 16 when absent,
/// matching the Python vLLM / OpenAI API convention)
#[serde(default = "default_completion_max_tokens")]
pub max_tokens: Option<u32>,
/// How many completions to generate for each prompt
pub n: Option<u32>,
/// Number between -2.0 and 2.0. Positive values penalize new tokens based
/// on whether they appear in the text so far
pub presence_penalty: Option<f32>,
/// If specified, our system will make a best effort to sample
/// deterministically
pub seed: Option<i64>,
/// Up to 4 sequences where the API will stop generating further tokens
#[validate(custom(function = "validate_stop"))]
pub stop: Option<StringOrArray>,
/// Whether to stream back partial progress
#[serde(default)]
pub stream: bool,
/// The suffix that comes after a completion of inserted text
pub suffix: Option<String>,
/// What sampling temperature to use, between 0 and 2
pub temperature: Option<f32>,
/// An alternative to sampling with temperature (nucleus sampling)
pub top_p: Option<f32>,
/// A unique identifier representing your end-user
pub user: Option<String>,
// -------- vLLM Sampling Parameters --------
/// Options for streaming response
pub stream_options: Option<StreamOptions>,
/// Use beam search instead of sampling
#[serde(default)]
pub use_beam_search: bool,
/// Top-k sampling parameter
pub top_k: Option<u32>,
/// Min-p nucleus sampling parameter
pub min_p: Option<f32>,
/// Repetition penalty for reducing repetitive text
pub repetition_penalty: Option<f32>,
/// Length penalty for beam search
pub length_penalty: Option<f32>,
/// Specific token IDs to use as stop conditions
pub stop_token_ids: Option<Vec<u32>>,
/// Include stop string in output
#[serde(default)]
pub include_stop_str_in_output: bool,
/// Ignore end-of-sequence tokens during generation
#[serde(default)]
pub ignore_eos: bool,
/// Minimum number of tokens to generate
pub min_tokens: Option<u32>,
/// Skip special tokens during detokenization
#[serde(default = "default_true")]
pub skip_special_tokens: bool,
/// Add spaces between special tokens during detokenization
#[serde(default = "default_true")]
pub spaces_between_special_tokens: bool,
/// Truncate prompt tokens to this length
pub truncate_prompt_tokens: Option<i64>,
/// Restrict output to these token IDs only
pub allowed_token_ids: Option<Vec<u32>>,
/// Number of prompt logprobs to return
pub prompt_logprobs: Option<i32>,
// -------- Extra vLLM Parameters --------
/// Whether to add special tokens (e.g. BOS) to the prompt
#[serde(default = "default_true")]
pub add_special_tokens: bool,
/// Format specification for structured output (JSON mode, JSON schema,
/// etc.)
pub response_format: Option<Value>,
/// Additional kwargs for structured outputs
pub structured_outputs: Option<Value>,
/// Request scheduling priority (lower means earlier; default 0)
pub priority: Option<i32>,
/// External request ID used for response correlation.
pub request_id: Option<String>,
/// Tokens represented as strings of the form 'token_id:{token_id}' in
/// logprobs
pub return_tokens_as_token_ids: Option<bool>,
/// Include token IDs alongside generated text
pub return_token_ids: Option<bool>,
/// Salt for prefix cache isolation in multi-user environments
pub cache_salt: Option<String>,
/// KV transfer parameters for disaggregated serving
pub kv_transfer_params: Option<HashMap<String, Value>>,
/// Additional request parameters with string or numeric values for custom
/// extensions
pub vllm_xargs: Option<HashMap<String, Value>>,
/// Additional fields
#[serde(flatten)]
pub other: Map<String, Value>,
}
impl Normalizable for CompletionRequest {}
/// Mirrors the Python vLLM `CompletionResponse` class.
#[serde_with::skip_serializing_none]
#[derive(Debug, Clone, Serialize)]
pub(super) struct CompletionResponse {
pub id: String,
pub object: String,
pub created: u64,
pub model: String,
pub choices: Vec<CompletionChoice>,
pub usage: Option<Usage>,
pub system_fingerprint: Option<String>,
pub kv_transfer_params: Option<Value>,
}
/// Mirrors the Python vLLM `CompletionResponseChoice` class.
#[serde_with::skip_serializing_none]
#[derive(Debug, Clone, Serialize)]
pub(super) struct CompletionChoice {
pub index: u32,
pub text: String,
pub logprobs: Option<LogProbs>,
pub finish_reason: Option<String>,
pub stop_reason: Option<Value>,
pub prompt_logprobs: Option<Vec<Option<HashMap<String, f32>>>>,
pub token_ids: Option<Vec<u32>>,
pub prompt_token_ids: Option<Vec<u32>>,
}
/// Mirrors the Python vLLM `CompletionStreamResponse` class.
#[serde_with::skip_serializing_none]
#[derive(Debug, Clone, Serialize)]
pub(super) struct CompletionStreamResponse {
pub id: String,
pub object: String,
pub created: u64,
pub model: String,
pub choices: Vec<CompletionStreamChoice>,
pub usage: Option<Usage>,
}
impl CompletionStreamResponse {
/// Create a stream response with the standard envelope fields pre-filled.
pub fn new(id: &str, model: &str, created: u64) -> Self {
Self {
id: id.to_string(),
object: "text_completion".to_string(),
created,
model: model.to_string(),
choices: Vec::new(),
usage: None,
}
}
}
/// Mirrors the Python vLLM `CompletionResponseStreamChoice` class.
#[serde_with::skip_serializing_none]
#[derive(Debug, Clone, Default, Serialize)]
pub(super) struct CompletionStreamChoice {
pub index: u32,
pub text: String,
pub logprobs: Option<LogProbs>,
pub finish_reason: Option<String>,
pub stop_reason: Option<Value>,
pub token_ids: Option<Vec<u32>>,
pub prompt_token_ids: Option<Vec<u32>>,
}
#[derive(Debug, Clone, Serialize)]
#[serde(untagged)]
pub(super) enum CompletionSseChunk {
/// Ordinary OpenAI completions delta/final chunk.
Chunk(CompletionStreamResponse),
/// Final usage chunk emitted before `[DONE]` when `include_usage=true`.
Usage(CompletionStreamResponse),
}
@@ -0,0 +1,178 @@
use vllm_text::Prompt;
use super::types::CompletionRequest;
use crate::error::{ApiError, bail_invalid_request};
/// Enforce the minimal compatibility contract for the Rust OpenAI server.
pub(super) fn validate_request_compat(
request: &CompletionRequest,
served_model_names: &[String],
) -> Result<(), ApiError> {
// This path is intentionally scoped to the minimum surface needed by
// `vllm-bench` random workload compatibility, so unsupported legacy
// completions features fail early here.
if !served_model_names.iter().any(|n| n == &request.model) {
return Err(ApiError::model_not_found(request.model.clone()));
}
if request.stream_options.is_some() && !request.stream {
bail_invalid_request!(
param = "stream_options",
"stream_options are only supported when stream=true."
);
}
if request.n.unwrap_or(1) > 1 {
bail_invalid_request!(param = "n", "Only n=1 is supported.");
}
if request.max_tokens == Some(0) {
bail_invalid_request!(param = "max_tokens", "max_tokens must be greater than 0.");
}
if request.echo && matches!(request.prompt, Prompt::TokenIds(_)) {
bail_invalid_request!(
param = "echo",
"echo is not supported with token-ID prompts."
);
}
if request.suffix.is_some() {
bail_invalid_request!(param = "suffix", "suffix is not supported.");
}
if let Some(logprobs) = request.logprobs
&& logprobs > i32::MAX as u32
{
bail_invalid_request!(
param = "logprobs",
"`logprobs` must fit within a signed 32-bit integer."
);
}
if let Some(prompt_logprobs) = request.prompt_logprobs {
if request.stream && (prompt_logprobs > 0 || prompt_logprobs == -1) {
bail_invalid_request!(
param = "prompt_logprobs",
"`prompt_logprobs` are not available when `stream=true`."
);
}
if prompt_logprobs < 0 && prompt_logprobs != -1 {
bail_invalid_request!(
param = "prompt_logprobs",
"`prompt_logprobs` must be a non-negative value or -1."
);
}
}
if request.use_beam_search {
bail_invalid_request!(
param = "use_beam_search",
"use_beam_search is not supported."
);
}
// ---- Reject parameters that are accepted for deserialization but not yet
// implemented ----
if request.length_penalty.is_some() {
bail_invalid_request!(param = "length_penalty", "length_penalty is not supported.");
}
if !request.spaces_between_special_tokens {
bail_invalid_request!(
param = "spaces_between_special_tokens",
"spaces_between_special_tokens is not supported."
);
}
if request.truncate_prompt_tokens.is_some() {
bail_invalid_request!(
param = "truncate_prompt_tokens",
"truncate_prompt_tokens is not supported."
);
}
if let Some(options) = &request.stream_options
&& options.continuous_usage_stats.is_some()
{
bail_invalid_request!(
param = "stream_options",
"continuous_usage_stats is not supported."
);
}
Ok(())
}
#[cfg(test)]
mod tests {
use serde_json::json;
use super::validate_request_compat;
use crate::routes::openai::completions::types::CompletionRequest;
fn base_request() -> CompletionRequest {
serde_json::from_value(json!({
"model": "Qwen/Qwen1.5-0.5B-Chat",
"prompt": "hello",
"stream": true,
}))
.expect("parse request")
}
fn served_names(names: &[&str]) -> Vec<String> {
names.iter().map(|s| s.to_string()).collect()
}
#[test]
fn validate_request_compat_accepts_logprobs() {
let request = CompletionRequest {
logprobs: Some(1),
..base_request()
};
assert!(
validate_request_compat(&request, &served_names(&["Qwen/Qwen1.5-0.5B-Chat"])).is_ok()
);
}
#[test]
fn validate_request_compat_accepts_any_served_name() {
let request = base_request();
assert!(
validate_request_compat(
&request,
&served_names(&["other-alias", "Qwen/Qwen1.5-0.5B-Chat"])
)
.is_ok()
);
}
#[test]
fn validate_request_compat_rejects_unknown_model() {
let request = base_request();
assert!(validate_request_compat(&request, &served_names(&["other-model"])).is_err());
}
#[test]
fn validate_request_compat_rejects_streaming_prompt_logprobs() {
let request = CompletionRequest {
prompt_logprobs: Some(1),
..base_request()
};
assert!(
validate_request_compat(&request, &served_names(&["Qwen/Qwen1.5-0.5B-Chat"])).is_err()
);
}
#[test]
fn validate_request_compat_accepts_non_stream_prompt_logprobs() {
let request = CompletionRequest {
stream: false,
prompt_logprobs: Some(-1),
..base_request()
};
assert!(
validate_request_compat(&request, &served_names(&["Qwen/Qwen1.5-0.5B-Chat"])).is_ok()
);
}
}
+8
View File
@@ -0,0 +1,8 @@
pub mod chat_completions;
mod completions;
mod models;
pub(crate) mod utils;
pub use chat_completions::chat_completions;
pub use completions::completions;
pub use models::list_models;
@@ -0,0 +1,24 @@
use std::sync::Arc;
use axum::Json;
use axum::extract::State;
use crate::routes::openai::utils::types::{ListModelsResponse, ModelObject};
use crate::state::AppState;
/// Return all configured served model names in OpenAI `list models` format.
pub async fn list_models(State(state): State<Arc<AppState>>) -> Json<ListModelsResponse> {
Json(ListModelsResponse {
object: "list".to_string(),
data: state
.served_model_names()
.iter()
.map(|name| ModelObject {
id: name.clone(),
object: "model".to_string(),
created: 0,
owned_by: "vllm-frontend-rs".to_string(),
})
.collect(),
})
}
@@ -0,0 +1,256 @@
use std::collections::HashMap;
use itertools::Itertools as _;
use vllm_text::{
CollectedTextOutput, DecodedLogprobs, DecodedPositionLogprobs, DecodedPromptLogprobs,
DecodedTokenLogprob,
};
use super::types::{ChatLogProbs, ChatLogProbsContent, LogProbs, TopLogProb};
use crate::error::{ApiError, server_error};
/// Convert decoded token-position logprobs into the OpenAI completions
/// `logprobs` shape.
pub fn decoded_logprobs_to_openai(
logprobs: &DecodedLogprobs,
initial_text_offset: u32,
return_tokens_as_token_ids: bool,
) -> Result<LogProbs, ApiError> {
let mut text_offset = Vec::with_capacity(logprobs.positions.len());
let mut token_logprobs = Vec::with_capacity(logprobs.positions.len());
let mut tokens = Vec::with_capacity(logprobs.positions.len());
let mut top_logprobs = Vec::with_capacity(logprobs.positions.len());
let mut current_offset = initial_text_offset;
for position in &logprobs.positions {
let chosen = position.entries.first().ok_or_else(|| {
server_error!("decoded logprobs position unexpectedly had no token candidates")
})?;
let token_str = format_token(chosen, return_tokens_as_token_ids);
text_offset.push(current_offset);
token_logprobs.push(Some(clamp_logprob(chosen.logprob)));
current_offset = current_offset.saturating_add(text_len(&token_str));
tokens.push(token_str);
top_logprobs.push(Some(position_top_logprobs_map(
position,
return_tokens_as_token_ids,
)));
}
Ok(LogProbs {
tokens,
token_logprobs,
top_logprobs,
text_offset,
})
}
/// Convert decoded prompt logprobs into the OpenAI completions `logprobs`
/// shape.
///
/// The first prompt token is included with `None` logprob metadata, matching
/// Python vLLM's echoed completions behavior.
pub fn decoded_prompt_logprobs_to_openai(
prompt_logprobs: &DecodedPromptLogprobs,
initial_text_offset: u32,
return_tokens_as_token_ids: bool,
) -> Result<LogProbs, ApiError> {
let mut text_offset = Vec::with_capacity(prompt_logprobs.scored_positions.len() + 1);
let mut token_logprobs = Vec::with_capacity(prompt_logprobs.scored_positions.len() + 1);
let mut tokens = Vec::with_capacity(prompt_logprobs.scored_positions.len() + 1);
let mut top_logprobs = Vec::with_capacity(prompt_logprobs.scored_positions.len() + 1);
let mut current_offset = initial_text_offset;
let first_token_str = if return_tokens_as_token_ids {
format!("token_id:{}", prompt_logprobs.first_token_id)
} else {
prompt_logprobs.first_token.clone()
};
text_offset.push(current_offset);
token_logprobs.push(None);
current_offset = current_offset.saturating_add(text_len(&first_token_str));
tokens.push(first_token_str);
top_logprobs.push(None);
for position in &prompt_logprobs.scored_positions {
let chosen = position.entries.first().ok_or_else(|| {
server_error!("decoded prompt logprobs position unexpectedly had no token candidates")
})?;
let token_str = format_token(chosen, return_tokens_as_token_ids);
text_offset.push(current_offset);
token_logprobs.push(Some(clamp_logprob(chosen.logprob)));
current_offset = current_offset.saturating_add(text_len(&token_str));
tokens.push(token_str);
top_logprobs.push(Some(position_top_logprobs_map(
position,
return_tokens_as_token_ids,
)));
}
Ok(LogProbs {
tokens,
token_logprobs,
top_logprobs,
text_offset,
})
}
/// Convert decoded prompt logprobs into the vLLM-style prompt-logprobs response
/// shape.
pub fn decoded_prompt_logprobs_to_maps(
prompt_logprobs: &DecodedPromptLogprobs,
return_tokens_as_token_ids: bool,
) -> Vec<Option<HashMap<String, f32>>> {
std::iter::once(None)
.chain(prompt_logprobs.scored_positions.iter().map(|position| {
Some(position_top_logprobs_map(
position,
return_tokens_as_token_ids,
))
}))
.collect()
}
/// Convert decoded token-position logprobs into the OpenAI chat `logprobs`
/// shape.
pub fn decoded_logprobs_to_openai_chat(
logprobs: &DecodedLogprobs,
return_tokens_as_token_ids: bool,
) -> Result<ChatLogProbs, ApiError> {
let content = logprobs
.positions
.iter()
.map(|pos| position_to_chat_logprobs_content(pos, return_tokens_as_token_ids))
.try_collect()?;
Ok(ChatLogProbs {
content: Some(content),
})
}
/// Count visible text positions using OpenAI completions' character-offset
/// convention.
pub fn text_len(text: &str) -> u32 {
u32::try_from(text.chars().count()).unwrap_or(u32::MAX)
}
/// Concatenate two OpenAI-style completion logprobs payloads in token order.
pub fn append_openai_logprobs(mut prefix: LogProbs, suffix: LogProbs) -> LogProbs {
prefix.tokens.extend(suffix.tokens);
prefix.token_logprobs.extend(suffix.token_logprobs);
prefix.top_logprobs.extend(suffix.top_logprobs);
prefix.text_offset.extend(suffix.text_offset);
prefix
}
/// Build the non-stream completions `logprobs` payload from collected text
/// output.
///
/// When `echoed_prompt` is true, the returned payload matches Python vLLM's
/// echoed completions behavior by concatenating prompt and completion logprobs
/// into one OpenAI `LogProbs` object.
pub fn collected_logprobs_to_openai(
collected: &CollectedTextOutput,
echoed_prompt: bool,
initial_completion_offset: u32,
return_tokens_as_token_ids: bool,
) -> Result<LogProbs, ApiError> {
if echoed_prompt {
let prompt_logprobs = collected.prompt_logprobs.as_ref().ok_or_else(|| {
server_error!(
"echoed completion logprobs require prompt logprobs but generation returned none"
)
})?;
let prompt_logprobs =
decoded_prompt_logprobs_to_openai(prompt_logprobs, 0, return_tokens_as_token_ids)?;
let completion_start = prompt_logprobs
.text_offset
.last()
.zip(prompt_logprobs.tokens.last())
.map(|(&offset, token)| offset.saturating_add(text_len(token)))
.unwrap_or(0);
return match collected.logprobs.as_ref() {
Some(completion_logprobs) => Ok(append_openai_logprobs(
prompt_logprobs,
decoded_logprobs_to_openai(
completion_logprobs,
completion_start,
return_tokens_as_token_ids,
)?,
)),
None => Ok(prompt_logprobs),
};
}
let completion_logprobs = collected.logprobs.as_ref().ok_or_else(|| {
server_error!("completion response requested logprobs but generation returned none")
})?;
decoded_logprobs_to_openai(
completion_logprobs,
initial_completion_offset,
return_tokens_as_token_ids,
)
}
/// Format a token entry as either its decoded string or `token_id:{id}`.
fn format_token(entry: &DecodedTokenLogprob, as_token_id: bool) -> String {
if as_token_id {
format!("token_id:{}", entry.token_id)
} else {
entry.token.clone()
}
}
fn position_top_logprobs_map(
position: &DecodedPositionLogprobs,
return_tokens_as_token_ids: bool,
) -> HashMap<String, f32> {
position
.entries
.iter()
.map(|entry| {
(
format_token(entry, return_tokens_as_token_ids),
clamp_logprob(entry.logprob),
)
})
.collect()
}
fn position_to_chat_logprobs_content(
position: &DecodedPositionLogprobs,
return_tokens_as_token_ids: bool,
) -> Result<ChatLogProbsContent, ApiError> {
let chosen = position.entries.first().ok_or_else(|| {
server_error!("decoded chat logprobs position unexpectedly had no token candidates")
})?;
let token_str = format_token(chosen, return_tokens_as_token_ids);
Ok(ChatLogProbsContent {
token: token_str.clone(),
logprob: clamp_logprob(chosen.logprob),
bytes: Some(token_bytes(&token_str)),
top_logprobs: position
.entries
.iter()
.map(|entry| {
let t = format_token(entry, return_tokens_as_token_ids);
TopLogProb {
logprob: clamp_logprob(entry.logprob),
bytes: Some(token_bytes(&t)),
token: t,
}
})
.collect(),
})
}
fn token_bytes(token: &str) -> Vec<u8> {
token.as_bytes().to_vec()
}
pub fn clamp_logprob(logprob: f32) -> f32 {
logprob.max(-9999.0)
}
@@ -0,0 +1,4 @@
pub mod logprobs;
pub mod structured_outputs;
pub mod types;
pub mod validated_json;
@@ -0,0 +1,132 @@
use serde::{Deserialize, Serialize};
use serde_json::Value;
use vllm_engine_core_client::protocol::StructuredOutputsParams;
use crate::error::ApiError;
/// JSON schema specification nested inside a `json_schema` response format.
///
/// Mirrors the Python vLLM `JsonSchemaResponseFormat` class.
#[serde_with::skip_serializing_none]
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
pub struct JsonSchemaFormat {
pub name: String,
#[serde(default)]
pub description: Option<String>,
/// The actual JSON schema object.
#[serde(alias = "json_schema")]
pub schema: Value,
#[serde(default)]
pub strict: Option<bool>,
}
/// Supported `response_format` types for chat and completion requests.
///
/// This is our own definition (rather than the `openai-protocol` crate's) so
/// that we can support the vLLM-specific `structural_tag` variant.
///
/// Original Python definitions:
/// <https://github.com/vllm-project/vllm/blob/f22d6e026/vllm/entrypoints/openai/engine/protocol.py#L116-L157>
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
#[serde(tag = "type", rename_all = "snake_case")]
pub enum ResponseFormat {
Text,
JsonObject,
JsonSchema {
json_schema: JsonSchemaFormat,
},
/// vLLM-specific structural tag format. The entire object (including the
/// `type` field) is JSON-serialized and passed as
/// `StructuredOutputsParams.structural_tag`.
///
/// We capture the payload as a catch-all map so both the legacy
/// (`structures`/`triggers`) and current (`format`) shapes are
/// preserved without needing typed structs.
StructuralTag {
#[serde(flatten)]
extra: serde_json::Map<String, Value>,
},
}
/// Convert an explicit `structured_outputs` JSON blob into
/// [`StructuredOutputsParams`].
fn deserialize_structured_outputs(
raw: &serde_json::Value,
) -> Result<StructuredOutputsParams, ApiError> {
serde_json::from_value(raw.clone()).map_err(|e| {
ApiError::invalid_request(
format!("invalid structured_outputs: {e}"),
Some("structured_outputs"),
)
})
}
/// Convert a typed [`ResponseFormat`] and/or raw `structured_outputs` blob into
/// engine-core [`StructuredOutputsParams`].
///
/// Mirrors the Python vLLM conversion in
/// `ChatCompletionRequest.to_sampling_params()`: <https://github.com/vllm-project/vllm/blob/f22d6e026/vllm/entrypoints/openai/chat_completion/protocol.py#L457-L487>
pub fn convert_from_response_format(
response_format: Option<&ResponseFormat>,
structured_outputs: &Option<serde_json::Value>,
) -> Result<Option<StructuredOutputsParams>, ApiError> {
if let Some(raw) = structured_outputs {
return Ok(Some(deserialize_structured_outputs(raw)?));
}
let Some(fmt) = response_format else {
return Ok(None);
};
match fmt {
ResponseFormat::Text => Ok(None),
ResponseFormat::JsonObject => Ok(Some(StructuredOutputsParams {
json_object: Some(true),
..Default::default()
})),
ResponseFormat::JsonSchema { json_schema } => Ok(Some(StructuredOutputsParams {
json: Some(json_schema.schema.clone()),
..Default::default()
})),
ResponseFormat::StructuralTag { .. } => {
// The Python frontend dumps the entire response_format object (including the
// `type` field) as a JSON string for the engine-core backend.
let tag_json = serde_json::to_string(fmt).map_err(|e| {
ApiError::invalid_request(
format!("failed to serialize structural_tag: {e}"),
Some("response_format"),
)
})?;
Ok(Some(StructuredOutputsParams {
structural_tag: Some(tag_json),
..Default::default()
}))
}
}
}
/// Convert raw `response_format` and/or `structured_outputs` JSON blobs into
/// engine-core [`StructuredOutputsParams`].
///
/// Used by the completions endpoint which keeps both fields as opaque
/// `serde_json::Value`.
pub fn convert_from_response_format_value(
response_format: &Option<serde_json::Value>,
structured_outputs: &Option<serde_json::Value>,
) -> Result<Option<StructuredOutputsParams>, ApiError> {
if let Some(raw) = structured_outputs {
return Ok(Some(deserialize_structured_outputs(raw)?));
}
let Some(raw) = response_format else {
return Ok(None);
};
// Deserialize into our typed enum and delegate.
let fmt: ResponseFormat = serde_json::from_value(raw.clone()).map_err(|e| {
ApiError::invalid_request(
format!("invalid response_format: {e}"),
Some("response_format"),
)
})?;
convert_from_response_format(Some(&fmt), &None)
}
@@ -0,0 +1,425 @@
use std::collections::HashMap;
use std::slice;
use llm_multimodal::ImageDetail;
use serde::{Deserialize, Serialize};
use serde_json::Value;
// ============================================================================
// Constants
// ============================================================================
/// Default model identifier used when no model is specified.
pub const UNKNOWN_MODEL_ID: &str = "unknown";
// ============================================================================
// Default value helpers
// ============================================================================
/// Helper function for serde default value (returns true).
pub fn default_true() -> bool {
true
}
// ============================================================================
// String/Array Utilities
// ============================================================================
/// A type that can be either a single string or an array of strings.
#[derive(Debug, Clone, PartialEq, Deserialize, Serialize)]
#[serde(untagged)]
pub enum StringOrArray {
String(String),
Array(Vec<String>),
}
impl StringOrArray {
pub fn as_slice(&self) -> &[String] {
match self {
StringOrArray::String(s) => slice::from_ref(s),
StringOrArray::Array(arr) => arr,
}
}
#[allow(unused)]
pub fn into_vec(self) -> Vec<String> {
match self {
StringOrArray::String(s) => vec![s],
StringOrArray::Array(arr) => arr,
}
}
}
/// Validates stop sequences (non-empty strings)
pub fn validate_stop(stop: &StringOrArray) -> Result<(), validator::ValidationError> {
if stop.as_slice().iter().any(|s| s.is_empty()) {
return Err(validator::ValidationError::new(
"stop strings cannot be empty",
));
}
Ok(())
}
// ============================================================================
// Validation helpers
// ============================================================================
/// Validates top_p: 0.0 < top_p <= 1.0.
pub fn validate_top_p_value(top_p: f32) -> Result<(), validator::ValidationError> {
if !(top_p > 0.0 && top_p <= 1.0) {
return Err(validator::ValidationError::new(
"top_p must be in (0, 1] - greater than 0.0 and at most 1.0",
));
}
Ok(())
}
// ============================================================================
// Content Parts (for multimodal messages)
// ============================================================================
#[derive(Debug, Clone, Deserialize, Serialize, PartialEq)]
#[serde(tag = "type")]
pub enum ContentPart {
#[serde(rename = "text")]
Text { text: String },
#[serde(rename = "image_url")]
ImageUrl {
image_url: ImageUrl,
#[serde(skip_serializing_if = "Option::is_none")]
uuid: Option<String>,
},
#[serde(rename = "video_url")]
VideoUrl { video_url: VideoUrl },
}
#[serde_with::skip_serializing_none]
#[derive(Debug, Clone, Deserialize, Serialize, PartialEq)]
pub struct ImageUrl {
pub url: String,
pub detail: Option<ImageDetail>,
}
#[derive(Debug, Clone, Deserialize, Serialize, PartialEq)]
pub struct VideoUrl {
pub url: String,
}
// ============================================================================
// Streaming
// ============================================================================
/// Mirrors the Python vLLM `StreamOptions` class.
#[derive(Debug, Clone, Deserialize, Serialize)]
pub struct StreamOptions {
pub include_usage: Option<bool>,
pub continuous_usage_stats: Option<bool>,
}
// ============================================================================
// Tools and Function Calling
// ============================================================================
#[derive(Debug, Clone, Deserialize, Serialize)]
pub struct Tool {
#[serde(rename = "type")]
pub tool_type: String,
pub function: Function,
}
#[serde_with::skip_serializing_none]
#[derive(Debug, Clone, Deserialize, Serialize)]
pub struct Function {
pub name: String,
pub description: Option<String>,
pub parameters: Value,
/// Whether to enable strict schema adherence (OpenAI structured outputs).
pub strict: Option<bool>,
}
#[derive(Debug, Clone, Deserialize, Serialize)]
pub struct ToolCall {
pub id: String,
#[serde(rename = "type")]
pub tool_type: String,
pub function: FunctionCallResponse,
}
#[derive(Debug, Clone, Deserialize, Serialize)]
pub struct FunctionCallResponse {
pub name: String,
#[serde(default)]
pub arguments: Option<String>,
}
#[serde_with::skip_serializing_none]
#[derive(Debug, Clone, Deserialize, Serialize)]
pub struct ToolCallDelta {
pub index: u32,
pub id: Option<String>,
#[serde(rename = "type")]
pub tool_type: Option<String>,
pub function: Option<FunctionCallDelta>,
}
#[serde_with::skip_serializing_none]
#[derive(Debug, Clone, Deserialize, Serialize)]
pub struct FunctionCallDelta {
pub name: Option<String>,
pub arguments: Option<String>,
}
/// Tool choice value for simple string options.
#[derive(Debug, Clone, Deserialize, Serialize)]
#[serde(rename_all = "snake_case")]
pub enum ToolChoiceValue {
Auto,
Required,
None,
}
/// Tool choice for the Chat Completion API.
#[derive(Debug, Clone, Deserialize, Serialize)]
#[serde(untagged)]
pub enum ToolChoice {
Value(ToolChoiceValue),
Function {
#[serde(rename = "type")]
tool_type: String,
function: FunctionChoice,
},
AllowedTools {
#[serde(rename = "type")]
tool_type: String,
mode: String,
tools: Vec<ToolReference>,
},
}
impl Default for ToolChoice {
fn default() -> Self {
Self::Value(ToolChoiceValue::Auto)
}
}
/// Function choice specification for `ToolChoice::Function`.
#[derive(Debug, Clone, Deserialize, Serialize)]
pub struct FunctionChoice {
pub name: String,
}
/// Tool reference for `ToolChoice::AllowedTools`.
#[derive(Debug, Clone, Deserialize, Serialize)]
#[serde(tag = "type")]
#[serde(rename_all = "snake_case")]
pub enum ToolReference {
#[serde(rename = "function")]
Function { name: String },
#[serde(rename = "mcp")]
Mcp {
server_label: String,
#[serde(skip_serializing_if = "Option::is_none")]
name: Option<String>,
},
#[serde(rename = "file_search")]
FileSearch,
#[serde(rename = "web_search_preview")]
WebSearchPreview,
#[serde(rename = "computer_use_preview")]
ComputerUsePreview,
#[serde(rename = "code_interpreter")]
CodeInterpreter,
#[serde(rename = "image_generation")]
ImageGeneration,
}
impl ToolReference {
/// Get a unique identifier for this tool reference.
pub fn identifier(&self) -> String {
match self {
ToolReference::Function { name } => format!("function:{name}"),
ToolReference::Mcp {
server_label,
name: Some(n),
} => format!("mcp:{server_label}:{n}"),
ToolReference::Mcp {
server_label,
name: _,
} => format!("mcp:{server_label}"),
ToolReference::FileSearch => "file_search".to_string(),
ToolReference::WebSearchPreview => "web_search_preview".to_string(),
ToolReference::ComputerUsePreview => "computer_use_preview".to_string(),
ToolReference::CodeInterpreter => "code_interpreter".to_string(),
ToolReference::ImageGeneration => "image_generation".to_string(),
}
}
}
// ============================================================================
// Chat Messages
// ============================================================================
#[serde_with::skip_serializing_none]
#[derive(Debug, Clone, Deserialize, Serialize)]
#[serde(tag = "role")]
pub enum ChatMessage {
#[serde(rename = "system")]
System {
content: MessageContent,
name: Option<String>,
},
#[serde(rename = "user")]
User {
content: MessageContent,
name: Option<String>,
},
#[serde(rename = "assistant")]
Assistant {
content: Option<MessageContent>,
name: Option<String>,
tool_calls: Option<Vec<ToolCall>>,
/// Reasoning content for reasoning-capable models.
#[serde(alias = "reasoning_content")]
#[serde(alias = "thinking")]
reasoning: Option<String>,
},
#[serde(rename = "tool")]
Tool {
content: MessageContent,
tool_call_id: String,
},
#[serde(rename = "function")]
Function { content: String, name: String },
#[serde(rename = "developer")]
Developer {
content: MessageContent,
tools: Option<Vec<Tool>>,
name: Option<String>,
},
}
#[derive(Debug, Clone, Deserialize, Serialize, PartialEq)]
#[serde(untagged)]
pub enum MessageContent {
Text(String),
Parts(Vec<ContentPart>),
}
// ============================================================================
// Usage and Logging
// ============================================================================
/// Mirrors the Python vLLM `UsageInfo` class.
#[serde_with::skip_serializing_none]
#[derive(Debug, Clone, Serialize)]
pub struct Usage {
pub prompt_tokens: u32,
pub total_tokens: u32,
pub completion_tokens: Option<u32>,
pub prompt_tokens_details: Option<PromptTokenUsageInfo>,
}
impl Usage {
/// Create a Usage from prompt and completion token counts.
pub fn from_counts(prompt_tokens: u32, completion_tokens: u32) -> Self {
Self {
prompt_tokens,
total_tokens: prompt_tokens + completion_tokens,
completion_tokens: Some(completion_tokens),
prompt_tokens_details: None,
}
}
}
/// Mirrors the Python vLLM `PromptTokenUsageInfo` class.
#[serde_with::skip_serializing_none]
#[derive(Debug, Clone, Serialize)]
pub struct PromptTokenUsageInfo {
pub cached_tokens: Option<u32>,
}
/// OpenAI completions-style logprobs.
#[derive(Debug, Clone, Deserialize, Serialize)]
pub struct LogProbs {
pub tokens: Vec<String>,
pub token_logprobs: Vec<Option<f32>>,
pub top_logprobs: Vec<Option<HashMap<String, f32>>>,
pub text_offset: Vec<u32>,
}
/// Mirrors the Python vLLM `ChatCompletionLogProbs` class.
#[serde_with::skip_serializing_none]
#[derive(Debug, Clone, Serialize)]
pub struct ChatLogProbs {
pub content: Option<Vec<ChatLogProbsContent>>,
}
/// Mirrors the Python vLLM `ChatCompletionLogProbsContent` class.
#[serde_with::skip_serializing_none]
#[derive(Debug, Clone, Serialize)]
pub struct ChatLogProbsContent {
pub token: String,
pub logprob: f32,
pub bytes: Option<Vec<u8>>,
pub top_logprobs: Vec<TopLogProb>,
}
/// Mirrors the Python vLLM `ChatCompletionLogProb` class.
#[serde_with::skip_serializing_none]
#[derive(Debug, Clone, Serialize)]
pub struct TopLogProb {
pub token: String,
pub logprob: f32,
pub bytes: Option<Vec<u8>>,
}
// ============================================================================
// Error Types
// ============================================================================
#[derive(Debug, Clone, Deserialize, Serialize)]
pub struct ErrorResponse {
pub error: ErrorDetail,
}
#[serde_with::skip_serializing_none]
#[derive(Debug, Clone, Deserialize, Serialize)]
pub struct ErrorDetail {
pub message: String,
#[serde(rename = "type")]
pub error_type: String,
pub param: Option<String>,
pub code: Option<String>,
}
// ============================================================================
// Model types
// ============================================================================
/// A single model entry in the `/v1/models` response.
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct ModelObject {
pub id: String,
pub object: String,
pub created: i64,
pub owned_by: String,
}
/// Response body for `GET /v1/models`.
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct ListModelsResponse {
pub object: String,
pub data: Vec<ModelObject>,
}
// ============================================================================
// Normalizable trait
// ============================================================================
/// Trait for request types that need post-deserialization normalization.
pub trait Normalizable {
/// Normalize the request by applying defaults and transformations.
fn normalize(&mut self) {
// Default: no-op
}
}
@@ -0,0 +1,55 @@
//! Validated JSON extractor for automatic request validation.
//! Variation of https://github.com/lightseekorg/smg/blob/main/crates/protocols/src/validated.rs
use axum::Json;
use axum::extract::rejection::JsonRejection;
use axum::extract::{FromRequest, Request};
use serde::de::DeserializeOwned;
use validator::Validate;
use super::types::Normalizable;
use crate::error::{ApiError, invalid_request};
/// A JSON extractor that automatically validates and normalizes the request
/// body.
///
/// This extractor deserializes the request body and automatically calls
/// `.validate()` on types that implement the `Validate` trait. If validation
/// fails, it returns [`ApiError::InvalidRequest`] with details about the
/// validation errors.
pub struct ValidatedJson<T>(pub T);
impl<S, T> FromRequest<S> for ValidatedJson<T>
where
T: DeserializeOwned + Validate + Normalizable + Send,
S: Send + Sync,
{
type Rejection = ApiError;
async fn from_request(req: Request, state: &S) -> Result<Self, Self::Rejection> {
let Json(mut data) = Json::<T>::from_request(req, state)
.await
.map_err(|err: JsonRejection| ApiError::json_parse_error(err.body_text()))?;
data.normalize();
data.validate()
.map_err(|validation_errors| invalid_request!("{}", validation_errors))?;
Ok(ValidatedJson(data))
}
}
impl<T> std::ops::Deref for ValidatedJson<T> {
type Target = T;
fn deref(&self) -> &Self::Target {
&self.0
}
}
impl<T> std::ops::DerefMut for ValidatedJson<T> {
fn deref_mut(&mut self) -> &mut Self::Target {
&mut self.0
}
}
+78
View File
@@ -0,0 +1,78 @@
use std::sync::Arc;
use axum::Json;
use axum::extract::{Query, State};
use axum::http::StatusCode;
use serde::{Deserialize, Serialize};
use crate::error::ApiError;
use crate::state::AppState;
use crate::utils::utility_call_error;
#[derive(Serialize)]
pub(crate) struct IsSleepingResponse {
is_sleeping: bool,
}
#[derive(Debug, Deserialize)]
pub(crate) struct SleepParams {
#[serde(default = "default_sleep_level")]
level: u32,
#[serde(default = "default_sleep_mode")]
mode: String,
}
#[derive(Debug, Default, Deserialize)]
pub(crate) struct WakeUpParams {
#[serde(default)]
tags: Option<Vec<String>>,
}
const fn default_sleep_level() -> u32 {
1
}
fn default_sleep_mode() -> String {
"abort".to_string()
}
/// Put the engine to sleep.
pub async fn sleep(
State(state): State<Arc<AppState>>,
Query(params): Query<SleepParams>,
) -> Result<StatusCode, ApiError> {
state
.engine_core_client()
.sleep(params.level, &params.mode)
.await
.map_err(|error| utility_call_error("sleep", error))?;
Ok(StatusCode::OK)
}
/// Wake the engine from sleep mode.
pub async fn wake_up(
State(state): State<Arc<AppState>>,
Query(params): Query<WakeUpParams>,
) -> Result<StatusCode, ApiError> {
state
.engine_core_client()
.wake_up(params.tags)
.await
.map_err(|error| utility_call_error("wake_up", error))?;
Ok(StatusCode::OK)
}
/// Return whether the engine is currently sleeping at any level.
pub async fn is_sleeping(
State(state): State<Arc<AppState>>,
) -> Result<Json<IsSleepingResponse>, ApiError> {
let is_sleeping = state
.engine_core_client()
.is_sleeping()
.await
.map_err(|error| utility_call_error("is_sleeping", error))?;
Ok(Json(IsSleepingResponse { is_sleeping }))
}
File diff suppressed because it is too large Load Diff
+120
View File
@@ -0,0 +1,120 @@
use std::sync::Arc;
use std::sync::atomic::{AtomicU64, Ordering};
use tokio::time::{Duration, Instant, sleep_until};
use tracing::warn;
use vllm_chat::ChatLlm;
use vllm_engine_core_client::EngineCoreClient;
const SHUTDOWN_REFCOUNT_POLL_INTERVAL: Duration = Duration::from_millis(100);
/// Shared router state for the minimal single-model OpenAI server.
pub struct AppState {
/// All public model IDs served by this frontend. The first entry is the
/// primary ID used in responses; all entries are valid in requests.
served_model_names: Vec<String>,
/// Shared chat facade used by all requests.
pub chat: ChatLlm,
/// Whether to log a summary line for each completed request.
pub enable_log_requests: bool,
/// Number of in-flight inference requests currently owned by this frontend.
server_load: AtomicU64,
}
impl AppState {
/// Construct one application state instance.
///
/// `served_model_names` must be non-empty; the first entry is the primary
/// model ID returned in API responses.
///
/// # Panics
///
/// Panics if `served_model_names` is empty.
pub fn new(served_model_names: Vec<String>, chat: ChatLlm) -> Self {
assert!(
!served_model_names.is_empty(),
"served_model_names must not be empty"
);
Self {
served_model_names,
chat,
enable_log_requests: false,
server_load: AtomicU64::new(0),
}
}
/// Enable per-request completion logging.
pub fn with_log_requests(mut self, enabled: bool) -> Self {
self.enable_log_requests = enabled;
self
}
/// The primary model name echoed back in API responses (the first served
/// name).
pub fn primary_model_name(&self) -> &str {
self.served_model_names.first().map(String::as_str).unwrap_or_default()
}
/// All model names served by this frontend.
pub fn served_model_names(&self) -> &[String] {
&self.served_model_names
}
/// Return a reference to the underlying engine core client for utility
/// calls.
pub(crate) fn engine_core_client(&self) -> &EngineCoreClient {
self.chat.engine_core_client()
}
/// Return the current in-flight inference request count for the `/load`
/// endpoint.
pub fn server_load(&self) -> u64 {
self.server_load.load(Ordering::Relaxed)
}
/// Increment the in-flight inference request count, called by the load
/// tracking middleware.
pub(crate) fn increment_server_load(&self) {
self.server_load.fetch_add(1, Ordering::Relaxed);
}
/// Decrement the in-flight inference request count, called by the load
/// tracking middleware.
pub(crate) fn decrement_server_load(&self) {
self.server_load.fetch_sub(1, Ordering::Relaxed);
}
/// Wait until all request-owned references are dropped, then shut down the
/// engine client.
///
/// If the deadline elapses while request/connection tasks still hold state
/// references, skip the clean engine-client shutdown and let process
/// teardown reclaim the remaining resources.
pub async fn shutdown(mut self: Arc<Self>, deadline: Instant) -> anyhow::Result<()> {
loop {
match Arc::try_unwrap(self) {
Ok(state) => {
state.chat.shutdown().await?;
return Ok(());
}
Err(state) => self = state,
}
let ref_count = Arc::strong_count(&self);
let now = Instant::now();
if now >= deadline {
warn!(
ref_count,
"shutdown deadline elapsed before app state became idle; skipping engine-client shutdown"
);
return Ok(());
}
sleep_until(std::cmp::min(
deadline,
now + SHUTDOWN_REFCOUNT_POLL_INTERVAL,
))
.await;
}
}
}
+106
View File
@@ -0,0 +1,106 @@
use std::collections::HashMap;
use std::time::{SystemTime, UNIX_EPOCH};
use axum::http::HeaderMap;
use serde_json::Value;
use thiserror_ext::AsReport;
use uuid::Uuid;
use crate::error::ApiError;
#[derive(Debug, Clone, PartialEq, Eq, Default)]
pub struct ResolvedRequestContext {
pub request_id: String,
pub data_parallel_rank: Option<u32>,
}
/// Return the current Unix timestamp in seconds for OpenAI response objects.
pub fn unix_timestamp() -> u64 {
SystemTime::now()
.duration_since(UNIX_EPOCH)
.map(|duration| duration.as_secs())
.unwrap_or_default()
}
/// Construct an API error for a failed utility call to the engine core.
pub fn utility_call_error(method: &str, error: impl AsReport) -> ApiError {
ApiError::server_error(format!("failed to call {method}: {}", error.as_report()))
}
/// Merge `kv_transfer_params` into the `vllm_xargs` map, mirroring the Python
/// vLLM behavior where `kv_transfer_params` is injected into `extra_args` for
/// engine-core consumption.
pub fn merge_kv_transfer_params(
mut xargs: Option<HashMap<String, Value>>,
kv_transfer_params: Option<&HashMap<String, Value>>,
) -> Option<HashMap<String, Value>> {
if let Some(kv_params) = kv_transfer_params {
let map = xargs.get_or_insert_with(HashMap::new);
map.insert(
"kv_transfer_params".to_string(),
// This is safe because we know that `kv_params` is already valid JSON.
serde_json::to_value(kv_params).unwrap(),
);
}
xargs
}
/// Convert OpenAI-style `logit_bias` with string token-ID keys into the
/// internal `HashMap<u32, f32>` representation, validating that every key
/// parses as a `u32`.
pub fn convert_logit_bias(
logit_bias: Option<HashMap<String, f32>>,
) -> Result<Option<HashMap<u32, f32>>, ApiError> {
logit_bias
.map(|bias| {
bias.into_iter()
.map(|(key, value)| {
key.parse().map(|k| (k, value)).map_err(|_| {
ApiError::invalid_request(
format!(
"Invalid key in 'logit_bias': '{key}' is not a valid token ID. \
Token IDs must be non-negative integers."
),
Some("logit_bias"),
)
})
})
.collect()
})
.transpose()
}
/// Extract common request metadata from HTTP headers: the external request ID
/// and the optional data-parallel rank used for engine routing.
pub fn resolve_request_context(
headers: &HeaderMap,
request_id: Option<&str>,
) -> ResolvedRequestContext {
// `None` when the header is absent or cannot be parsed as a `u32`.
let data_parallel_rank = headers
.get("X-data-parallel-rank")
.and_then(|v| v.to_str().ok())
.and_then(|s| s.trim().parse().ok());
// Extract request id from header.
let request_id_header = headers.get("X-Request-Id").and_then(|value| value.to_str().ok());
let request_id = resolve_base_request_id(request_id_header, request_id);
ResolvedRequestContext {
request_id,
data_parallel_rank,
}
}
/// Resolve the base external request ID before API-specific prefixes such as
/// `chatcmpl-`.
pub fn resolve_base_request_id(
request_id_header: Option<&str>,
request_id: Option<&str>,
) -> String {
request_id_header.or(request_id).map(ToOwned::to_owned).unwrap_or_else(|| {
let mut id = Uuid::new_v4().simple().to_string();
id.truncate(8);
id
})
}