[Rust Frontend] Move code from vllm-frontend-rs (#43283)
Signed-off-by: Bugen Zhao <i@bugenzhao.com> Signed-off-by: Nick Hill <nickhill123@gmail.com> Signed-off-by: Eric Curtin <eric.curtin@docker.com> Signed-off-by: Dev-X25874 <283057883+Dev-X25874@users.noreply.github.com> Signed-off-by: Will.hou <1205157517@qq.com> Signed-off-by: Will.hou <willamhou@ceresman.com> Co-authored-by: Nick Hill <nickhill123@gmail.com> Co-authored-by: Eric Curtin <eric.curtin@docker.com> Co-authored-by: Dev-X25874 <283057883+Dev-X25874@users.noreply.github.com> Co-authored-by: Will.hou <1205157517@qq.com> Co-authored-by: Will.hou <willamhou@ceresman.com> Please see https://github.com/Inferact/vllm-frontend-rs for full original commit history.
This commit is contained in:
co-authored by
Nick Hill
Eric Curtin
Dev-X25874
Will.hou
Will.hou
parent
39d5fa96a7
commit
39910f2b25
@@ -0,0 +1,57 @@
|
||||
[package]
|
||||
name = "vllm-server"
|
||||
version.workspace = true
|
||||
edition.workspace = true
|
||||
license.workspace = true
|
||||
|
||||
[dependencies]
|
||||
anyhow.workspace = true
|
||||
asynk-strim-attr.workspace = true
|
||||
axum.workspace = true
|
||||
futures.workspace = true
|
||||
http-body.workspace = true
|
||||
itertools.workspace = true
|
||||
libc.workspace = true
|
||||
llm-multimodal.workspace = true
|
||||
prost.workspace = true
|
||||
prost-types.workspace = true
|
||||
rmpv.workspace = true
|
||||
serde.workspace = true
|
||||
serde_json.workspace = true
|
||||
serde_with.workspace = true
|
||||
socket2.workspace = true
|
||||
thiserror-ext.workspace = true
|
||||
tokio.workspace = true
|
||||
tokio-stream.workspace = true
|
||||
tokio-util.workspace = true
|
||||
tonic.workspace = true
|
||||
tonic-prost.workspace = true
|
||||
tower-http.workspace = true
|
||||
tracing.workspace = true
|
||||
tracing-futures.workspace = true
|
||||
tracing-subscriber.workspace = true
|
||||
uuid.workspace = true
|
||||
validator.workspace = true
|
||||
vllm-chat.workspace = true
|
||||
vllm-engine-core-client.workspace = true
|
||||
vllm-llm.workspace = true
|
||||
vllm-metrics.workspace = true
|
||||
vllm-text.workspace = true
|
||||
|
||||
[build-dependencies]
|
||||
tonic-prost-build.workspace = true
|
||||
|
||||
[dev-dependencies]
|
||||
anyhow.workspace = true
|
||||
async-openai = { workspace = true, features = ["full"] }
|
||||
bytes.workspace = true
|
||||
clap.workspace = true
|
||||
expect-test.workspace = true
|
||||
rmp-serde.workspace = true
|
||||
serial_test.workspace = true
|
||||
tower.workspace = true
|
||||
vllm-engine-core-client = { workspace = true, features = ["test-util"] }
|
||||
zeromq.workspace = true
|
||||
|
||||
[lints]
|
||||
workspace = true
|
||||
@@ -0,0 +1,12 @@
|
||||
fn main() -> Result<(), Box<dyn std::error::Error>> {
|
||||
let manifest_dir = std::env::var("CARGO_MANIFEST_DIR").unwrap();
|
||||
let proto_dir = format!("{manifest_dir}/../../proto");
|
||||
|
||||
tonic_prost_build::configure()
|
||||
.build_server(true)
|
||||
.build_client(true)
|
||||
.protoc_arg("--experimental_allow_proto3_optional") // be compatible with old compilers
|
||||
.compile_protos(&[format!("{proto_dir}/vllm_grpc.proto")], &[proto_dir])?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
@@ -0,0 +1,39 @@
|
||||
# Server Smoke Test
|
||||
|
||||
Start a fresh headless `vllm` engine:
|
||||
|
||||
```bash
|
||||
source ../vllm/.venv/bin/activate
|
||||
HF_HUB_OFFLINE=1 \
|
||||
VLLM_LOGGING_LEVEL=DEBUG \
|
||||
VLLM_CPU_KVCACHE_SPACE=2 \
|
||||
VLLM_HOST_IP=127.0.0.1 \
|
||||
VLLM_LOOPBACK_IP=127.0.0.1 \
|
||||
python3 -m vllm.entrypoints.cli.main serve Qwen/Qwen3-0.6B \
|
||||
--headless \
|
||||
--data-parallel-address 127.0.0.1 \
|
||||
--data-parallel-rpc-port 62100 \
|
||||
--data-parallel-size-local 1 \
|
||||
--max-model-len 512 \
|
||||
--dtype float16
|
||||
```
|
||||
|
||||
Run the Rust server smoke test:
|
||||
|
||||
```bash
|
||||
cargo run -p vllm-server --example external_engine_openai_qwen -- \
|
||||
--handshake-address tcp://127.0.0.1:62100
|
||||
```
|
||||
|
||||
The example starts the Rust OpenAI-compatible server on an ephemeral local port,
|
||||
connects to it via the `async-openai` Rust client, lists models, and then checks
|
||||
that a streamed chat completion yields the assistant role chunk, final-answer
|
||||
content chunks, and a terminal finish chunk. This example intentionally uses
|
||||
`async-openai`'s standard typed `create_stream` API instead of BYOT, so it does
|
||||
not inspect the nonstandard `reasoning_content` field even though the Rust
|
||||
server may emit it for reasoning-capable models such as Qwen3. For reasoning
|
||||
behavior itself, use the `vllm-chat` smoke test or the `vllm-server`
|
||||
route tests.
|
||||
|
||||
IMPORTANT: Restart `vllm` each time you run the smoke test. The current headless
|
||||
engine cannot safely handle frontend reconnects after the client shuts down.
|
||||
@@ -0,0 +1,215 @@
|
||||
use std::time::Duration;
|
||||
|
||||
use anyhow::{Context, Result, bail};
|
||||
use async_openai::Client;
|
||||
use async_openai::config::OpenAIConfig;
|
||||
use async_openai::types::chat::{
|
||||
ChatCompletionRequestUserMessageArgs, CreateChatCompletionRequest,
|
||||
CreateChatCompletionRequestArgs,
|
||||
};
|
||||
use async_openai::types::models::ListModelResponse;
|
||||
use clap::Parser;
|
||||
use futures::StreamExt as _;
|
||||
use tokio_util::sync::CancellationToken;
|
||||
use tracing_subscriber::EnvFilter;
|
||||
use vllm_engine_core_client::TransportMode;
|
||||
use vllm_server::{
|
||||
ChatTemplateContentFormatOption, Config, CoordinatorMode, HttpListenerMode, ParserSelection,
|
||||
RendererSelection, serve,
|
||||
};
|
||||
|
||||
#[derive(Debug, Parser)]
|
||||
#[command(
|
||||
about = "Smoke-test the Rust OpenAI server with async-openai against an external Qwen vLLM engine."
|
||||
)]
|
||||
struct Args {
|
||||
#[arg(long)]
|
||||
handshake_address: String,
|
||||
#[arg(long, default_value_t = 1)]
|
||||
engine_count: usize,
|
||||
#[arg(long, default_value = "Qwen/Qwen3-0.6B")]
|
||||
model: String,
|
||||
#[arg(long, default_value = "127.0.0.1")]
|
||||
host: String,
|
||||
#[arg(long, default_value_t = 30)]
|
||||
ready_timeout_secs: u64,
|
||||
#[arg(
|
||||
long,
|
||||
default_value = "What is the capital of France? Answer with one word."
|
||||
)]
|
||||
prompt: String,
|
||||
}
|
||||
|
||||
#[tokio::main(flavor = "multi_thread")]
|
||||
async fn main() -> Result<()> {
|
||||
init_tracing();
|
||||
let args = Args::parse();
|
||||
let port = unique_local_port()?;
|
||||
let config = Config {
|
||||
transport_mode: TransportMode::HandshakeOwner {
|
||||
handshake_address: args.handshake_address,
|
||||
advertised_host: args.host,
|
||||
engine_count: args.engine_count,
|
||||
ready_timeout: Duration::from_secs(args.ready_timeout_secs),
|
||||
local_input_address: None,
|
||||
local_output_address: None,
|
||||
},
|
||||
coordinator_mode: CoordinatorMode::MaybeInProc,
|
||||
model: args.model,
|
||||
served_model_name: vec![],
|
||||
listener_mode: HttpListenerMode::BindTcp {
|
||||
host: "127.0.0.1".to_string(),
|
||||
port,
|
||||
},
|
||||
tool_call_parser: ParserSelection::Auto,
|
||||
reasoning_parser: ParserSelection::Auto,
|
||||
renderer: RendererSelection::Auto,
|
||||
chat_template: None,
|
||||
default_chat_template_kwargs: None,
|
||||
chat_template_content_format: ChatTemplateContentFormatOption::Auto,
|
||||
enable_log_requests: false,
|
||||
disable_log_stats: false,
|
||||
grpc_port: None,
|
||||
shutdown_timeout: Duration::ZERO,
|
||||
};
|
||||
|
||||
let bind_address = format!("127.0.0.1:{port}");
|
||||
let shutdown = CancellationToken::new();
|
||||
let server_config = config.clone();
|
||||
let server_shutdown = shutdown.clone();
|
||||
let server_task = tokio::spawn(async move { serve(server_config, server_shutdown).await });
|
||||
|
||||
let client = Client::with_config(
|
||||
OpenAIConfig::new()
|
||||
.with_api_key("unused")
|
||||
.with_api_base(format!("http://{bind_address}/v1")),
|
||||
);
|
||||
|
||||
print_models(&client).await?;
|
||||
let final_text = stream_completion(&client, &config.model, &args.prompt).await?;
|
||||
|
||||
println!();
|
||||
println!("final_text={final_text:?}");
|
||||
|
||||
shutdown_server(server_task, shutdown).await
|
||||
}
|
||||
|
||||
fn init_tracing() {
|
||||
let filter = EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new("info"));
|
||||
let _ = tracing_subscriber::fmt().with_env_filter(filter).try_init();
|
||||
}
|
||||
|
||||
fn unique_local_port() -> Result<u16> {
|
||||
let listener = std::net::TcpListener::bind("127.0.0.1:0")
|
||||
.context("failed to allocate local smoke-test port")?;
|
||||
let port = listener.local_addr().context("failed to read local smoke-test port")?.port();
|
||||
drop(listener);
|
||||
Ok(port)
|
||||
}
|
||||
|
||||
async fn print_models(client: &Client<OpenAIConfig>) -> Result<()> {
|
||||
let models = wait_for_models(client).await?;
|
||||
let model_ids = models.data.into_iter().map(|model| model.id).collect::<Vec<_>>();
|
||||
println!("models={model_ids:?}");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn wait_for_models(client: &Client<OpenAIConfig>) -> Result<ListModelResponse> {
|
||||
let mut last_error = None;
|
||||
for _ in 0..240 {
|
||||
match client.models().list().await {
|
||||
Ok(models) => return Ok(models),
|
||||
Err(error) => {
|
||||
last_error = Some(error);
|
||||
tokio::time::sleep(Duration::from_millis(500)).await;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
match last_error {
|
||||
Some(error) => Err(error).context("OpenAI server did not become ready in time"),
|
||||
None => bail!("OpenAI server readiness loop finished without a result"),
|
||||
}
|
||||
}
|
||||
|
||||
async fn stream_completion(
|
||||
client: &Client<OpenAIConfig>,
|
||||
model: &str,
|
||||
prompt: &str,
|
||||
) -> Result<String> {
|
||||
// Keep this smoke test on async-openai's standard `create_stream` path so it
|
||||
// exercises the ordinary typed chat-completions client without BYOT
|
||||
// request/response types.
|
||||
//
|
||||
// The current async-openai chat-completions stream delta type does not expose
|
||||
// our OpenAI-compatible `reasoning_content` extension field, so this
|
||||
// example only validates the assistant role chunk, visible `content`
|
||||
// deltas, and terminal finish chunk. Reasoning coverage lives in our own
|
||||
// route tests and in the `vllm-chat` smoke example.
|
||||
let request: CreateChatCompletionRequest = CreateChatCompletionRequestArgs::default()
|
||||
.model(model)
|
||||
.stream(true)
|
||||
.temperature(0.0)
|
||||
.max_completion_tokens(128u32)
|
||||
.messages([ChatCompletionRequestUserMessageArgs::default()
|
||||
.content(prompt)
|
||||
.build()
|
||||
.context("failed to build user chat message")?
|
||||
.into()])
|
||||
.build()
|
||||
.context("failed to build chat completion request")?;
|
||||
|
||||
let mut stream = client
|
||||
.chat()
|
||||
.create_stream(request)
|
||||
.await
|
||||
.context("failed to create streaming chat completion")?;
|
||||
|
||||
let mut final_text = String::new();
|
||||
let mut saw_role = false;
|
||||
let mut saw_finish_reason = false;
|
||||
let mut saw_text = false;
|
||||
|
||||
while let Some(chunk) = stream.next().await {
|
||||
let chunk = chunk.context("streaming chat completion failed")?;
|
||||
for choice in chunk.choices {
|
||||
if choice.delta.role.is_some() {
|
||||
saw_role = true;
|
||||
}
|
||||
if let Some(delta) = choice.delta.content {
|
||||
if !saw_text {
|
||||
print!("[answer] ");
|
||||
}
|
||||
print!("{delta}");
|
||||
final_text.push_str(&delta);
|
||||
saw_text = true;
|
||||
}
|
||||
if choice.finish_reason.is_some() {
|
||||
saw_finish_reason = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if !saw_role {
|
||||
bail!("stream ended without an assistant role chunk");
|
||||
}
|
||||
if !saw_finish_reason {
|
||||
bail!("stream ended without a terminal finish reason");
|
||||
}
|
||||
if final_text.is_empty() {
|
||||
bail!("stream ended without any content deltas");
|
||||
}
|
||||
|
||||
Ok(final_text)
|
||||
}
|
||||
|
||||
async fn shutdown_server(
|
||||
server_task: tokio::task::JoinHandle<anyhow::Result<()>>,
|
||||
shutdown: CancellationToken,
|
||||
) -> Result<()> {
|
||||
shutdown.cancel();
|
||||
server_task
|
||||
.await
|
||||
.context("server task join failed")?
|
||||
.context("server task failed")
|
||||
}
|
||||
@@ -0,0 +1,110 @@
|
||||
use std::collections::HashMap;
|
||||
use std::time::Duration;
|
||||
|
||||
use anyhow::Result;
|
||||
use serde_json::Value;
|
||||
use vllm_chat::{ChatTemplateContentFormatOption, ParserSelection, RendererSelection};
|
||||
use vllm_engine_core_client::{CoordinatorMode as EngineCoreCoordinatorMode, TransportMode};
|
||||
|
||||
/// How the HTTP server obtains its listening socket.
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub enum HttpListenerMode {
|
||||
/// Bind a fresh TCP listener on the given host/port.
|
||||
BindTcp { host: String, port: u16 },
|
||||
/// Bind a fresh Unix domain listener on the given filesystem path.
|
||||
BindUnix { path: String },
|
||||
/// Adopt an already-open listening socket inherited from a supervisor
|
||||
/// process.
|
||||
InheritedFd { fd: i32 },
|
||||
}
|
||||
|
||||
/// Which coordinator implementation should be active when one is present for a
|
||||
/// frontend client.
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub enum CoordinatorMode {
|
||||
/// Do not run a coordinator at all.
|
||||
None,
|
||||
/// Run the Rust in-process coordinator for managed `serve` deployments, if
|
||||
/// there are multiple engines and the model is MoE.
|
||||
MaybeInProc,
|
||||
/// Connect to an external coordinator owned by another process.
|
||||
External { address: String },
|
||||
}
|
||||
|
||||
/// Normalized runtime configuration for the minimal OpenAI-compatible server.
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct Config {
|
||||
/// Frontend-to-engine transport setup.
|
||||
pub transport_mode: TransportMode,
|
||||
/// Requested frontend-side coordinator behavior.
|
||||
pub coordinator_mode: CoordinatorMode,
|
||||
/// Backend model identifier used for engine-core loading.
|
||||
pub model: String,
|
||||
/// Model name(s) exposed to clients via the OpenAI API. When non-empty,
|
||||
/// the first entry is used as the primary ID in responses and all entries
|
||||
/// are accepted in requests. When empty, falls back to `model`.
|
||||
pub served_model_name: Vec<String>,
|
||||
/// HTTP listener setup.
|
||||
pub listener_mode: HttpListenerMode,
|
||||
/// Tool-call parser selection.
|
||||
pub tool_call_parser: ParserSelection,
|
||||
/// Reasoning parser selection.
|
||||
pub reasoning_parser: ParserSelection,
|
||||
/// Chat renderer selection.
|
||||
pub renderer: RendererSelection,
|
||||
/// Server-default chat template override, as a file path or inline
|
||||
/// template.
|
||||
pub chat_template: Option<String>,
|
||||
/// Server-default keyword arguments merged into every chat-template render.
|
||||
pub default_chat_template_kwargs: Option<HashMap<String, Value>>,
|
||||
/// How to serialize `message.content` for chat-template rendering.
|
||||
pub chat_template_content_format: ChatTemplateContentFormatOption,
|
||||
/// Log a summary line for each completed request.
|
||||
pub enable_log_requests: bool,
|
||||
/// When `true`, suppress periodic stats logging (throughput, queue depth,
|
||||
/// cache usage).
|
||||
pub disable_log_stats: bool,
|
||||
/// TCP port for the gRPC Generate service. When `None`, no gRPC server is
|
||||
/// started.
|
||||
pub grpc_port: Option<u16>,
|
||||
/// Maximum time to wait for active HTTP/gRPC requests to drain on shutdown.
|
||||
pub shutdown_timeout: Duration,
|
||||
}
|
||||
|
||||
impl Config {
|
||||
/// Validate frontend configuration that can be checked before engine
|
||||
/// startup.
|
||||
pub fn validate(&self) -> Result<()> {
|
||||
vllm_chat::validate_parser_overrides(&self.tool_call_parser, &self.reasoning_parser)?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Return the number of engines implied by the configured transport mode.
|
||||
pub fn engine_count(&self) -> usize {
|
||||
match &self.transport_mode {
|
||||
TransportMode::HandshakeOwner { engine_count, .. }
|
||||
| TransportMode::Bootstrapped { engine_count, .. } => *engine_count,
|
||||
}
|
||||
}
|
||||
|
||||
/// Resolve the effective coordinator mode.
|
||||
pub fn effective_coordinator_mode(
|
||||
&self,
|
||||
model_is_moe: bool,
|
||||
) -> Option<EngineCoreCoordinatorMode> {
|
||||
match &self.coordinator_mode {
|
||||
CoordinatorMode::None => None,
|
||||
CoordinatorMode::MaybeInProc => {
|
||||
if model_is_moe && self.engine_count() > 1 {
|
||||
Some(EngineCoreCoordinatorMode::InProc)
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
CoordinatorMode::External { address } => Some(EngineCoreCoordinatorMode::External {
|
||||
address: address.clone(),
|
||||
}),
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,74 @@
|
||||
use axum::Json;
|
||||
use axum::http::StatusCode;
|
||||
use axum::response::{IntoResponse, Response};
|
||||
use thiserror_ext::{Construct, Macro};
|
||||
|
||||
use crate::routes::openai::utils::types::{ErrorDetail, ErrorResponse};
|
||||
|
||||
/// Small OpenAI-style error family used by the minimal HTTP layer.
|
||||
#[derive(Debug, Construct, Macro)]
|
||||
pub enum ApiError {
|
||||
/// The request is syntactically valid OpenAI JSON but asks for unsupported
|
||||
/// behavior.
|
||||
InvalidRequest {
|
||||
message: String,
|
||||
param: Option<&'static str>,
|
||||
},
|
||||
/// The requested model name does not match the single configured model.
|
||||
ModelNotFound { model: String },
|
||||
/// The request body could not be parsed as valid JSON.
|
||||
JsonParseError { message: String },
|
||||
/// An unexpected internal failure happened before streaming started.
|
||||
ServerError { message: String },
|
||||
}
|
||||
|
||||
impl ApiError {
|
||||
/// Return the HTTP status code associated with this API error.
|
||||
pub fn status_code(&self) -> StatusCode {
|
||||
match self {
|
||||
Self::InvalidRequest { .. } => StatusCode::BAD_REQUEST,
|
||||
Self::ModelNotFound { .. } => StatusCode::NOT_FOUND,
|
||||
Self::ServerError { .. } => StatusCode::INTERNAL_SERVER_ERROR,
|
||||
Self::JsonParseError { .. } => StatusCode::BAD_REQUEST,
|
||||
}
|
||||
}
|
||||
|
||||
/// Convert this error into the standard OpenAI-compatible JSON error
|
||||
/// payload.
|
||||
pub fn to_error_response(&self) -> ErrorResponse {
|
||||
let error = match self {
|
||||
Self::InvalidRequest { message, param } => ErrorDetail {
|
||||
message: message.clone(),
|
||||
error_type: "invalid_request_error".to_string(),
|
||||
param: param.map(|p| p.to_string()),
|
||||
code: Some("invalid_request_error".to_string()),
|
||||
},
|
||||
Self::ModelNotFound { model } => ErrorDetail {
|
||||
message: format!("The model `{model}` does not exist."),
|
||||
error_type: "invalid_request_error".to_string(),
|
||||
param: Some("model".to_string()),
|
||||
code: Some("model_not_found".to_string()),
|
||||
},
|
||||
Self::ServerError { message } => ErrorDetail {
|
||||
message: message.clone(),
|
||||
error_type: "server_error".to_string(),
|
||||
param: None,
|
||||
code: Some("server_error".to_string()),
|
||||
},
|
||||
Self::JsonParseError { message } => ErrorDetail {
|
||||
message: message.clone(),
|
||||
error_type: "invalid_request_error".to_string(),
|
||||
param: None,
|
||||
code: Some("json_parse_error".to_string()),
|
||||
},
|
||||
};
|
||||
|
||||
ErrorResponse { error }
|
||||
}
|
||||
}
|
||||
|
||||
impl IntoResponse for ApiError {
|
||||
fn into_response(self) -> Response {
|
||||
(self.status_code(), Json(self.to_error_response())).into_response()
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,679 @@
|
||||
//! Conversion between gRPC protobuf types and internal `vllm-text`
|
||||
//! request/response types.
|
||||
|
||||
use tonic::Status;
|
||||
use uuid::Uuid;
|
||||
use vllm_engine_core_client::protocol::{StopReason, StructuredOutputsParams};
|
||||
use vllm_text::{
|
||||
DecodedLogprobs, DecodedPromptLogprobs, FinishReason, Finished, Prompt, SamplingParams,
|
||||
TextDecodeOptions, TextRequest,
|
||||
};
|
||||
|
||||
use super::pb;
|
||||
|
||||
// ========================================================================================
|
||||
// Request conversion
|
||||
// ========================================================================================
|
||||
|
||||
/// Convert a gRPC `GenerateRequest` into the internal `TextRequest`.
|
||||
///
|
||||
/// If `req.model` is non-empty, it must match one of `served_model_names`;
|
||||
/// otherwise the request is rejected with `NotFound`. An empty string is
|
||||
/// treated as "unset" (proto3 default) and accepted.
|
||||
pub fn to_text_request(
|
||||
req: pb::GenerateRequest,
|
||||
stream: bool,
|
||||
served_model_names: &[String],
|
||||
) -> Result<TextRequest, Status> {
|
||||
if !req.model.is_empty() && !served_model_names.iter().any(|n| n == &req.model) {
|
||||
return Err(Status::not_found(format!(
|
||||
"model `{}` not found",
|
||||
req.model
|
||||
)));
|
||||
}
|
||||
|
||||
if req.truncate_prompt_tokens != 0 {
|
||||
return Err(Status::invalid_argument(
|
||||
"truncate_prompt_tokens is not supported",
|
||||
));
|
||||
}
|
||||
|
||||
let prompt = match req.prompt {
|
||||
Some(pb::generate_request::Prompt::Text(text)) => Prompt::Text(text),
|
||||
Some(pb::generate_request::Prompt::TokenIds(ids)) => Prompt::TokenIds(ids.ids),
|
||||
None => return Err(Status::invalid_argument("prompt is required")),
|
||||
};
|
||||
|
||||
let request_id = if req.request_id.is_empty() {
|
||||
Uuid::new_v4().to_string()
|
||||
} else {
|
||||
req.request_id
|
||||
};
|
||||
|
||||
let sampling = req.sampling.as_ref();
|
||||
let decoding = req.decoding.as_ref();
|
||||
let stopping = req.stopping.as_ref();
|
||||
let response = req.response.as_ref();
|
||||
let kv = req.kv.as_ref();
|
||||
|
||||
let mut sampling_params =
|
||||
build_sampling_params(req.temperature, sampling, decoding, stopping, response)?;
|
||||
|
||||
// Thread KVCacheParameters → SamplingParams fields.
|
||||
if let Some(kv) = kv {
|
||||
// Thread kv_transfer_params through vllm_xargs, matching the HTTP route
|
||||
// convention.
|
||||
if let Some(kv_struct) = kv.kv_transfer_params.as_ref() {
|
||||
let kv_json = proto_struct_to_json(kv_struct);
|
||||
let map = sampling_params.vllm_xargs.get_or_insert_with(Default::default);
|
||||
map.insert("kv_transfer_params".to_string(), kv_json);
|
||||
}
|
||||
if kv.bypass_prefix_cache {
|
||||
sampling_params.skip_reading_prefix_cache = Some(true);
|
||||
}
|
||||
}
|
||||
|
||||
let decode_options = TextDecodeOptions {
|
||||
skip_special_tokens: true,
|
||||
include_stop_str_in_output: stopping.is_some_and(|s| s.include_stop_strings),
|
||||
stop_strings: stopping.map(|s| &s.stop_strings).filter(|ss| !ss.is_empty()).cloned(),
|
||||
min_tokens: stopping.map_or(0, |s| s.min_new_tokens),
|
||||
};
|
||||
|
||||
Ok(TextRequest {
|
||||
request_id,
|
||||
prompt,
|
||||
mm_features: None,
|
||||
sampling_params,
|
||||
decode_options,
|
||||
intermediate: stream,
|
||||
priority: req.priority,
|
||||
cache_salt: kv.map(|k| &k.cache_salt).filter(|s| !s.is_empty()).cloned(),
|
||||
add_special_tokens: true,
|
||||
data_parallel_rank: None,
|
||||
})
|
||||
}
|
||||
|
||||
fn build_sampling_params(
|
||||
temperature: Option<f32>,
|
||||
sampling: Option<&pb::RandomSampling>,
|
||||
decoding: Option<&pb::DecodingParameters>,
|
||||
stopping: Option<&pb::StoppingCriteria>,
|
||||
response: Option<&pb::ResponseOptions>,
|
||||
) -> Result<SamplingParams, Status> {
|
||||
// Temperature is a top-level GenerateRequest field. Default to greedy (0.0) for
|
||||
// the gRPC API when the caller does not specify a value. This differs from
|
||||
// the HTTP/OpenAI API (which defaults to 1.0) and matches the convention of
|
||||
// programmatic generation APIs.
|
||||
let temperature = temperature.or(Some(0.0));
|
||||
let mut params = SamplingParams {
|
||||
temperature,
|
||||
..SamplingParams::default()
|
||||
};
|
||||
|
||||
// RandomSampling: for every remaining sampling field the protobuf default (`0`)
|
||||
// is treated as "unset" and leaves the resolved value to the lowering
|
||||
// stage, which falls back to the model-provided default or a
|
||||
// neutral/disabled value otherwise.
|
||||
if let Some(s) = sampling {
|
||||
// num_sequences (n > 1) is not supported yet by the TextLlm layer; the response
|
||||
// path also hardcodes SequenceOutput.index = 0, so accepting >1 would silently
|
||||
// truncate output cardinality. Reject explicitly.
|
||||
if s.num_sequences > 1 {
|
||||
return Err(Status::invalid_argument(
|
||||
"num_sequences > 1 is not supported",
|
||||
));
|
||||
}
|
||||
if s.top_k != 0 {
|
||||
params.top_k = Some(s.top_k);
|
||||
}
|
||||
if s.top_p != 0.0 {
|
||||
params.top_p = Some(s.top_p);
|
||||
}
|
||||
if s.min_p != 0.0 {
|
||||
params.min_p = Some(s.min_p);
|
||||
}
|
||||
params.seed = s.seed;
|
||||
}
|
||||
|
||||
// DecodingParameters
|
||||
if let Some(d) = decoding {
|
||||
if d.presence_penalty != 0.0 {
|
||||
params.presence_penalty = Some(d.presence_penalty);
|
||||
}
|
||||
if d.frequency_penalty != 0.0 {
|
||||
params.frequency_penalty = Some(d.frequency_penalty);
|
||||
}
|
||||
if d.repetition_penalty != 0.0 {
|
||||
params.repetition_penalty = Some(d.repetition_penalty);
|
||||
}
|
||||
if !d.logit_bias.is_empty() {
|
||||
params.logit_bias = Some(d.logit_bias.clone());
|
||||
}
|
||||
if !d.allowed_token_ids.is_empty() {
|
||||
params.allowed_token_ids = Some(d.allowed_token_ids.clone());
|
||||
}
|
||||
params.structured_outputs = convert_structured_output(d)?;
|
||||
}
|
||||
|
||||
// StoppingCriteria
|
||||
if let Some(s) = stopping {
|
||||
if s.max_new_tokens != 0 {
|
||||
params.max_tokens = Some(s.max_new_tokens);
|
||||
}
|
||||
if s.min_new_tokens != 0 {
|
||||
params.min_tokens = Some(s.min_new_tokens);
|
||||
}
|
||||
if !s.stop_token_ids.is_empty() {
|
||||
params.stop_token_ids = Some(s.stop_token_ids.clone());
|
||||
}
|
||||
params.ignore_eos = s.ignore_eos;
|
||||
}
|
||||
|
||||
// ResponseOptions → logprobs
|
||||
if let Some(r) = response {
|
||||
if r.output_logprobs {
|
||||
let (count, token_ids) = candidate_logprob_spec(r.output_candidates.as_ref());
|
||||
params.logprobs = Some(count);
|
||||
params.logprob_token_ids = token_ids;
|
||||
}
|
||||
if r.prompt_logprobs {
|
||||
// The engine-core protocol has only one shared `logprob_token_ids` field
|
||||
// for output and prompt logprobs, so a per-token-id selector for prompt
|
||||
// candidates can't be honored independently. Reject it instead of silently
|
||||
// dropping the list.
|
||||
if matches!(
|
||||
r.prompt_candidates.as_ref().and_then(|c| c.select.as_ref()),
|
||||
Some(pb::candidate_tokens::Select::TokenIds(_))
|
||||
) {
|
||||
return Err(Status::invalid_argument(
|
||||
"prompt_candidates token_ids selector is not supported",
|
||||
));
|
||||
}
|
||||
let (count, _) = candidate_logprob_spec(r.prompt_candidates.as_ref());
|
||||
params.prompt_logprobs = Some(count);
|
||||
}
|
||||
}
|
||||
|
||||
Ok(params)
|
||||
}
|
||||
|
||||
/// Map the proto `CandidateTokens` selector to a `(logprobs_count,
|
||||
/// logprob_token_ids)` pair.
|
||||
///
|
||||
/// - `top_n(k)` → `(k, None)` — return top-k candidates by probability
|
||||
/// - `all` → `(-1, None)` — return the full vocabulary
|
||||
/// - `token_ids(n)` → `(1, Some(vec of n token ids))` — return logprobs for specific tokens (the
|
||||
/// count `n` is stored in the proto as the number of token IDs that follow, but the actual IDs
|
||||
/// are carried via `logprob_token_ids` on `SamplingParams`)
|
||||
/// - absent → `(1, None)` — just the sampled/scored token
|
||||
fn candidate_logprob_spec(candidates: Option<&pb::CandidateTokens>) -> (i32, Option<Vec<u32>>) {
|
||||
match candidates.and_then(|c| c.select.as_ref()) {
|
||||
Some(pb::candidate_tokens::Select::TopN(n)) => (*n as i32, None),
|
||||
Some(pb::candidate_tokens::Select::All(true)) => (-1, None),
|
||||
Some(pb::candidate_tokens::Select::TokenIds(ids)) => (1, Some(ids.ids.clone())),
|
||||
_ => (1, None),
|
||||
}
|
||||
}
|
||||
|
||||
fn convert_structured_output(
|
||||
d: &pb::DecodingParameters,
|
||||
) -> Result<Option<StructuredOutputsParams>, Status> {
|
||||
let so = match d.structured_output.as_ref() {
|
||||
None => return Ok(None),
|
||||
Some(so) => so,
|
||||
};
|
||||
use pb::decoding_parameters::StructuredOutput;
|
||||
let params = match so {
|
||||
StructuredOutput::Json(schema) => {
|
||||
let json: serde_json::Value = serde_json::from_str(schema)
|
||||
.map_err(|e| Status::invalid_argument(format!("invalid json schema: {e}")))?;
|
||||
StructuredOutputsParams {
|
||||
json: Some(json),
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
StructuredOutput::Regex(regex) => StructuredOutputsParams {
|
||||
regex: Some(regex.clone()),
|
||||
..Default::default()
|
||||
},
|
||||
StructuredOutput::Choice(choices) => StructuredOutputsParams {
|
||||
choice: Some(choices.choices.clone()),
|
||||
..Default::default()
|
||||
},
|
||||
StructuredOutput::Grammar(grammar) => StructuredOutputsParams {
|
||||
grammar: Some(grammar.clone()),
|
||||
..Default::default()
|
||||
},
|
||||
StructuredOutput::JsonObject(true) => StructuredOutputsParams {
|
||||
json_object: Some(true),
|
||||
..Default::default()
|
||||
},
|
||||
StructuredOutput::JsonObject(false) => return Ok(None),
|
||||
StructuredOutput::StructuralTag(tag) => StructuredOutputsParams {
|
||||
structural_tag: Some(tag.clone()),
|
||||
..Default::default()
|
||||
},
|
||||
};
|
||||
Ok(Some(params))
|
||||
}
|
||||
|
||||
// ========================================================================================
|
||||
// Response conversion
|
||||
// ========================================================================================
|
||||
|
||||
/// Convert a `DecodedTextEvent::Start` into the prompt info portion of a gRPC
|
||||
/// response.
|
||||
pub fn to_prompt_info(
|
||||
prompt_token_ids: &[u32],
|
||||
prompt_logprobs: Option<&DecodedPromptLogprobs>,
|
||||
opts: &ResponseOpts,
|
||||
) -> pb::PromptInfo {
|
||||
let token_ids = if opts.prompt_token_ids {
|
||||
prompt_token_ids.to_vec()
|
||||
} else {
|
||||
vec![]
|
||||
};
|
||||
|
||||
let (logprobs, ranks, candidate_tokens) = match prompt_logprobs {
|
||||
Some(plp) if opts.prompt_logprobs => prompt_logprobs_to_proto(plp),
|
||||
_ => (vec![], vec![], vec![]),
|
||||
};
|
||||
|
||||
pb::PromptInfo {
|
||||
num_prompt_tokens: prompt_token_ids.len() as u32,
|
||||
token_ids,
|
||||
logprobs,
|
||||
ranks,
|
||||
candidate_tokens,
|
||||
}
|
||||
}
|
||||
|
||||
/// Convert a `DecodedTextEvent::TextDelta` into a gRPC `SequenceOutput`.
|
||||
pub fn to_sequence_output(
|
||||
delta: &str,
|
||||
token_ids: &[u32],
|
||||
logprobs: Option<&DecodedLogprobs>,
|
||||
finished: Option<&Finished>,
|
||||
opts: &ResponseOpts,
|
||||
) -> pb::SequenceOutput {
|
||||
let (lp_values, rank_values, candidates) = match logprobs {
|
||||
Some(lp) if opts.output_logprobs => output_logprobs_to_proto(lp),
|
||||
_ => (vec![], vec![], vec![]),
|
||||
};
|
||||
|
||||
pb::SequenceOutput {
|
||||
index: 0, // TODO: multi-sequence (n > 1) not supported
|
||||
text: if opts.output_text {
|
||||
delta.to_string()
|
||||
} else {
|
||||
String::new()
|
||||
},
|
||||
num_tokens: token_ids.len() as u32,
|
||||
token_ids: if opts.output_token_ids {
|
||||
token_ids.to_vec()
|
||||
} else {
|
||||
vec![]
|
||||
},
|
||||
logprobs: lp_values,
|
||||
ranks: rank_values,
|
||||
candidate_tokens: candidates,
|
||||
finish_info: finished.map(|f| to_finish_info(f, token_ids)),
|
||||
}
|
||||
}
|
||||
|
||||
fn to_finish_info(finished: &Finished, token_ids: &[u32]) -> pb::FinishInfo {
|
||||
use pb::finish_info::FinishReason as PbFinishReason;
|
||||
|
||||
let (finish_reason, stop_reason) = match &finished.finish_reason {
|
||||
FinishReason::Stop(reason) => {
|
||||
let sr = match reason {
|
||||
Some(StopReason::TokenId(id)) => {
|
||||
Some(pb::finish_info::StopReason::StopTokenId(*id))
|
||||
}
|
||||
Some(StopReason::Text(s)) => {
|
||||
Some(pb::finish_info::StopReason::StopString(s.clone()))
|
||||
}
|
||||
// EOS-driven stop: engine-core matched the primary EOS token id but did not
|
||||
// echo it back as a `stop_reason`. The matched token is, by construction, the
|
||||
// last token of the terminal output batch (see vllm's `check_stop` in
|
||||
// vllm/v1/core/sched/utils.py), so we recover it from there.
|
||||
None => token_ids.last().copied().map(pb::finish_info::StopReason::EosTokenId),
|
||||
};
|
||||
(PbFinishReason::Stop as i32, sr)
|
||||
}
|
||||
FinishReason::Length => (PbFinishReason::Length as i32, None),
|
||||
FinishReason::Abort | FinishReason::Error | FinishReason::Repetition => {
|
||||
(PbFinishReason::Aborted as i32, None)
|
||||
}
|
||||
};
|
||||
|
||||
pb::FinishInfo {
|
||||
num_output_tokens: finished.output_token_count as u32,
|
||||
finish_reason,
|
||||
stop_reason,
|
||||
kv_transfer_params: finished.kv_transfer_params.as_ref().and_then(json_to_proto_struct),
|
||||
}
|
||||
}
|
||||
|
||||
// ========================================================================================
|
||||
// Logprobs helpers
|
||||
// ========================================================================================
|
||||
|
||||
/// Convert output logprobs to the flat proto representation.
|
||||
///
|
||||
/// Returns (logprob_values, ranks, candidate_tokens) — all parallel arrays
|
||||
/// indexed by position.
|
||||
fn output_logprobs_to_proto(
|
||||
lp: &DecodedLogprobs,
|
||||
) -> (Vec<f32>, Vec<u32>, Vec<pb::CandidateTokenInfo>) {
|
||||
positions_to_proto(&lp.positions)
|
||||
}
|
||||
|
||||
/// Convert prompt logprobs to the flat proto representation.
|
||||
fn prompt_logprobs_to_proto(
|
||||
plp: &DecodedPromptLogprobs,
|
||||
) -> (Vec<f32>, Vec<u32>, Vec<pb::CandidateTokenInfo>) {
|
||||
// The proto PromptInfo has flat parallel arrays covering all prompt positions.
|
||||
// DecodedPromptLogprobs has first_token separately + scored_positions for the
|
||||
// rest. The first prompt position has no scores, so we emit zeros for it.
|
||||
let (mut logprobs, mut ranks, mut candidates) = positions_to_proto(&plp.scored_positions);
|
||||
logprobs.insert(0, 0.0);
|
||||
ranks.insert(0, 0);
|
||||
candidates.insert(0, pb::CandidateTokenInfo { tokens: vec![] });
|
||||
(logprobs, ranks, candidates)
|
||||
}
|
||||
|
||||
/// Shared helper: convert a slice of decoded position logprobs to flat proto
|
||||
/// arrays.
|
||||
fn positions_to_proto(
|
||||
positions: &[vllm_text::DecodedPositionLogprobs],
|
||||
) -> (Vec<f32>, Vec<u32>, Vec<pb::CandidateTokenInfo>) {
|
||||
let mut logprobs = Vec::with_capacity(positions.len());
|
||||
let mut ranks = Vec::with_capacity(positions.len());
|
||||
let mut candidates = Vec::with_capacity(positions.len());
|
||||
|
||||
for pos in positions {
|
||||
// First entry is the sampled/scored token.
|
||||
if let Some(first) = pos.entries.first() {
|
||||
logprobs.push(first.logprob);
|
||||
ranks.push(first.rank);
|
||||
}
|
||||
|
||||
// Extra candidates beyond the first.
|
||||
let entries = pos.entries.iter().skip(1);
|
||||
candidates.push(pb::CandidateTokenInfo {
|
||||
tokens: entries
|
||||
.map(|e| pb::candidate_token_info::TokenInfo {
|
||||
id: e.token_id,
|
||||
logprob: e.logprob,
|
||||
rank: e.rank,
|
||||
})
|
||||
.collect(),
|
||||
});
|
||||
}
|
||||
|
||||
(logprobs, ranks, candidates)
|
||||
}
|
||||
|
||||
// ========================================================================================
|
||||
// KV transfer params conversion (serde_json::Value ↔ prost_types::Struct)
|
||||
// ========================================================================================
|
||||
|
||||
fn proto_struct_to_json(s: &prost_types::Struct) -> serde_json::Value {
|
||||
serde_json::Value::Object(
|
||||
s.fields.iter().map(|(k, v)| (k.clone(), proto_value_to_json(v))).collect(),
|
||||
)
|
||||
}
|
||||
|
||||
fn proto_value_to_json(v: &prost_types::Value) -> serde_json::Value {
|
||||
use prost_types::value::Kind;
|
||||
match v.kind.as_ref() {
|
||||
None | Some(Kind::NullValue(_)) => serde_json::Value::Null,
|
||||
Some(Kind::BoolValue(b)) => serde_json::Value::Bool(*b),
|
||||
Some(Kind::NumberValue(n)) => serde_json::json!(*n),
|
||||
Some(Kind::StringValue(s)) => serde_json::Value::String(s.clone()),
|
||||
Some(Kind::ListValue(list)) => {
|
||||
serde_json::Value::Array(list.values.iter().map(proto_value_to_json).collect())
|
||||
}
|
||||
Some(Kind::StructValue(s)) => proto_struct_to_json(s),
|
||||
}
|
||||
}
|
||||
|
||||
fn json_to_proto_struct(value: &serde_json::Value) -> Option<prost_types::Struct> {
|
||||
match value {
|
||||
serde_json::Value::Object(map) => Some(prost_types::Struct {
|
||||
fields: map.iter().map(|(k, v)| (k.clone(), json_to_proto_value(v))).collect(),
|
||||
}),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
fn json_to_proto_value(v: &serde_json::Value) -> prost_types::Value {
|
||||
use prost_types::value::Kind;
|
||||
let kind = match v {
|
||||
serde_json::Value::Null => Kind::NullValue(0),
|
||||
serde_json::Value::Bool(b) => Kind::BoolValue(*b),
|
||||
serde_json::Value::Number(n) => Kind::NumberValue(n.as_f64().unwrap_or(0.0)),
|
||||
serde_json::Value::String(s) => Kind::StringValue(s.clone()),
|
||||
serde_json::Value::Array(arr) => Kind::ListValue(prost_types::ListValue {
|
||||
values: arr.iter().map(json_to_proto_value).collect(),
|
||||
}),
|
||||
serde_json::Value::Object(map) => Kind::StructValue(prost_types::Struct {
|
||||
fields: map.iter().map(|(k, v)| (k.clone(), json_to_proto_value(v))).collect(),
|
||||
}),
|
||||
};
|
||||
prost_types::Value { kind: Some(kind) }
|
||||
}
|
||||
|
||||
// ========================================================================================
|
||||
// Options extracted from the request for response building
|
||||
// ========================================================================================
|
||||
|
||||
/// Response-shaping options extracted from the proto `ResponseOptions`.
|
||||
#[derive(Default)]
|
||||
pub struct ResponseOpts {
|
||||
pub prompt_token_ids: bool,
|
||||
pub prompt_logprobs: bool,
|
||||
pub output_text: bool,
|
||||
pub output_token_ids: bool,
|
||||
pub output_logprobs: bool,
|
||||
}
|
||||
|
||||
impl ResponseOpts {
|
||||
pub fn from_proto(r: Option<&pb::ResponseOptions>) -> Self {
|
||||
match r {
|
||||
Some(r) => Self {
|
||||
prompt_token_ids: r.prompt_token_ids,
|
||||
prompt_logprobs: r.prompt_logprobs,
|
||||
output_text: r.output_text.unwrap_or(true),
|
||||
output_token_ids: r.output_token_ids,
|
||||
output_logprobs: r.output_logprobs,
|
||||
},
|
||||
None => Self {
|
||||
output_text: true,
|
||||
..Default::default()
|
||||
},
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use vllm_engine_core_client::protocol::StopReason;
|
||||
use vllm_text::{FinishReason, Finished, Prompt};
|
||||
|
||||
use super::pb::finish_info::{FinishReason as PbFinishReason, StopReason as PbStopReason};
|
||||
use super::{ResponseOpts, pb, to_finish_info, to_sequence_output, to_text_request};
|
||||
|
||||
fn base_request() -> pb::GenerateRequest {
|
||||
pb::GenerateRequest {
|
||||
request_id: "req".to_string(),
|
||||
model: "test-model".to_string(),
|
||||
prompt: Some(pb::generate_request::Prompt::Text("hi".to_string())),
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn temperature_propagates_from_top_level_request_field() {
|
||||
let req = pb::GenerateRequest {
|
||||
temperature: Some(0.7),
|
||||
..base_request()
|
||||
};
|
||||
let text = to_text_request(req, false, &["test-model".to_string()]).expect("convert ok");
|
||||
assert_eq!(text.sampling_params.temperature, Some(0.7));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn unset_temperature_defaults_to_greedy() {
|
||||
let text = to_text_request(base_request(), false, &["test-model".to_string()])
|
||||
.expect("convert ok");
|
||||
// The gRPC API defaults to greedy (0.0) when temperature is not specified.
|
||||
assert_eq!(text.sampling_params.temperature, Some(0.0));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn absent_seed_is_none() {
|
||||
let req = pb::GenerateRequest {
|
||||
sampling: Some(pb::RandomSampling {
|
||||
seed: None,
|
||||
..Default::default()
|
||||
}),
|
||||
..base_request()
|
||||
};
|
||||
let text = to_text_request(req, false, &["test-model".to_string()]).expect("convert ok");
|
||||
assert_eq!(text.sampling_params.seed, None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn zero_seed_is_valid() {
|
||||
let req = pb::GenerateRequest {
|
||||
sampling: Some(pb::RandomSampling {
|
||||
seed: Some(0),
|
||||
..Default::default()
|
||||
}),
|
||||
..base_request()
|
||||
};
|
||||
let text = to_text_request(req, false, &["test-model".to_string()]).expect("convert ok");
|
||||
assert_eq!(text.sampling_params.seed, Some(0));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn bypass_prefix_cache_maps_to_skip_reading_prefix_cache() {
|
||||
let req = pb::GenerateRequest {
|
||||
kv: Some(pb::KvCacheParameters {
|
||||
bypass_prefix_cache: true,
|
||||
..Default::default()
|
||||
}),
|
||||
..base_request()
|
||||
};
|
||||
let text = to_text_request(req, false, &["test-model".to_string()]).expect("convert ok");
|
||||
assert_eq!(text.sampling_params.skip_reading_prefix_cache, Some(true));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn bypass_prefix_cache_false_leaves_field_unset() {
|
||||
let req = pb::GenerateRequest {
|
||||
kv: Some(pb::KvCacheParameters {
|
||||
bypass_prefix_cache: false,
|
||||
..Default::default()
|
||||
}),
|
||||
..base_request()
|
||||
};
|
||||
let text = to_text_request(req, false, &["test-model".to_string()]).expect("convert ok");
|
||||
assert_eq!(text.sampling_params.skip_reading_prefix_cache, None);
|
||||
// Prompt conversion still succeeds and reaches the expected variant.
|
||||
assert!(matches!(text.prompt, Prompt::Text(s) if s == "hi"));
|
||||
}
|
||||
|
||||
fn finished(reason: FinishReason) -> Finished {
|
||||
Finished {
|
||||
prompt_token_count: 0,
|
||||
output_token_count: 0,
|
||||
finish_reason: reason,
|
||||
kv_transfer_params: None,
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn eos_stop_reports_last_output_token_as_eos_id() {
|
||||
let fin = finished(FinishReason::Stop(None));
|
||||
let token_ids = [1_u32, 2, 3, 151643];
|
||||
|
||||
let info = to_finish_info(&fin, &token_ids);
|
||||
|
||||
assert_eq!(info.finish_reason, PbFinishReason::Stop as i32);
|
||||
assert_eq!(info.stop_reason, Some(PbStopReason::EosTokenId(151643)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn eos_stop_with_empty_token_ids_leaves_stop_reason_unset() {
|
||||
let fin = finished(FinishReason::Stop(None));
|
||||
|
||||
let info = to_finish_info(&fin, &[]);
|
||||
|
||||
assert_eq!(info.finish_reason, PbFinishReason::Stop as i32);
|
||||
assert_eq!(info.stop_reason, None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn explicit_stop_token_id_is_preserved() {
|
||||
let fin = finished(FinishReason::Stop(Some(StopReason::TokenId(42))));
|
||||
// Terminal token list should be ignored when an explicit stop reason is
|
||||
// present.
|
||||
let info = to_finish_info(&fin, &[7, 42]);
|
||||
|
||||
assert_eq!(info.finish_reason, PbFinishReason::Stop as i32);
|
||||
assert_eq!(info.stop_reason, Some(PbStopReason::StopTokenId(42)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn explicit_stop_string_is_preserved() {
|
||||
let fin = finished(FinishReason::Stop(Some(StopReason::Text("</stop>".into()))));
|
||||
|
||||
let info = to_finish_info(&fin, &[1, 2, 3]);
|
||||
|
||||
assert_eq!(info.finish_reason, PbFinishReason::Stop as i32);
|
||||
assert_eq!(
|
||||
info.stop_reason,
|
||||
Some(PbStopReason::StopString("</stop>".into()))
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn length_finish_has_no_stop_reason() {
|
||||
let fin = finished(FinishReason::Length);
|
||||
|
||||
let info = to_finish_info(&fin, &[1, 2, 3]);
|
||||
|
||||
assert_eq!(info.finish_reason, PbFinishReason::Length as i32);
|
||||
assert_eq!(info.stop_reason, None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn abort_finish_is_mapped_to_aborted() {
|
||||
let fin = finished(FinishReason::Abort);
|
||||
|
||||
let info = to_finish_info(&fin, &[]);
|
||||
|
||||
assert_eq!(info.finish_reason, PbFinishReason::Aborted as i32);
|
||||
assert_eq!(info.stop_reason, None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn to_sequence_output_threads_token_ids_into_eos_id() {
|
||||
let fin = finished(FinishReason::Stop(None));
|
||||
let opts = ResponseOpts {
|
||||
output_text: true,
|
||||
output_token_ids: true,
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let out = to_sequence_output("hello", &[10, 20, 30], None, Some(&fin), &opts);
|
||||
|
||||
let finish = out.finish_info.expect("finish_info should be present");
|
||||
assert_eq!(finish.finish_reason, PbFinishReason::Stop as i32);
|
||||
assert_eq!(finish.stop_reason, Some(PbStopReason::EosTokenId(30)));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,158 @@
|
||||
//! gRPC Generate service backed by the shared [`vllm_text::TextLlm`] facade.
|
||||
|
||||
mod convert;
|
||||
|
||||
use std::pin::Pin;
|
||||
use std::sync::Arc;
|
||||
|
||||
use futures::{Stream, StreamExt as _};
|
||||
use thiserror_ext::AsReport as _;
|
||||
use tokio::sync::mpsc;
|
||||
use tokio_stream::wrappers::ReceiverStream;
|
||||
use tonic::{Request, Response, Status};
|
||||
use tracing::info;
|
||||
use vllm_text::{DecodedTextEvent, TextOutputStreamExt as _};
|
||||
|
||||
use self::convert::ResponseOpts;
|
||||
use crate::state::AppState;
|
||||
|
||||
/// Generated protobuf/gRPC types for the `vllm` package.
|
||||
pub mod pb {
|
||||
tonic::include_proto!("vllm");
|
||||
}
|
||||
|
||||
pub use pb::generate_server::GenerateServer;
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests;
|
||||
|
||||
/// gRPC Generate service implementation backed by the shared application state.
|
||||
pub struct GenerateServiceImpl {
|
||||
state: Arc<AppState>,
|
||||
}
|
||||
|
||||
impl GenerateServiceImpl {
|
||||
pub fn new(state: Arc<AppState>) -> Self {
|
||||
Self { state }
|
||||
}
|
||||
}
|
||||
|
||||
#[tonic::async_trait]
|
||||
impl pb::generate_server::Generate for GenerateServiceImpl {
|
||||
type GenerateStreamStream =
|
||||
Pin<Box<dyn Stream<Item = Result<pb::GenerateResponse, Status>> + Send>>;
|
||||
|
||||
/// Unary generate: collect all output and return a single response.
|
||||
async fn generate(
|
||||
&self,
|
||||
request: Request<pb::GenerateRequest>,
|
||||
) -> Result<Response<pb::GenerateResponse>, Status> {
|
||||
let proto_req = request.into_inner();
|
||||
let response_opts = ResponseOpts::from_proto(proto_req.response.as_ref());
|
||||
let text_request =
|
||||
convert::to_text_request(proto_req, false, self.state.served_model_names())?;
|
||||
|
||||
let request_id = text_request.request_id.clone();
|
||||
info!(%request_id, "grpc generate (unary)");
|
||||
|
||||
let stream = self.state.chat.text().generate(text_request).await;
|
||||
let stream = stream.map_err(|e| Status::internal(e.to_report_string()))?;
|
||||
|
||||
let collected = stream
|
||||
.collect_output()
|
||||
.await
|
||||
.map_err(|e| Status::internal(e.to_report_string()))?;
|
||||
|
||||
// Build the single aggregated response.
|
||||
let prompt_info = convert::to_prompt_info(
|
||||
&collected.prompt_token_ids,
|
||||
collected.prompt_logprobs.as_ref(),
|
||||
&response_opts,
|
||||
);
|
||||
|
||||
let finish_info = vllm_text::Finished {
|
||||
prompt_token_count: collected.prompt_token_ids.len(),
|
||||
output_token_count: collected.token_ids.len(),
|
||||
finish_reason: collected.finish_reason,
|
||||
kv_transfer_params: collected.kv_transfer_params,
|
||||
};
|
||||
|
||||
let outputs = convert::to_sequence_output(
|
||||
&collected.text,
|
||||
&collected.token_ids,
|
||||
collected.logprobs.as_ref(),
|
||||
Some(&finish_info),
|
||||
&response_opts,
|
||||
);
|
||||
|
||||
Ok(Response::new(pb::GenerateResponse {
|
||||
prompt_info: Some(prompt_info),
|
||||
outputs: Some(outputs),
|
||||
}))
|
||||
}
|
||||
|
||||
/// Streaming generate: yield incremental responses as tokens are produced.
|
||||
async fn generate_stream(
|
||||
&self,
|
||||
request: Request<pb::GenerateRequest>,
|
||||
) -> Result<Response<Self::GenerateStreamStream>, Status> {
|
||||
let proto_req = request.into_inner();
|
||||
let response_opts = ResponseOpts::from_proto(proto_req.response.as_ref());
|
||||
let text_request =
|
||||
convert::to_text_request(proto_req, true, self.state.served_model_names())?;
|
||||
|
||||
let request_id = text_request.request_id.clone();
|
||||
info!(%request_id, "grpc generate (stream)");
|
||||
|
||||
let stream = self.state.chat.text().generate(text_request).await;
|
||||
let stream = stream.map_err(|e| Status::internal(e.to_report_string()))?;
|
||||
|
||||
let (tx, rx) = mpsc::channel(32);
|
||||
|
||||
tokio::spawn(async move {
|
||||
futures::pin_mut!(stream);
|
||||
while let Some(event) = stream.next().await {
|
||||
let response = match event {
|
||||
Err(e) => Err(Status::internal(e.to_report_string())),
|
||||
Ok(DecodedTextEvent::Start {
|
||||
prompt_token_ids,
|
||||
prompt_logprobs,
|
||||
}) => {
|
||||
let prompt_info = convert::to_prompt_info(
|
||||
&prompt_token_ids,
|
||||
prompt_logprobs.as_ref(),
|
||||
&response_opts,
|
||||
);
|
||||
Ok(pb::GenerateResponse {
|
||||
prompt_info: Some(prompt_info),
|
||||
outputs: None,
|
||||
})
|
||||
}
|
||||
Ok(DecodedTextEvent::TextDelta {
|
||||
delta,
|
||||
token_ids,
|
||||
logprobs,
|
||||
finished,
|
||||
}) => Ok(pb::GenerateResponse {
|
||||
prompt_info: None,
|
||||
outputs: Some(convert::to_sequence_output(
|
||||
&delta,
|
||||
&token_ids,
|
||||
logprobs.as_ref(),
|
||||
finished.as_ref(),
|
||||
&response_opts,
|
||||
)),
|
||||
}),
|
||||
};
|
||||
|
||||
if tx.send(response).await.is_err() {
|
||||
// Client disconnected.
|
||||
break;
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
let response_stream = ReceiverStream::new(rx);
|
||||
Ok(Response::new(Box::pin(response_stream)))
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,662 @@
|
||||
use std::future::Future;
|
||||
use std::pin::Pin;
|
||||
use std::sync::Arc;
|
||||
use std::task::{Context, Poll};
|
||||
|
||||
use futures::StreamExt as _;
|
||||
use serial_test::serial;
|
||||
use tonic::transport::Server as TonicServer;
|
||||
use vllm_chat::{
|
||||
ChatBackend, ChatLlm, ChatRenderer, ChatRequest, ChatTextBackend, DefaultChatOutputProcessor,
|
||||
DynChatOutputProcessor, DynChatRenderer, NewChatOutputProcessorOptions, RenderedPrompt,
|
||||
};
|
||||
use vllm_engine_core_client::protocol::{
|
||||
EngineCoreFinishReason, EngineCoreOutput, EngineCoreOutputs, EngineCoreRequest,
|
||||
};
|
||||
use vllm_engine_core_client::test_utils::{IpcNamespace, spawn_mock_engine_task};
|
||||
use vllm_engine_core_client::{EngineCoreClient, EngineCoreClientConfig, EngineId};
|
||||
use vllm_llm::Llm;
|
||||
use vllm_text::tokenizer::{DynTokenizer, Tokenizer};
|
||||
use vllm_text::{Prompt, TextBackend};
|
||||
use zeromq::prelude::{SocketRecv, SocketSend};
|
||||
use zeromq::{DealerSocket, PushSocket, ZmqMessage};
|
||||
|
||||
use super::pb::generate_client::GenerateClient;
|
||||
use super::{GenerateServer, GenerateServiceImpl, pb};
|
||||
use crate::state::AppState;
|
||||
|
||||
// ========================================================================================
|
||||
// Helpers (mirrors the patterns in routes/tests.rs)
|
||||
// ========================================================================================
|
||||
|
||||
type TestFuture<'a> = Pin<Box<dyn Future<Output = ()> + Send + 'a>>;
|
||||
|
||||
fn boxed_test_future<'a>(future: impl Future<Output = ()> + Send + 'a) -> TestFuture<'a> {
|
||||
Box::pin(future)
|
||||
}
|
||||
|
||||
struct MockEngineTask {
|
||||
shutdown_tx: Option<tokio::sync::oneshot::Sender<()>>,
|
||||
join_handle: Option<tokio::task::JoinHandle<()>>,
|
||||
}
|
||||
|
||||
impl MockEngineTask {
|
||||
fn new(
|
||||
(shutdown_tx, join_handle): (
|
||||
tokio::sync::oneshot::Sender<()>,
|
||||
tokio::task::JoinHandle<()>,
|
||||
),
|
||||
) -> Self {
|
||||
Self {
|
||||
shutdown_tx: Some(shutdown_tx),
|
||||
join_handle: Some(join_handle),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Future for MockEngineTask {
|
||||
type Output = Result<(), tokio::task::JoinError>;
|
||||
|
||||
fn poll(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Self::Output> {
|
||||
if let Some(shutdown_tx) = self.shutdown_tx.take() {
|
||||
let _ = shutdown_tx.send(());
|
||||
}
|
||||
match self.join_handle.as_mut() {
|
||||
Some(join_handle) => Pin::new(join_handle).poll(cx),
|
||||
None => Poll::Ready(Ok(())),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for MockEngineTask {
|
||||
fn drop(&mut self) {
|
||||
if let Some(join_handle) = &self.join_handle {
|
||||
join_handle.abort();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn request_output(
|
||||
request_id: &str,
|
||||
new_token_ids: Vec<u32>,
|
||||
finish_reason: Option<EngineCoreFinishReason>,
|
||||
) -> EngineCoreOutput {
|
||||
EngineCoreOutput {
|
||||
request_id: request_id.to_string(),
|
||||
new_token_ids,
|
||||
new_logprobs: None,
|
||||
new_prompt_logprobs_tensors: None,
|
||||
pooling_output: None,
|
||||
finish_reason,
|
||||
stop_reason: None,
|
||||
events: None,
|
||||
kv_transfer_params: None,
|
||||
trace_headers: None,
|
||||
prefill_stats: None,
|
||||
routed_experts: None,
|
||||
num_nans_in_logits: 0,
|
||||
}
|
||||
}
|
||||
|
||||
fn engine_outputs_for_request(
|
||||
request_id: &str,
|
||||
output_specs: Vec<(Vec<u32>, Option<EngineCoreFinishReason>)>,
|
||||
) -> EngineCoreOutputs {
|
||||
EngineCoreOutputs {
|
||||
engine_index: 0,
|
||||
outputs: output_specs
|
||||
.into_iter()
|
||||
.map(|(token_ids, finish_reason)| request_output(request_id, token_ids, finish_reason))
|
||||
.collect(),
|
||||
scheduler_stats: None,
|
||||
timestamp: 0.0,
|
||||
utility_output: None,
|
||||
finished_requests: None,
|
||||
wave_complete: None,
|
||||
start_wave: None,
|
||||
}
|
||||
}
|
||||
|
||||
fn default_stream_output_specs() -> Vec<(Vec<u32>, Option<EngineCoreFinishReason>)> {
|
||||
vec![
|
||||
(vec![b'h' as u32], None),
|
||||
(vec![b'i' as u32], None),
|
||||
(vec![b'!' as u32], Some(EngineCoreFinishReason::Stop)),
|
||||
]
|
||||
}
|
||||
|
||||
async fn send_outputs(push: &mut PushSocket, outputs: EngineCoreOutputs) {
|
||||
push.send(ZmqMessage::from(
|
||||
rmp_serde::to_vec_named(&outputs).expect("encode outputs"),
|
||||
))
|
||||
.await
|
||||
.expect("send outputs");
|
||||
}
|
||||
|
||||
async fn recv_engine_message(dealer: &mut DealerSocket) -> Vec<bytes::Bytes> {
|
||||
dealer.recv().await.expect("recv engine message").into_vec()
|
||||
}
|
||||
|
||||
fn test_llm(client: EngineCoreClient) -> Llm {
|
||||
Llm::new(client).with_request_id_randomization(false)
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
struct FakeTextBackend;
|
||||
|
||||
#[derive(Debug)]
|
||||
struct FakeTokenizer;
|
||||
|
||||
impl Tokenizer for FakeTokenizer {
|
||||
fn encode(
|
||||
&self,
|
||||
text: &str,
|
||||
_add_special_tokens: bool,
|
||||
) -> vllm_text::tokenizer::Result<Vec<u32>> {
|
||||
Ok(text.bytes().map(u32::from).collect())
|
||||
}
|
||||
|
||||
fn decode(
|
||||
&self,
|
||||
token_ids: &[u32],
|
||||
_skip_special_tokens: bool,
|
||||
) -> vllm_text::tokenizer::Result<String> {
|
||||
Ok(
|
||||
String::from_utf8_lossy(&token_ids.iter().map(|id| *id as u8).collect::<Vec<_>>())
|
||||
.into_owned(),
|
||||
)
|
||||
}
|
||||
|
||||
fn token_to_id(&self, token: &str) -> Option<u32> {
|
||||
token.bytes().next().map(u32::from)
|
||||
}
|
||||
}
|
||||
|
||||
impl TextBackend for FakeTextBackend {
|
||||
fn tokenizer(&self) -> DynTokenizer {
|
||||
Arc::new(FakeTokenizer)
|
||||
}
|
||||
|
||||
fn model_id(&self) -> &str {
|
||||
"test-model"
|
||||
}
|
||||
}
|
||||
|
||||
impl ChatBackend for FakeTextBackend {
|
||||
fn chat_renderer(&self) -> DynChatRenderer {
|
||||
Arc::new(self.clone())
|
||||
}
|
||||
|
||||
fn new_chat_output_processor(
|
||||
&self,
|
||||
request: &mut ChatRequest,
|
||||
options: NewChatOutputProcessorOptions<'_>,
|
||||
) -> vllm_chat::Result<DynChatOutputProcessor> {
|
||||
Ok(Box::new(DefaultChatOutputProcessor::new(
|
||||
request,
|
||||
self.model_id(),
|
||||
self.tokenizer(),
|
||||
options.tool_call_parser,
|
||||
options.reasoning_parser,
|
||||
)?))
|
||||
}
|
||||
}
|
||||
|
||||
impl ChatRenderer for FakeTextBackend {
|
||||
fn render(&self, _request: &ChatRequest) -> vllm_chat::Result<RenderedPrompt> {
|
||||
Ok(RenderedPrompt {
|
||||
prompt: Prompt::Text(String::new()),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
/// Spin up a gRPC server backed by a mock engine that serves a single request
|
||||
/// with the given output specs. Returns the client, the gRPC server task, and
|
||||
/// the mock engine task.
|
||||
async fn grpc_test_server(
|
||||
engine_id: impl Into<EngineId>,
|
||||
output_specs: Vec<(Vec<u32>, Option<EngineCoreFinishReason>)>,
|
||||
) -> (
|
||||
GenerateClient<tonic::transport::Channel>,
|
||||
tokio::task::JoinHandle<()>,
|
||||
MockEngineTask,
|
||||
) {
|
||||
let ipc = IpcNamespace::new().expect("create ipc namespace");
|
||||
let handshake_address = ipc.handshake_endpoint();
|
||||
let engine_id = engine_id.into();
|
||||
|
||||
let engine_task = MockEngineTask::new(spawn_mock_engine_task(
|
||||
handshake_address.clone(),
|
||||
engine_id.clone(),
|
||||
move |dealer, push| {
|
||||
boxed_test_future(async move {
|
||||
let add = recv_engine_message(dealer).await;
|
||||
let request: EngineCoreRequest =
|
||||
rmp_serde::from_slice(&add[1]).expect("decode request");
|
||||
send_outputs(
|
||||
push,
|
||||
engine_outputs_for_request(&request.request_id, output_specs),
|
||||
)
|
||||
.await;
|
||||
})
|
||||
},
|
||||
));
|
||||
|
||||
let client = EngineCoreClient::connect(
|
||||
EngineCoreClientConfig::new_single(handshake_address)
|
||||
.with_model_name("test-model")
|
||||
.with_local_input_output_addresses(
|
||||
Some(ipc.input_endpoint()),
|
||||
Some(ipc.output_endpoint()),
|
||||
),
|
||||
)
|
||||
.await
|
||||
.expect("connect client");
|
||||
|
||||
let chat = ChatLlm::from_shared_backend(
|
||||
test_llm(client),
|
||||
Arc::new(FakeTextBackend) as Arc<dyn ChatTextBackend>,
|
||||
);
|
||||
let state = Arc::new(AppState::new(vec!["test-model".to_string()], chat));
|
||||
let svc = GenerateServer::new(GenerateServiceImpl::new(state));
|
||||
|
||||
// Bind to an OS-assigned port.
|
||||
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.expect("bind grpc listener");
|
||||
let addr = listener.local_addr().expect("local addr");
|
||||
|
||||
let server_task = tokio::spawn(async move {
|
||||
let incoming = tokio_stream::wrappers::TcpListenerStream::new(listener);
|
||||
TonicServer::builder()
|
||||
.add_service(svc)
|
||||
.serve_with_incoming(incoming)
|
||||
.await
|
||||
.expect("grpc server");
|
||||
});
|
||||
|
||||
// Connect the client.
|
||||
let grpc_client = GenerateClient::connect(format!("http://{addr}"))
|
||||
.await
|
||||
.expect("connect grpc client");
|
||||
|
||||
(grpc_client, server_task, engine_task)
|
||||
}
|
||||
|
||||
// ========================================================================================
|
||||
// Tests
|
||||
// ========================================================================================
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
#[serial]
|
||||
async fn unary_generate_returns_collected_text() {
|
||||
let (mut client, server_task, engine_task) =
|
||||
grpc_test_server(b"engine-grpc-unary", default_stream_output_specs()).await;
|
||||
|
||||
let response = client
|
||||
.generate(pb::GenerateRequest {
|
||||
request_id: "test-unary-1".to_string(),
|
||||
model: "test-model".to_string(),
|
||||
prompt: Some(pb::generate_request::Prompt::Text("hello".to_string())),
|
||||
stopping: Some(pb::StoppingCriteria {
|
||||
max_new_tokens: 10,
|
||||
..Default::default()
|
||||
}),
|
||||
response: Some(pb::ResponseOptions {
|
||||
output_text: Some(true),
|
||||
..Default::default()
|
||||
}),
|
||||
..Default::default()
|
||||
})
|
||||
.await
|
||||
.expect("unary generate")
|
||||
.into_inner();
|
||||
|
||||
// Unary collects all tokens into one response.
|
||||
let outputs = response.outputs.expect("outputs present");
|
||||
assert_eq!(outputs.text, "hi");
|
||||
|
||||
let finish = outputs.finish_info.expect("finish_info present");
|
||||
assert_eq!(
|
||||
finish.finish_reason,
|
||||
pb::finish_info::FinishReason::Stop as i32
|
||||
);
|
||||
assert_eq!(finish.num_output_tokens, 3);
|
||||
|
||||
let prompt = response.prompt_info.expect("prompt_info present");
|
||||
assert_eq!(prompt.num_prompt_tokens, 5); // "hello" = 5 bytes
|
||||
|
||||
engine_task.await.expect("mock engine task");
|
||||
server_task.abort();
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
#[serial]
|
||||
async fn unary_generate_with_token_ids_prompt() {
|
||||
let (mut client, server_task, engine_task) =
|
||||
grpc_test_server(b"engine-grpc-token-ids", default_stream_output_specs()).await;
|
||||
|
||||
let response = client
|
||||
.generate(pb::GenerateRequest {
|
||||
request_id: "test-token-ids".to_string(),
|
||||
model: "test-model".to_string(),
|
||||
prompt: Some(pb::generate_request::Prompt::TokenIds(pb::TokenIds {
|
||||
ids: vec![1, 2, 3],
|
||||
})),
|
||||
stopping: Some(pb::StoppingCriteria {
|
||||
max_new_tokens: 10,
|
||||
..Default::default()
|
||||
}),
|
||||
..Default::default()
|
||||
})
|
||||
.await
|
||||
.expect("unary generate with token ids")
|
||||
.into_inner();
|
||||
|
||||
let outputs = response.outputs.expect("outputs present");
|
||||
assert_eq!(outputs.text, "hi");
|
||||
assert_eq!(
|
||||
response.prompt_info.expect("prompt_info").num_prompt_tokens,
|
||||
3
|
||||
);
|
||||
|
||||
engine_task.await.expect("mock engine task");
|
||||
server_task.abort();
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
#[serial]
|
||||
async fn unary_generate_returns_token_ids_when_requested() {
|
||||
let (mut client, server_task, engine_task) =
|
||||
grpc_test_server(b"engine-grpc-tok-resp", default_stream_output_specs()).await;
|
||||
|
||||
let response = client
|
||||
.generate(pb::GenerateRequest {
|
||||
request_id: "test-tok-resp".to_string(),
|
||||
model: "test-model".to_string(),
|
||||
prompt: Some(pb::generate_request::Prompt::Text("hi".to_string())),
|
||||
stopping: Some(pb::StoppingCriteria {
|
||||
max_new_tokens: 10,
|
||||
..Default::default()
|
||||
}),
|
||||
response: Some(pb::ResponseOptions {
|
||||
output_text: Some(true),
|
||||
output_token_ids: true,
|
||||
prompt_token_ids: true,
|
||||
..Default::default()
|
||||
}),
|
||||
..Default::default()
|
||||
})
|
||||
.await
|
||||
.expect("unary generate")
|
||||
.into_inner();
|
||||
|
||||
let outputs = response.outputs.expect("outputs present");
|
||||
assert_eq!(
|
||||
outputs.token_ids,
|
||||
vec![b'h' as u32, b'i' as u32, b'!' as u32]
|
||||
);
|
||||
|
||||
let prompt = response.prompt_info.expect("prompt_info present");
|
||||
assert_eq!(prompt.token_ids, vec![b'h' as u32, b'i' as u32]);
|
||||
|
||||
engine_task.await.expect("mock engine task");
|
||||
server_task.abort();
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
#[serial]
|
||||
async fn unary_generate_missing_prompt_returns_invalid_argument() {
|
||||
let (mut client, server_task, _engine_task) =
|
||||
grpc_test_server(b"engine-grpc-no-prompt", default_stream_output_specs()).await;
|
||||
|
||||
let status = client
|
||||
.generate(pb::GenerateRequest {
|
||||
request_id: "test-no-prompt".to_string(),
|
||||
model: "test-model".to_string(),
|
||||
prompt: None,
|
||||
..Default::default()
|
||||
})
|
||||
.await
|
||||
.expect_err("should fail without prompt");
|
||||
|
||||
assert_eq!(status.code(), tonic::Code::InvalidArgument);
|
||||
assert!(status.message().contains("prompt"));
|
||||
|
||||
server_task.abort();
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
#[serial]
|
||||
async fn streaming_generate_yields_incremental_responses() {
|
||||
let (mut client, server_task, engine_task) =
|
||||
grpc_test_server(b"engine-grpc-stream", default_stream_output_specs()).await;
|
||||
|
||||
let stream = client
|
||||
.generate_stream(pb::GenerateRequest {
|
||||
request_id: "test-stream-1".to_string(),
|
||||
model: "test-model".to_string(),
|
||||
prompt: Some(pb::generate_request::Prompt::Text("hello".to_string())),
|
||||
stopping: Some(pb::StoppingCriteria {
|
||||
max_new_tokens: 10,
|
||||
..Default::default()
|
||||
}),
|
||||
response: Some(pb::ResponseOptions {
|
||||
output_text: Some(true),
|
||||
..Default::default()
|
||||
}),
|
||||
..Default::default()
|
||||
})
|
||||
.await
|
||||
.expect("streaming generate")
|
||||
.into_inner();
|
||||
|
||||
let responses: Vec<pb::GenerateResponse> =
|
||||
stream.map(|r| r.expect("stream item")).collect().await;
|
||||
|
||||
// First response carries prompt info, subsequent ones carry output deltas.
|
||||
assert!(
|
||||
responses.len() >= 2,
|
||||
"expected at least 2 streamed responses, got {}",
|
||||
responses.len()
|
||||
);
|
||||
|
||||
// First message should have prompt info.
|
||||
let first = &responses[0];
|
||||
let prompt_info = first.prompt_info.as_ref().expect("first response has prompt_info");
|
||||
assert_eq!(prompt_info.num_prompt_tokens, 5); // "hello"
|
||||
|
||||
// Collect all text deltas.
|
||||
let full_text: String = responses
|
||||
.iter()
|
||||
.filter_map(|r| r.outputs.as_ref())
|
||||
.map(|o| o.text.as_str())
|
||||
.collect();
|
||||
assert_eq!(full_text, "hi");
|
||||
|
||||
// Last output response should have finish info.
|
||||
let last_output = responses
|
||||
.iter()
|
||||
.rev()
|
||||
.find_map(|r| r.outputs.as_ref())
|
||||
.expect("at least one output");
|
||||
let finish = last_output.finish_info.as_ref().expect("finish_info on last output");
|
||||
assert_eq!(
|
||||
finish.finish_reason,
|
||||
pb::finish_info::FinishReason::Stop as i32
|
||||
);
|
||||
|
||||
engine_task.await.expect("mock engine task");
|
||||
server_task.abort();
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
#[serial]
|
||||
async fn streaming_generate_missing_prompt_returns_invalid_argument() {
|
||||
let (mut client, server_task, _engine_task) = grpc_test_server(
|
||||
b"engine-grpc-stream-no-prompt",
|
||||
default_stream_output_specs(),
|
||||
)
|
||||
.await;
|
||||
|
||||
let status = client
|
||||
.generate_stream(pb::GenerateRequest {
|
||||
request_id: "test-stream-no-prompt".to_string(),
|
||||
model: "test-model".to_string(),
|
||||
prompt: None,
|
||||
..Default::default()
|
||||
})
|
||||
.await
|
||||
.expect_err("should fail without prompt");
|
||||
|
||||
assert_eq!(status.code(), tonic::Code::InvalidArgument);
|
||||
|
||||
server_task.abort();
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
#[serial]
|
||||
async fn unary_generate_with_sampling_params() {
|
||||
let (mut client, server_task, engine_task) =
|
||||
grpc_test_server(b"engine-grpc-sampling", default_stream_output_specs()).await;
|
||||
|
||||
let response = client
|
||||
.generate(pb::GenerateRequest {
|
||||
request_id: "test-sampling".to_string(),
|
||||
model: "test-model".to_string(),
|
||||
prompt: Some(pb::generate_request::Prompt::Text("test".to_string())),
|
||||
temperature: Some(0.7),
|
||||
sampling: Some(pb::RandomSampling {
|
||||
top_k: 50,
|
||||
top_p: 0.9,
|
||||
seed: Some(42),
|
||||
..Default::default()
|
||||
}),
|
||||
stopping: Some(pb::StoppingCriteria {
|
||||
max_new_tokens: 5,
|
||||
..Default::default()
|
||||
}),
|
||||
..Default::default()
|
||||
})
|
||||
.await
|
||||
.expect("generate with sampling params")
|
||||
.into_inner();
|
||||
|
||||
// Verify the request was accepted and produced output.
|
||||
let outputs = response.outputs.expect("outputs present");
|
||||
assert_eq!(outputs.text, "hi");
|
||||
|
||||
engine_task.await.expect("mock engine task");
|
||||
server_task.abort();
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
#[serial]
|
||||
async fn unary_generate_rejects_wrong_model() {
|
||||
let (mut client, server_task, _engine_task) =
|
||||
grpc_test_server(b"engine-grpc-wrong-model", default_stream_output_specs()).await;
|
||||
|
||||
let status = client
|
||||
.generate(pb::GenerateRequest {
|
||||
request_id: "test-wrong-model".to_string(),
|
||||
model: "other-model".to_string(),
|
||||
prompt: Some(pb::generate_request::Prompt::Text("hi".to_string())),
|
||||
stopping: Some(pb::StoppingCriteria {
|
||||
max_new_tokens: 10,
|
||||
..Default::default()
|
||||
}),
|
||||
..Default::default()
|
||||
})
|
||||
.await
|
||||
.expect_err("should fail with wrong model");
|
||||
|
||||
assert_eq!(status.code(), tonic::Code::NotFound);
|
||||
assert!(status.message().contains("other-model"));
|
||||
|
||||
server_task.abort();
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
#[serial]
|
||||
async fn streaming_generate_rejects_wrong_model() {
|
||||
let (mut client, server_task, _engine_task) = grpc_test_server(
|
||||
b"engine-grpc-stream-wrong-model",
|
||||
default_stream_output_specs(),
|
||||
)
|
||||
.await;
|
||||
|
||||
let status = client
|
||||
.generate_stream(pb::GenerateRequest {
|
||||
request_id: "test-stream-wrong-model".to_string(),
|
||||
model: "other-model".to_string(),
|
||||
prompt: Some(pb::generate_request::Prompt::Text("hi".to_string())),
|
||||
stopping: Some(pb::StoppingCriteria {
|
||||
max_new_tokens: 10,
|
||||
..Default::default()
|
||||
}),
|
||||
..Default::default()
|
||||
})
|
||||
.await
|
||||
.expect_err("should fail with wrong model");
|
||||
|
||||
assert_eq!(status.code(), tonic::Code::NotFound);
|
||||
assert!(status.message().contains("other-model"));
|
||||
|
||||
server_task.abort();
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
#[serial]
|
||||
async fn unary_generate_accepts_empty_model() {
|
||||
let (mut client, server_task, engine_task) =
|
||||
grpc_test_server(b"engine-grpc-empty-model", default_stream_output_specs()).await;
|
||||
|
||||
// Empty `model` (proto3 default) is treated as "unset" and should be accepted.
|
||||
let response = client
|
||||
.generate(pb::GenerateRequest {
|
||||
request_id: "test-empty-model".to_string(),
|
||||
model: String::new(),
|
||||
prompt: Some(pb::generate_request::Prompt::Text("hi".to_string())),
|
||||
stopping: Some(pb::StoppingCriteria {
|
||||
max_new_tokens: 10,
|
||||
..Default::default()
|
||||
}),
|
||||
..Default::default()
|
||||
})
|
||||
.await
|
||||
.expect("unary generate with empty model")
|
||||
.into_inner();
|
||||
|
||||
let outputs = response.outputs.expect("outputs present");
|
||||
assert_eq!(outputs.text, "hi");
|
||||
|
||||
engine_task.await.expect("mock engine task");
|
||||
server_task.abort();
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
#[serial]
|
||||
async fn unary_generate_output_text_defaults_to_true() {
|
||||
let (mut client, server_task, engine_task) =
|
||||
grpc_test_server(b"engine-grpc-default-text", default_stream_output_specs()).await;
|
||||
|
||||
// No response options at all — output_text should default to true.
|
||||
let response = client
|
||||
.generate(pb::GenerateRequest {
|
||||
request_id: "test-default-text".to_string(),
|
||||
model: "test-model".to_string(),
|
||||
prompt: Some(pb::generate_request::Prompt::Text("x".to_string())),
|
||||
stopping: Some(pb::StoppingCriteria {
|
||||
max_new_tokens: 10,
|
||||
..Default::default()
|
||||
}),
|
||||
..Default::default()
|
||||
})
|
||||
.await
|
||||
.expect("unary generate")
|
||||
.into_inner();
|
||||
|
||||
let outputs = response.outputs.expect("outputs present");
|
||||
assert_eq!(outputs.text, "hi");
|
||||
|
||||
engine_task.await.expect("mock engine task");
|
||||
server_task.abort();
|
||||
}
|
||||
@@ -0,0 +1,237 @@
|
||||
//! Minimal OpenAI-compatible HTTP server above [`vllm_chat`].
|
||||
|
||||
mod config;
|
||||
mod error;
|
||||
mod grpc;
|
||||
mod listener;
|
||||
mod middleware;
|
||||
mod routes;
|
||||
mod state;
|
||||
mod utils;
|
||||
|
||||
use std::sync::{Arc, OnceLock};
|
||||
|
||||
use anyhow::{Context as _, Result};
|
||||
use axum::serve::ListenerExt as _;
|
||||
pub use config::{Config, CoordinatorMode, HttpListenerMode};
|
||||
use tokio::net::TcpListener;
|
||||
use tokio::time::{Instant, sleep_until};
|
||||
use tokio_stream::wrappers::TcpListenerStream;
|
||||
use tokio_util::either::Either;
|
||||
use tokio_util::sync::CancellationToken;
|
||||
use tonic::transport::Server as TonicServer;
|
||||
use tracing::{info, trace, warn};
|
||||
use vllm_chat::{ChatLlm, LoadModelBackendsOptions, load_model_backends};
|
||||
pub use vllm_chat::{ChatTemplateContentFormatOption, ParserSelection, RendererSelection};
|
||||
use vllm_engine_core_client::{EngineCoreClient, EngineCoreClientConfig};
|
||||
use vllm_llm::Llm;
|
||||
use vllm_text::TextLlm;
|
||||
|
||||
use crate::listener::Listener;
|
||||
use crate::routes::build_router;
|
||||
use crate::state::AppState;
|
||||
|
||||
/// Build the shared application state for one configured model and one engine
|
||||
/// client.
|
||||
async fn build_state(config: &Config) -> Result<Arc<AppState>> {
|
||||
// Load both backends from the same model metadata so they stay in sync.
|
||||
let loaded = load_model_backends(
|
||||
&config.model,
|
||||
LoadModelBackendsOptions {
|
||||
renderer: config.renderer,
|
||||
chat_template: config.chat_template.clone(),
|
||||
chat_template_content_format: config.chat_template_content_format,
|
||||
default_chat_template_kwargs: config
|
||||
.default_chat_template_kwargs
|
||||
.clone()
|
||||
.unwrap_or_default(),
|
||||
},
|
||||
)
|
||||
.await
|
||||
.context("failed to create chat/text backends")?;
|
||||
let text_backend = loaded.text_backend;
|
||||
let chat_backend = loaded.chat_backend;
|
||||
|
||||
let coordinator_mode = config.effective_coordinator_mode(text_backend.is_moe());
|
||||
info!(
|
||||
engine_count = config.engine_count(),
|
||||
model_is_moe = text_backend.is_moe(),
|
||||
?coordinator_mode,
|
||||
"resolved coordinator mode"
|
||||
);
|
||||
|
||||
let client = EngineCoreClient::connect(EngineCoreClientConfig {
|
||||
transport_mode: config.transport_mode.clone(),
|
||||
coordinator_mode,
|
||||
model_name: config.model.clone(),
|
||||
client_index: 0,
|
||||
})
|
||||
.await
|
||||
.context("failed to connect to engine core")?;
|
||||
|
||||
let llm = Llm::new(client).with_log_stats(!config.disable_log_stats);
|
||||
let text = TextLlm::new(llm, text_backend);
|
||||
|
||||
let chat = ChatLlm::new(text, chat_backend)
|
||||
.with_tool_call_parser(config.tool_call_parser.clone())
|
||||
.with_reasoning_parser(config.reasoning_parser.clone());
|
||||
|
||||
// If no served names are specified, fall back to the backend model path so
|
||||
// that the API always has at least one valid model ID.
|
||||
let served_model_names = if config.served_model_name.is_empty() {
|
||||
vec![config.model.clone()]
|
||||
} else {
|
||||
config.served_model_name.clone()
|
||||
};
|
||||
|
||||
Ok(Arc::new(
|
||||
AppState::new(served_model_names, chat).with_log_requests(config.enable_log_requests),
|
||||
))
|
||||
}
|
||||
|
||||
/// Run the OpenAI-compatible HTTP server until the supplied shutdown token is
|
||||
/// cancelled.
|
||||
///
|
||||
/// The server owns one `vllm-chat` facade, which in turn owns the lower
|
||||
/// `vllm-text` and `vllm-llm` layers, and shuts them down before returning.
|
||||
pub async fn serve(config: Config, shutdown: CancellationToken) -> Result<()> {
|
||||
config.validate().context("invalid OpenAI frontend configuration")?;
|
||||
|
||||
// Also check shutdown during the (potentially long) startup handshake.
|
||||
let state = tokio::select! {
|
||||
result = build_state(&config) => result?,
|
||||
_ = shutdown.cancelled() => return Ok(()),
|
||||
};
|
||||
let listener = Listener::bind(&config.listener_mode)
|
||||
.await
|
||||
.context("failed to bind listener for OpenAI server")?;
|
||||
let bind_address = listener.local_addr()?;
|
||||
let model = state.primary_model_name().to_owned();
|
||||
let app = build_router(state.clone());
|
||||
|
||||
// Optionally bind the gRPC Generate server on a separate port. Bind
|
||||
// synchronously here so bind errors (port in use, permission denied, ...)
|
||||
// surface before we start serving, rather than being deferred until
|
||||
// shutdown. The gRPC listener follows the same host as the HTTP listener so
|
||||
// that enabling --grpc-port does not accidentally expose the service on all
|
||||
// interfaces when HTTP is intentionally local-only.
|
||||
let grpc_setup = if let Some(grpc_port) = config.grpc_port {
|
||||
let grpc_host = match &config.listener_mode {
|
||||
HttpListenerMode::BindTcp { host, .. } => host.as_str(),
|
||||
HttpListenerMode::BindUnix { .. } | HttpListenerMode::InheritedFd { .. } => "0.0.0.0",
|
||||
};
|
||||
let grpc_listener = TcpListener::bind((grpc_host, grpc_port))
|
||||
.await
|
||||
.with_context(|| format!("failed to bind gRPC listener on {grpc_host}:{grpc_port}"))?;
|
||||
let addr = grpc_listener.local_addr()?;
|
||||
let svc = grpc::GenerateServer::new(grpc::GenerateServiceImpl::new(state.clone()));
|
||||
info!(%addr, "starting gRPC server");
|
||||
Some((grpc_listener, svc))
|
||||
} else {
|
||||
None
|
||||
};
|
||||
|
||||
info!(%bind_address, %model, "starting OpenAI server");
|
||||
|
||||
// Set TCP_NODELAY on accepted connections to reduce latency.
|
||||
// By `tap_io` we will do this on every accepted connection.
|
||||
let listener = listener.tap_io(|io| {
|
||||
if let Either::Left(tcp_stream) = io
|
||||
&& let Err(err) = tcp_stream.set_nodelay(true)
|
||||
{
|
||||
trace!(error = %err, "failed to enable TCP_NODELAY on accepted HTTP connection");
|
||||
}
|
||||
});
|
||||
|
||||
// Run HTTP and gRPC concurrently under a child token of the caller's shutdown
|
||||
// token. Caller cancellation propagates into both protocols; if either
|
||||
// protocol exits first, we cancel this child token so its sibling also
|
||||
// begins a graceful drain.
|
||||
let server_shutdown = shutdown.child_token();
|
||||
let force_shutdown = CancellationToken::new();
|
||||
let shutdown_deadline = Arc::new(OnceLock::new());
|
||||
|
||||
// Spawn a task to trigger `force_shutdown` after shutdown deadline elapses.
|
||||
tokio::spawn({
|
||||
let shutdown = server_shutdown.clone();
|
||||
let force_shutdown = force_shutdown.clone();
|
||||
let shutdown_deadline = shutdown_deadline.clone();
|
||||
let shutdown_timeout = config.shutdown_timeout;
|
||||
|
||||
async move {
|
||||
shutdown.cancelled().await;
|
||||
let deadline = Instant::now() + shutdown_timeout;
|
||||
let _ = shutdown_deadline.set(deadline);
|
||||
|
||||
if shutdown_timeout.is_zero() {
|
||||
force_shutdown.cancel();
|
||||
} else {
|
||||
sleep_until(deadline).await;
|
||||
force_shutdown.cancel();
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
let http_fut = {
|
||||
let shutdown = server_shutdown.child_token();
|
||||
let server_shutdown = server_shutdown.clone();
|
||||
let force_shutdown = force_shutdown.clone();
|
||||
async move {
|
||||
let server =
|
||||
axum::serve(listener, app).with_graceful_shutdown(shutdown.cancelled_owned());
|
||||
|
||||
let result = tokio::select! {
|
||||
result = server => {
|
||||
result.context("HTTP server failed")
|
||||
}
|
||||
_ = force_shutdown.cancelled() => {
|
||||
warn!("HTTP graceful shutdown deadline elapsed; aborting server");
|
||||
Ok(())
|
||||
}
|
||||
};
|
||||
|
||||
server_shutdown.cancel();
|
||||
result
|
||||
}
|
||||
};
|
||||
|
||||
let grpc_fut = {
|
||||
let shutdown = server_shutdown.child_token();
|
||||
let server_shutdown = server_shutdown.clone();
|
||||
let force_shutdown = force_shutdown.clone();
|
||||
async move {
|
||||
let Some((grpc_listener, svc)) = grpc_setup else {
|
||||
// No gRPC configured: just wait for shutdown so we do not race the
|
||||
// join! by resolving early and tripping the cancellation token.
|
||||
shutdown.cancelled().await;
|
||||
return Ok(());
|
||||
};
|
||||
let server = TonicServer::builder().add_service(svc).serve_with_incoming_shutdown(
|
||||
TcpListenerStream::new(grpc_listener),
|
||||
shutdown.cancelled_owned(),
|
||||
);
|
||||
|
||||
let result = tokio::select! {
|
||||
result = server => {
|
||||
result.context("gRPC server failed")
|
||||
}
|
||||
_ = force_shutdown.cancelled() => {
|
||||
warn!("gRPC graceful shutdown deadline elapsed; aborting server");
|
||||
Ok(())
|
||||
}
|
||||
};
|
||||
|
||||
server_shutdown.cancel();
|
||||
result
|
||||
}
|
||||
};
|
||||
|
||||
let (http_res, grpc_res) = tokio::join!(http_fut, grpc_fut);
|
||||
http_res.and(grpc_res)?;
|
||||
|
||||
let shutdown_deadline = shutdown_deadline
|
||||
.get()
|
||||
.copied()
|
||||
.unwrap_or_else(|| Instant::now() + config.shutdown_timeout);
|
||||
state.shutdown(shutdown_deadline).await
|
||||
}
|
||||
@@ -0,0 +1,135 @@
|
||||
//! Unified HTTP listener wrapper for the Rust frontend.
|
||||
//!
|
||||
//! This module hides the difference between TCP and Unix-domain listeners so
|
||||
//! the rest of the server can bind or inherit one socket and pass it to
|
||||
//! `axum::serve(...)` through a single type.
|
||||
|
||||
use std::io::Result;
|
||||
use std::net::TcpListener as StdTcpListener;
|
||||
use std::os::fd::{FromRawFd, IntoRawFd, OwnedFd};
|
||||
use std::os::unix::net::UnixListener as StdUnixListener;
|
||||
|
||||
use socket2::Socket;
|
||||
use tokio::net::{TcpListener, TcpStream, UnixListener, UnixStream};
|
||||
use tokio_util::either::Either;
|
||||
|
||||
use crate::HttpListenerMode;
|
||||
|
||||
/// Runtime listener type used by the OpenAI-compatible HTTP server, which is
|
||||
/// either a TCP listener or a Unix-domain listener.
|
||||
#[derive(Debug)]
|
||||
pub enum Listener {
|
||||
Tcp(TcpListener),
|
||||
Unix(UnixListener),
|
||||
}
|
||||
|
||||
impl Listener {
|
||||
/// Bind or adopt the listener described by the frontend configuration.
|
||||
///
|
||||
/// For inherited sockets, the concrete listener kind is detected from the
|
||||
/// socket family of the supplied file descriptor.
|
||||
pub async fn bind(mode: &HttpListenerMode) -> Result<Self> {
|
||||
match mode {
|
||||
HttpListenerMode::BindTcp { host, port } => {
|
||||
Ok(Self::Tcp(TcpListener::bind((host.as_str(), *port)).await?))
|
||||
}
|
||||
HttpListenerMode::BindUnix { path } => Ok(Self::Unix(UnixListener::bind(path)?)),
|
||||
HttpListenerMode::InheritedFd { fd } => Self::from_inherited_fd(*fd),
|
||||
}
|
||||
}
|
||||
|
||||
/// Return a log-friendly local address string for either TCP or Unix
|
||||
/// sockets.
|
||||
pub fn local_addr(&self) -> Result<String> {
|
||||
match self {
|
||||
Self::Tcp(listener) => Ok(listener.local_addr()?.to_string()),
|
||||
Self::Unix(listener) => Ok(match listener.local_addr()?.as_pathname() {
|
||||
Some(path) => format!("unix:{}", path.display()),
|
||||
None => "unix:<unnamed>".to_string(),
|
||||
}),
|
||||
}
|
||||
}
|
||||
|
||||
fn from_inherited_fd(fd: i32) -> Result<Self> {
|
||||
// SAFETY: We trust the caller to only pass valid listener fds, and we only use
|
||||
// this fd once to create a single listener.
|
||||
let owned_fd = unsafe { OwnedFd::from_raw_fd(fd) };
|
||||
let socket = Socket::from(owned_fd);
|
||||
|
||||
// The Python supervisor pre-binds the socket to reserve the endpoint early, but
|
||||
// Rust is responsible for transitioning inherited stream sockets into
|
||||
// the listening state before accepting connections.
|
||||
socket.listen(libc::SOMAXCONN)?;
|
||||
socket.set_nonblocking(true)?;
|
||||
|
||||
if socket.local_addr()?.is_unix() {
|
||||
let std_listener = unsafe { StdUnixListener::from_raw_fd(socket.into_raw_fd()) };
|
||||
Ok(Self::Unix(UnixListener::from_std(std_listener)?))
|
||||
} else {
|
||||
let std_listener = unsafe { StdTcpListener::from_raw_fd(socket.into_raw_fd()) };
|
||||
Ok(Self::Tcp(TcpListener::from_std(std_listener)?))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Allow the unified listener to plug directly into `axum::serve(...)`.
|
||||
impl axum::serve::Listener for Listener {
|
||||
type Addr = Either<std::net::SocketAddr, tokio::net::unix::SocketAddr>;
|
||||
type Io = Either<TcpStream, UnixStream>;
|
||||
|
||||
async fn accept(&mut self) -> (Self::Io, Self::Addr) {
|
||||
match self {
|
||||
Self::Tcp(listener) => {
|
||||
let (io, addr) = listener.accept().await;
|
||||
(Either::Left(io), Either::Left(addr))
|
||||
}
|
||||
Self::Unix(listener) => {
|
||||
let (io, addr) = listener.accept().await;
|
||||
(Either::Right(io), Either::Right(addr))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn local_addr(&self) -> Result<Self::Addr> {
|
||||
match self {
|
||||
Self::Tcp(listener) => listener.local_addr().map(Either::Left),
|
||||
Self::Unix(listener) => listener.local_addr().map(Either::Right),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use std::net::{Ipv4Addr, SocketAddrV4};
|
||||
use std::os::fd::IntoRawFd;
|
||||
|
||||
use socket2::{Domain, SockAddr, Socket, Type};
|
||||
use uuid::Uuid;
|
||||
|
||||
use super::Listener;
|
||||
use crate::HttpListenerMode;
|
||||
|
||||
#[tokio::test(flavor = "current_thread")]
|
||||
async fn inherited_fd_detects_tcp_listener_without_uds_hint() {
|
||||
let socket = Socket::new(Domain::IPV4, Type::STREAM, None).unwrap();
|
||||
socket.bind(&SockAddr::from(SocketAddrV4::new(Ipv4Addr::LOCALHOST, 0))).unwrap();
|
||||
let fd = socket.into_raw_fd();
|
||||
|
||||
let listener = Listener::bind(&HttpListenerMode::InheritedFd { fd }).await.unwrap();
|
||||
|
||||
assert!(matches!(listener, Listener::Tcp(_)));
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "current_thread")]
|
||||
async fn inherited_fd_detects_unix_listener_from_fd() {
|
||||
let path = std::env::temp_dir().join(format!("vllm-rs-{}.sock", Uuid::new_v4()));
|
||||
let socket = Socket::new(Domain::UNIX, Type::STREAM, None).unwrap();
|
||||
socket.bind(&SockAddr::unix(&path).unwrap()).unwrap();
|
||||
let fd = socket.into_raw_fd();
|
||||
|
||||
let listener = Listener::bind(&HttpListenerMode::InheritedFd { fd }).await.unwrap();
|
||||
|
||||
assert!(matches!(listener, Listener::Unix(_)));
|
||||
let _ = std::fs::remove_file(path);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,110 @@
|
||||
use std::pin::Pin;
|
||||
use std::sync::{Arc, Weak};
|
||||
use std::task::{Context, Poll};
|
||||
|
||||
use axum::body::{Body, Bytes, HttpBody};
|
||||
use axum::extract::{MatchedPath, Request, State};
|
||||
use axum::middleware::Next;
|
||||
use axum::response::Response;
|
||||
use http_body::{Frame, SizeHint};
|
||||
|
||||
use crate::state::AppState;
|
||||
|
||||
/// Endpoints that will be tracked for server load.
|
||||
///
|
||||
/// Derived from the Python frontend's actual `@load_aware_call` coverage. This
|
||||
/// includes alias paths that delegate into decorated handlers, such as
|
||||
/// `/v1/rerank` and `/v2/rerank`.
|
||||
const TRACKED_HANDLERS: &[&str] = &[
|
||||
"/v1/responses",
|
||||
"/v1/responses/{response_id}",
|
||||
"/v1/responses/{response_id}/cancel",
|
||||
"/v1/messages",
|
||||
"/v1/messages/count_tokens",
|
||||
"/v1/chat/completions",
|
||||
"/v1/completions",
|
||||
"/v1/audio/transcriptions",
|
||||
"/v1/audio/translations",
|
||||
"/v1/embeddings",
|
||||
"/pooling",
|
||||
"/classify",
|
||||
"/score",
|
||||
"/v1/score",
|
||||
"/rerank",
|
||||
"/v1/rerank",
|
||||
"/v2/rerank",
|
||||
"/inference/v1/generate",
|
||||
];
|
||||
|
||||
/// Track frontend-local in-flight inference requests for the `/load` endpoint.
|
||||
pub async fn track_server_load(
|
||||
State(state): State<Arc<AppState>>,
|
||||
req: Request,
|
||||
next: Next,
|
||||
) -> Response {
|
||||
let handler = req
|
||||
.extensions()
|
||||
.get::<MatchedPath>()
|
||||
.map_or_else(|| "none", |path| path.as_str());
|
||||
|
||||
if !TRACKED_HANDLERS.contains(&handler) {
|
||||
return next.run(req).await;
|
||||
}
|
||||
|
||||
state.increment_server_load();
|
||||
let guard = ServerLoadGuard {
|
||||
state: Arc::downgrade(&state),
|
||||
};
|
||||
let response = next.run(req).await;
|
||||
|
||||
let (parts, body) = response.into_parts();
|
||||
Response::from_parts(
|
||||
parts,
|
||||
Body::new(LoadTrackedBody {
|
||||
inner: body,
|
||||
_guard: guard,
|
||||
}),
|
||||
)
|
||||
}
|
||||
|
||||
/// A guard that decrements the server load when dropped.
|
||||
struct ServerLoadGuard {
|
||||
state: Weak<AppState>,
|
||||
}
|
||||
|
||||
impl Drop for ServerLoadGuard {
|
||||
fn drop(&mut self) {
|
||||
if let Some(state) = self.state.upgrade() {
|
||||
state.decrement_server_load();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// A wrapper around response bodies that tracks server load by holding a
|
||||
/// `ServerLoadGuard`, which will decrement the load when the body is fully
|
||||
/// consumed and dropped.
|
||||
struct LoadTrackedBody {
|
||||
inner: Body,
|
||||
_guard: ServerLoadGuard,
|
||||
}
|
||||
|
||||
// Simply delegate all `HttpBody` methods to the inner body.
|
||||
impl HttpBody for LoadTrackedBody {
|
||||
type Data = Bytes;
|
||||
type Error = axum::Error;
|
||||
|
||||
fn poll_frame(
|
||||
mut self: Pin<&mut Self>,
|
||||
cx: &mut Context<'_>,
|
||||
) -> Poll<Option<Result<Frame<Self::Data>, Self::Error>>> {
|
||||
Pin::new(&mut self.inner).poll_frame(cx)
|
||||
}
|
||||
|
||||
fn is_end_stream(&self) -> bool {
|
||||
self.inner.is_end_stream()
|
||||
}
|
||||
|
||||
fn size_hint(&self) -> SizeHint {
|
||||
self.inner.size_hint()
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,79 @@
|
||||
use std::time::Instant;
|
||||
|
||||
use axum::extract::{MatchedPath, Request};
|
||||
use axum::middleware::Next;
|
||||
use axum::response::Response;
|
||||
use vllm_metrics::{HttpHandlerLabels, HttpRequestLabels, METRICS};
|
||||
|
||||
/// Endpoints that will be excluded from HTTP metrics tracking.
|
||||
///
|
||||
/// Original Python definition:
|
||||
/// <https://github.com/vllm-project/vllm/blob/bc2c0c86efb28e77677a3cfb8687e976914a313a/vllm/entrypoints/serve/instrumentator/metrics.py#L28-L38>
|
||||
const EXCLUDED_HANDLERS: &[&str] = &[
|
||||
"/metrics",
|
||||
"/health",
|
||||
"/load",
|
||||
"/ping",
|
||||
"/version",
|
||||
"/server_info",
|
||||
// Rust frontend extra:
|
||||
"/reset_prefix_cache",
|
||||
"/reset_mm_cache",
|
||||
"/reset_encoder_cache",
|
||||
"/collective_rpc",
|
||||
"/sleep",
|
||||
"/wake_up",
|
||||
"/is_sleeping",
|
||||
];
|
||||
|
||||
/// Record API-server HTTP metrics with Python-compatible
|
||||
/// (`PrometheusFastApiInstrumentator` style) family names and labels.
|
||||
pub async fn track_http_metrics(req: Request, next: Next) -> Response {
|
||||
let method = req.method().as_str().to_string();
|
||||
let handler = req
|
||||
.extensions()
|
||||
.get::<MatchedPath>()
|
||||
.map_or_else(|| "none".to_string(), |path| path.as_str().to_string());
|
||||
let excluded = EXCLUDED_HANDLERS.contains(&handler.as_str());
|
||||
let started_at = Instant::now();
|
||||
|
||||
let response = next.run(req).await;
|
||||
|
||||
if excluded {
|
||||
return response;
|
||||
}
|
||||
|
||||
let elapsed = started_at.elapsed().as_secs_f64();
|
||||
let status = status_group(response.status().as_u16());
|
||||
|
||||
let metrics = &METRICS.api_server;
|
||||
|
||||
metrics
|
||||
.http_requests
|
||||
.get_or_create(&HttpRequestLabels {
|
||||
method: method.clone(),
|
||||
status,
|
||||
handler: handler.clone(),
|
||||
})
|
||||
.inc();
|
||||
|
||||
metrics
|
||||
.http_request_duration_seconds
|
||||
.get_or_create(&HttpHandlerLabels { method, handler })
|
||||
.observe(elapsed);
|
||||
|
||||
metrics.http_request_duration_highr_seconds.observe(elapsed);
|
||||
|
||||
response
|
||||
}
|
||||
|
||||
fn status_group(status: u16) -> &'static str {
|
||||
match status / 100 {
|
||||
1 => "1xx",
|
||||
2 => "2xx",
|
||||
3 => "3xx",
|
||||
4 => "4xx",
|
||||
5 => "5xx",
|
||||
_ => "unknown",
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,5 @@
|
||||
mod load;
|
||||
mod metrics;
|
||||
|
||||
pub use load::track_server_load;
|
||||
pub use metrics::track_http_metrics;
|
||||
@@ -0,0 +1,68 @@
|
||||
mod cache;
|
||||
mod collective_rpc;
|
||||
mod health;
|
||||
mod inference;
|
||||
mod load;
|
||||
mod metrics;
|
||||
pub(crate) mod openai;
|
||||
mod sleep;
|
||||
|
||||
use std::sync::Arc;
|
||||
|
||||
use axum::Router;
|
||||
use axum::middleware::{from_fn, from_fn_with_state};
|
||||
use axum::routing::{get, post};
|
||||
use tower_http::trace::TraceLayer;
|
||||
|
||||
use crate::middleware;
|
||||
use crate::state::AppState;
|
||||
|
||||
fn server_dev_mode_enabled() -> bool {
|
||||
std::env::var("VLLM_SERVER_DEV_MODE")
|
||||
.ok()
|
||||
.and_then(|value| value.parse::<i64>().ok())
|
||||
.is_some_and(|value| value != 0)
|
||||
}
|
||||
|
||||
/// Build the minimal OpenAI-compatible router for one configured model.
|
||||
pub fn build_router(state: Arc<AppState>) -> Router {
|
||||
build_router_with_dev_mode(state, server_dev_mode_enabled())
|
||||
}
|
||||
|
||||
fn build_router_with_dev_mode(state: Arc<AppState>, dev_mode_enabled: bool) -> Router {
|
||||
let mut router = Router::new()
|
||||
// Health & monitoring
|
||||
.route("/health", get(health::health))
|
||||
.route("/metrics", get(metrics::scrape))
|
||||
.route("/load", get(load::load))
|
||||
// OpenAI-compatible endpoints
|
||||
.route("/v1/models", get(openai::list_models))
|
||||
.route("/v1/completions", post(openai::completions))
|
||||
.route("/v1/chat/completions", post(openai::chat_completions))
|
||||
// vLLM specific inference endpoints
|
||||
.route("/inference/v1/generate", post(inference::generate));
|
||||
|
||||
if dev_mode_enabled {
|
||||
// Development-only
|
||||
router = router
|
||||
.route("/reset_prefix_cache", post(cache::reset_prefix_cache))
|
||||
.route("/reset_mm_cache", post(cache::reset_mm_cache))
|
||||
.route("/reset_encoder_cache", post(cache::reset_encoder_cache))
|
||||
.route("/collective_rpc", post(collective_rpc::collective_rpc))
|
||||
.route("/sleep", post(sleep::sleep))
|
||||
.route("/wake_up", post(sleep::wake_up))
|
||||
.route("/is_sleeping", get(sleep::is_sleeping))
|
||||
}
|
||||
|
||||
router
|
||||
.with_state(state.clone())
|
||||
.layer(from_fn_with_state(state, middleware::track_server_load))
|
||||
.layer(from_fn(middleware::track_http_metrics))
|
||||
.layer(TraceLayer::new_for_http())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests;
|
||||
|
||||
#[cfg(test)]
|
||||
mod http_client_tests;
|
||||
@@ -0,0 +1,56 @@
|
||||
use std::sync::Arc;
|
||||
|
||||
use axum::extract::{Query, State};
|
||||
use axum::http::StatusCode;
|
||||
use serde::Deserialize;
|
||||
|
||||
use crate::error::ApiError;
|
||||
use crate::state::AppState;
|
||||
use crate::utils::utility_call_error;
|
||||
|
||||
#[derive(Debug, Default, Deserialize)]
|
||||
pub(crate) struct ResetPrefixCacheParams {
|
||||
#[serde(default)]
|
||||
reset_running_requests: bool,
|
||||
#[serde(default)]
|
||||
reset_external: bool,
|
||||
}
|
||||
|
||||
/// Reset the local prefix cache and optionally the connector-managed external
|
||||
/// cache.
|
||||
pub async fn reset_prefix_cache(
|
||||
State(state): State<Arc<AppState>>,
|
||||
Query(params): Query<ResetPrefixCacheParams>,
|
||||
) -> Result<StatusCode, ApiError> {
|
||||
state
|
||||
.engine_core_client()
|
||||
.reset_prefix_cache(params.reset_running_requests, params.reset_external)
|
||||
.await
|
||||
.map_err(|error| utility_call_error("reset_prefix_cache", error))?;
|
||||
|
||||
Ok(StatusCode::OK)
|
||||
}
|
||||
|
||||
/// Reset the multi-modal cache.
|
||||
pub async fn reset_mm_cache(State(state): State<Arc<AppState>>) -> Result<StatusCode, ApiError> {
|
||||
state
|
||||
.engine_core_client()
|
||||
.reset_mm_cache()
|
||||
.await
|
||||
.map_err(|error| utility_call_error("reset_mm_cache", error))?;
|
||||
|
||||
Ok(StatusCode::OK)
|
||||
}
|
||||
|
||||
/// Reset the encoder cache.
|
||||
pub async fn reset_encoder_cache(
|
||||
State(state): State<Arc<AppState>>,
|
||||
) -> Result<StatusCode, ApiError> {
|
||||
state
|
||||
.engine_core_client()
|
||||
.reset_encoder_cache()
|
||||
.await
|
||||
.map_err(|error| utility_call_error("reset_encoder_cache", error))?;
|
||||
|
||||
Ok(StatusCode::OK)
|
||||
}
|
||||
@@ -0,0 +1,51 @@
|
||||
use std::collections::BTreeMap;
|
||||
use std::sync::Arc;
|
||||
|
||||
use axum::Json;
|
||||
use axum::extract::State;
|
||||
use axum::extract::rejection::JsonRejection;
|
||||
use rmpv::Value as MsgpackValue;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use serde_json::Value as JsonValue;
|
||||
|
||||
use crate::error::ApiError;
|
||||
use crate::state::AppState;
|
||||
use crate::utils::utility_call_error;
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
pub(crate) struct CollectiveRpcRequest {
|
||||
method: Option<String>,
|
||||
#[serde(default)]
|
||||
timeout: Option<f64>,
|
||||
#[serde(default)]
|
||||
args: Vec<JsonValue>,
|
||||
#[serde(default)]
|
||||
kwargs: BTreeMap<String, JsonValue>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize)]
|
||||
pub(crate) struct CollectiveRpcResponse {
|
||||
results: Vec<MsgpackValue>,
|
||||
}
|
||||
|
||||
/// Execute a development-only collective RPC on the connected engine(s).
|
||||
pub async fn collective_rpc(
|
||||
State(state): State<Arc<AppState>>,
|
||||
body: Result<Json<CollectiveRpcRequest>, JsonRejection>,
|
||||
) -> Result<Json<CollectiveRpcResponse>, ApiError> {
|
||||
let Json(body) = body.map_err(|error| ApiError::json_parse_error(error.body_text()))?;
|
||||
let method = body.method.ok_or_else(|| {
|
||||
ApiError::invalid_request(
|
||||
"Missing 'method' in request body".to_string(),
|
||||
Some("method"),
|
||||
)
|
||||
})?;
|
||||
|
||||
let results = state
|
||||
.engine_core_client()
|
||||
.collective_rpc(&method, body.timeout, body.args, body.kwargs)
|
||||
.await
|
||||
.map_err(|error| utility_call_error("collective_rpc", error))?;
|
||||
|
||||
Ok(Json(CollectiveRpcResponse { results }))
|
||||
}
|
||||
@@ -0,0 +1,14 @@
|
||||
use std::sync::Arc;
|
||||
|
||||
use axum::extract::State;
|
||||
use axum::http::StatusCode;
|
||||
|
||||
use crate::state::AppState;
|
||||
|
||||
pub async fn health(State(state): State<Arc<AppState>>) -> StatusCode {
|
||||
if state.chat.engine_core_client().is_healthy() {
|
||||
StatusCode::OK
|
||||
} else {
|
||||
StatusCode::SERVICE_UNAVAILABLE
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,392 @@
|
||||
//! Integration tests that exercise the OpenAI-compatible HTTP API through a
|
||||
//! real TCP connection using the `async-openai` client library, backed by a
|
||||
//! mock engine.
|
||||
|
||||
use std::future::Future;
|
||||
use std::pin::Pin;
|
||||
use std::sync::Arc;
|
||||
use std::task::{Context, Poll};
|
||||
|
||||
use async_openai::Client;
|
||||
use async_openai::config::OpenAIConfig;
|
||||
use async_openai::types::chat::{
|
||||
ChatCompletionRequestUserMessageArgs, CreateChatCompletionRequestArgs,
|
||||
};
|
||||
use futures::StreamExt as _;
|
||||
use serial_test::serial;
|
||||
use vllm_chat::{
|
||||
ChatBackend, ChatLlm, ChatRenderer, ChatRequest, ChatTextBackend, DefaultChatOutputProcessor,
|
||||
DynChatOutputProcessor, DynChatRenderer, NewChatOutputProcessorOptions, RenderedPrompt,
|
||||
};
|
||||
use vllm_engine_core_client::protocol::{
|
||||
EngineCoreFinishReason, EngineCoreOutput, EngineCoreOutputs, EngineCoreRequest,
|
||||
};
|
||||
use vllm_engine_core_client::test_utils::{IpcNamespace, spawn_mock_engine_task};
|
||||
use vllm_engine_core_client::{EngineCoreClient, EngineCoreClientConfig, EngineId};
|
||||
use vllm_llm::Llm;
|
||||
use vllm_text::tokenizer::{DynTokenizer, Tokenizer};
|
||||
use vllm_text::{Prompt, TextBackend};
|
||||
use zeromq::prelude::{SocketRecv, SocketSend};
|
||||
use zeromq::{DealerSocket, PushSocket, ZmqMessage};
|
||||
|
||||
use crate::routes::build_router;
|
||||
use crate::state::AppState;
|
||||
|
||||
// ========================================================================================
|
||||
// Test infrastructure (mirrors routes/tests.rs helpers)
|
||||
// ========================================================================================
|
||||
|
||||
type TestFuture<'a> = Pin<Box<dyn Future<Output = ()> + Send + 'a>>;
|
||||
|
||||
fn boxed_test_future<'a>(future: impl Future<Output = ()> + Send + 'a) -> TestFuture<'a> {
|
||||
Box::pin(future)
|
||||
}
|
||||
|
||||
struct MockEngineTask {
|
||||
shutdown_tx: Option<tokio::sync::oneshot::Sender<()>>,
|
||||
join_handle: Option<tokio::task::JoinHandle<()>>,
|
||||
}
|
||||
|
||||
impl MockEngineTask {
|
||||
fn new(
|
||||
(shutdown_tx, join_handle): (
|
||||
tokio::sync::oneshot::Sender<()>,
|
||||
tokio::task::JoinHandle<()>,
|
||||
),
|
||||
) -> Self {
|
||||
Self {
|
||||
shutdown_tx: Some(shutdown_tx),
|
||||
join_handle: Some(join_handle),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Future for MockEngineTask {
|
||||
type Output = Result<(), tokio::task::JoinError>;
|
||||
|
||||
fn poll(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Self::Output> {
|
||||
if let Some(shutdown_tx) = self.shutdown_tx.take() {
|
||||
let _ = shutdown_tx.send(());
|
||||
}
|
||||
match self.join_handle.as_mut() {
|
||||
Some(join_handle) => Pin::new(join_handle).poll(cx),
|
||||
None => Poll::Ready(Ok(())),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for MockEngineTask {
|
||||
fn drop(&mut self) {
|
||||
if let Some(join_handle) = &self.join_handle {
|
||||
join_handle.abort();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn request_output(
|
||||
request_id: &str,
|
||||
new_token_ids: Vec<u32>,
|
||||
finish_reason: Option<EngineCoreFinishReason>,
|
||||
) -> EngineCoreOutput {
|
||||
EngineCoreOutput {
|
||||
request_id: request_id.to_string(),
|
||||
new_token_ids,
|
||||
new_logprobs: None,
|
||||
new_prompt_logprobs_tensors: None,
|
||||
pooling_output: None,
|
||||
finish_reason,
|
||||
stop_reason: None,
|
||||
events: None,
|
||||
kv_transfer_params: None,
|
||||
trace_headers: None,
|
||||
prefill_stats: None,
|
||||
routed_experts: None,
|
||||
num_nans_in_logits: 0,
|
||||
}
|
||||
}
|
||||
|
||||
fn engine_outputs_for_request(
|
||||
request_id: &str,
|
||||
output_specs: Vec<(Vec<u32>, Option<EngineCoreFinishReason>)>,
|
||||
) -> EngineCoreOutputs {
|
||||
EngineCoreOutputs {
|
||||
engine_index: 0,
|
||||
outputs: output_specs
|
||||
.into_iter()
|
||||
.map(|(token_ids, finish_reason)| request_output(request_id, token_ids, finish_reason))
|
||||
.collect(),
|
||||
scheduler_stats: None,
|
||||
timestamp: 0.0,
|
||||
utility_output: None,
|
||||
finished_requests: None,
|
||||
wave_complete: None,
|
||||
start_wave: None,
|
||||
}
|
||||
}
|
||||
|
||||
fn default_stream_output_specs() -> Vec<(Vec<u32>, Option<EngineCoreFinishReason>)> {
|
||||
vec![
|
||||
(vec![b'h' as u32], None),
|
||||
(vec![b'i' as u32], None),
|
||||
(vec![b'!' as u32], Some(EngineCoreFinishReason::Stop)),
|
||||
]
|
||||
}
|
||||
|
||||
async fn send_outputs(push: &mut PushSocket, outputs: EngineCoreOutputs) {
|
||||
push.send(ZmqMessage::from(
|
||||
rmp_serde::to_vec_named(&outputs).expect("encode outputs"),
|
||||
))
|
||||
.await
|
||||
.expect("send outputs");
|
||||
}
|
||||
|
||||
async fn recv_engine_message(dealer: &mut DealerSocket) -> Vec<bytes::Bytes> {
|
||||
dealer.recv().await.expect("recv engine message").into_vec()
|
||||
}
|
||||
|
||||
fn test_llm(client: EngineCoreClient) -> Llm {
|
||||
Llm::new(client).with_request_id_randomization(false)
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
struct FakeChatBackend;
|
||||
|
||||
#[derive(Debug)]
|
||||
struct FakeChatTokenizer;
|
||||
|
||||
impl Tokenizer for FakeChatTokenizer {
|
||||
fn encode(
|
||||
&self,
|
||||
text: &str,
|
||||
_add_special_tokens: bool,
|
||||
) -> vllm_text::tokenizer::Result<Vec<u32>> {
|
||||
Ok(text.bytes().map(u32::from).collect())
|
||||
}
|
||||
|
||||
fn decode(
|
||||
&self,
|
||||
token_ids: &[u32],
|
||||
_skip_special_tokens: bool,
|
||||
) -> vllm_text::tokenizer::Result<String> {
|
||||
Ok(
|
||||
String::from_utf8_lossy(&token_ids.iter().map(|id| *id as u8).collect::<Vec<_>>())
|
||||
.into_owned(),
|
||||
)
|
||||
}
|
||||
|
||||
fn token_to_id(&self, token: &str) -> Option<u32> {
|
||||
token.bytes().next().map(u32::from)
|
||||
}
|
||||
}
|
||||
|
||||
impl TextBackend for FakeChatBackend {
|
||||
fn tokenizer(&self) -> DynTokenizer {
|
||||
Arc::new(FakeChatTokenizer)
|
||||
}
|
||||
|
||||
fn model_id(&self) -> &str {
|
||||
"test-model"
|
||||
}
|
||||
}
|
||||
|
||||
impl ChatBackend for FakeChatBackend {
|
||||
fn chat_renderer(&self) -> DynChatRenderer {
|
||||
Arc::new(self.clone())
|
||||
}
|
||||
|
||||
fn new_chat_output_processor(
|
||||
&self,
|
||||
request: &mut ChatRequest,
|
||||
options: NewChatOutputProcessorOptions<'_>,
|
||||
) -> vllm_chat::Result<DynChatOutputProcessor> {
|
||||
Ok(Box::new(DefaultChatOutputProcessor::new(
|
||||
request,
|
||||
self.model_id(),
|
||||
self.tokenizer(),
|
||||
options.tool_call_parser,
|
||||
options.reasoning_parser,
|
||||
)?))
|
||||
}
|
||||
}
|
||||
|
||||
impl ChatRenderer for FakeChatBackend {
|
||||
fn render(&self, request: &ChatRequest) -> vllm_chat::Result<RenderedPrompt> {
|
||||
let mut prompt = String::new();
|
||||
for message in &request.messages {
|
||||
prompt.push_str(message.role().as_str());
|
||||
prompt.push_str(": ");
|
||||
prompt.push_str(&message.text_content()?);
|
||||
prompt.push('\n');
|
||||
}
|
||||
if request.chat_options.add_generation_prompt() {
|
||||
prompt.push_str("assistant:");
|
||||
}
|
||||
Ok(RenderedPrompt {
|
||||
prompt: Prompt::Text(prompt),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
/// Spin up an HTTP server on a random port backed by a mock engine.
|
||||
/// Returns the `async-openai` client, the HTTP server task, and the mock engine
|
||||
/// task.
|
||||
async fn http_test_server(
|
||||
engine_id: impl Into<EngineId>,
|
||||
output_specs: Vec<(Vec<u32>, Option<EngineCoreFinishReason>)>,
|
||||
) -> (
|
||||
Client<OpenAIConfig>,
|
||||
tokio::task::JoinHandle<()>,
|
||||
MockEngineTask,
|
||||
) {
|
||||
let ipc = IpcNamespace::new().expect("create ipc namespace");
|
||||
let handshake_address = ipc.handshake_endpoint();
|
||||
let engine_id = engine_id.into();
|
||||
|
||||
let engine_task = MockEngineTask::new(spawn_mock_engine_task(
|
||||
handshake_address.clone(),
|
||||
engine_id.clone(),
|
||||
move |dealer, push| {
|
||||
boxed_test_future(async move {
|
||||
let add = recv_engine_message(dealer).await;
|
||||
let request: EngineCoreRequest =
|
||||
rmp_serde::from_slice(&add[1]).expect("decode request");
|
||||
send_outputs(
|
||||
push,
|
||||
engine_outputs_for_request(&request.request_id, output_specs),
|
||||
)
|
||||
.await;
|
||||
})
|
||||
},
|
||||
));
|
||||
|
||||
let client = EngineCoreClient::connect(
|
||||
EngineCoreClientConfig::new_single(handshake_address)
|
||||
.with_model_name("test-model")
|
||||
.with_local_input_output_addresses(
|
||||
Some(ipc.input_endpoint()),
|
||||
Some(ipc.output_endpoint()),
|
||||
),
|
||||
)
|
||||
.await
|
||||
.expect("connect client");
|
||||
|
||||
let chat = ChatLlm::from_shared_backend(
|
||||
test_llm(client),
|
||||
Arc::new(FakeChatBackend) as Arc<dyn ChatTextBackend>,
|
||||
);
|
||||
let state = Arc::new(AppState::new(vec!["test-model".to_string()], chat));
|
||||
let app = build_router(state);
|
||||
|
||||
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.expect("bind http listener");
|
||||
let addr = listener.local_addr().expect("local addr");
|
||||
|
||||
let server_task = tokio::spawn(async move {
|
||||
axum::serve(listener, app).await.expect("http server");
|
||||
});
|
||||
|
||||
let openai_client = Client::with_config(
|
||||
OpenAIConfig::new()
|
||||
.with_api_key("unused")
|
||||
.with_api_base(format!("http://{addr}/v1")),
|
||||
);
|
||||
|
||||
(openai_client, server_task, engine_task)
|
||||
}
|
||||
|
||||
// ========================================================================================
|
||||
// Tests
|
||||
// ========================================================================================
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
#[serial]
|
||||
async fn list_models_via_http_client() {
|
||||
let (client, server_task, _engine_task) =
|
||||
http_test_server(b"engine-http-models", default_stream_output_specs()).await;
|
||||
|
||||
let models = client.models().list().await.expect("list models");
|
||||
let model_ids: Vec<&str> = models.data.iter().map(|m| m.id.as_str()).collect();
|
||||
assert_eq!(model_ids, vec!["test-model"]);
|
||||
|
||||
server_task.abort();
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
#[serial]
|
||||
async fn non_streaming_chat_via_http_client() {
|
||||
let (client, server_task, engine_task) =
|
||||
http_test_server(b"engine-http-chat", default_stream_output_specs()).await;
|
||||
|
||||
let request = CreateChatCompletionRequestArgs::default()
|
||||
.model("test-model")
|
||||
.stream(false)
|
||||
.max_completion_tokens(10u32)
|
||||
.messages([ChatCompletionRequestUserMessageArgs::default()
|
||||
.content("hello")
|
||||
.build()
|
||||
.expect("build user message")
|
||||
.into()])
|
||||
.build()
|
||||
.expect("build request");
|
||||
|
||||
let response = client.chat().create(request).await.expect("chat completion");
|
||||
|
||||
assert_eq!(response.model, "test-model");
|
||||
assert_eq!(response.choices.len(), 1);
|
||||
let choice = &response.choices[0];
|
||||
// The stop token `!` is suppressed from text.
|
||||
assert_eq!(choice.message.content.as_deref(), Some("hi"));
|
||||
assert_eq!(
|
||||
choice.finish_reason,
|
||||
Some(async_openai::types::chat::FinishReason::Stop)
|
||||
);
|
||||
|
||||
engine_task.await.expect("mock engine task");
|
||||
server_task.abort();
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
#[serial]
|
||||
async fn streaming_chat_via_http_client() {
|
||||
let (client, server_task, engine_task) =
|
||||
http_test_server(b"engine-http-stream", default_stream_output_specs()).await;
|
||||
|
||||
let request = CreateChatCompletionRequestArgs::default()
|
||||
.model("test-model")
|
||||
.stream(true)
|
||||
.max_completion_tokens(10u32)
|
||||
.messages([ChatCompletionRequestUserMessageArgs::default()
|
||||
.content("hello")
|
||||
.build()
|
||||
.expect("build user message")
|
||||
.into()])
|
||||
.build()
|
||||
.expect("build request");
|
||||
|
||||
let mut stream = client.chat().create_stream(request).await.expect("streaming chat completion");
|
||||
|
||||
let mut full_text = String::new();
|
||||
let mut saw_role = false;
|
||||
let mut saw_finish_reason = false;
|
||||
|
||||
while let Some(chunk) = stream.next().await {
|
||||
let chunk = chunk.expect("stream chunk");
|
||||
for choice in &chunk.choices {
|
||||
if choice.delta.role.is_some() {
|
||||
saw_role = true;
|
||||
}
|
||||
if let Some(ref delta) = choice.delta.content {
|
||||
full_text.push_str(delta);
|
||||
}
|
||||
if choice.finish_reason.is_some() {
|
||||
saw_finish_reason = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
assert!(saw_role, "expected an assistant role chunk");
|
||||
assert!(saw_finish_reason, "expected a terminal finish reason");
|
||||
assert_eq!(full_text, "hi");
|
||||
|
||||
engine_task.await.expect("mock engine task");
|
||||
server_task.abort();
|
||||
}
|
||||
@@ -0,0 +1,215 @@
|
||||
mod convert;
|
||||
mod types;
|
||||
mod validate;
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::sync::Arc;
|
||||
|
||||
use axum::Json;
|
||||
use axum::extract::State;
|
||||
use axum::http::HeaderMap;
|
||||
use axum::response::{IntoResponse, Response};
|
||||
use thiserror_ext::AsReport as _;
|
||||
use tracing::info;
|
||||
use tracing_futures::Instrument as _;
|
||||
use vllm_engine_core_client::protocol::logprobs::{Logprobs, PositionLogprobs};
|
||||
use vllm_llm::{CollectedGenerateOutput, GenerateOutputStreamExt as _};
|
||||
|
||||
use self::convert::prepare_generate_request;
|
||||
use self::types::{GenerateLogprob, GenerateRequest, GenerateResponse, GenerateResponseChoice};
|
||||
use crate::error::{ApiError, server_error};
|
||||
use crate::routes::openai::utils::logprobs::clamp_logprob;
|
||||
use crate::routes::openai::utils::types::{ChatLogProbs, ChatLogProbsContent, TopLogProb};
|
||||
use crate::routes::openai::utils::validated_json::ValidatedJson;
|
||||
use crate::state::AppState;
|
||||
use crate::utils::resolve_request_context;
|
||||
|
||||
/// Validate one token-in/token-out request and proxy it into the shared
|
||||
/// `vllm-text` stack.
|
||||
pub async fn generate(
|
||||
State(state): State<Arc<AppState>>,
|
||||
headers: HeaderMap,
|
||||
ValidatedJson(body): ValidatedJson<GenerateRequest>,
|
||||
) -> Response {
|
||||
let request_context = resolve_request_context(&headers, body.request_id.as_deref());
|
||||
let prepared = match prepare_generate_request(body, state.served_model_names(), request_context)
|
||||
{
|
||||
Ok(prepared) => prepared,
|
||||
Err(error) => return error.into_response(),
|
||||
};
|
||||
let request_span = tracing::info_span!(
|
||||
"generate",
|
||||
request_id = %prepared.request_id,
|
||||
engine_request_id = tracing::field::Empty,
|
||||
);
|
||||
|
||||
let log_request = state.enable_log_requests;
|
||||
let include_logprobs = prepared.include_logprobs;
|
||||
let include_prompt_logprobs = prepared.include_prompt_logprobs;
|
||||
|
||||
let raw_stream = match state
|
||||
.chat
|
||||
.text()
|
||||
.generate_raw(prepared.text_request)
|
||||
.instrument(request_span.clone())
|
||||
.await
|
||||
{
|
||||
Ok(stream) => stream,
|
||||
Err(error) => {
|
||||
return server_error!(
|
||||
"failed to submit raw generate request: {}",
|
||||
error.to_report_string()
|
||||
)
|
||||
.into_response();
|
||||
}
|
||||
};
|
||||
|
||||
let collected = match raw_stream.collect_output().instrument(request_span.clone()).await {
|
||||
Ok(collected) => collected,
|
||||
Err(error) => {
|
||||
return server_error!(
|
||||
"failed to collect raw generate response: {}",
|
||||
error.to_report_string()
|
||||
)
|
||||
.into_response();
|
||||
}
|
||||
};
|
||||
|
||||
if log_request {
|
||||
info!(
|
||||
parent: &request_span,
|
||||
prompt_tokens = collected.prompt_token_ids.len(),
|
||||
output_tokens = collected.token_ids.len(),
|
||||
finish_reason = collected.finish_reason.as_str(),
|
||||
"generate finished"
|
||||
);
|
||||
}
|
||||
|
||||
let response = match collect_generate(
|
||||
collected,
|
||||
prepared.request_id,
|
||||
include_logprobs,
|
||||
include_prompt_logprobs,
|
||||
) {
|
||||
Ok(response) => response,
|
||||
Err(error) => return error.into_response(),
|
||||
};
|
||||
|
||||
Json(response).into_response()
|
||||
}
|
||||
|
||||
fn collect_generate(
|
||||
collected: CollectedGenerateOutput,
|
||||
request_id: String,
|
||||
include_logprobs: bool,
|
||||
include_prompt_logprobs: bool,
|
||||
) -> Result<GenerateResponse, ApiError> {
|
||||
let logprobs = if include_logprobs {
|
||||
let logprobs = collected.logprobs.as_ref().ok_or_else(|| {
|
||||
ApiError::server_error(
|
||||
"raw generate response requested logprobs but generation returned none".to_string(),
|
||||
)
|
||||
})?;
|
||||
Some(raw_logprobs_to_openai_chat(logprobs)?)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let prompt_logprobs = if include_prompt_logprobs {
|
||||
let prompt_logprobs = collected.prompt_logprobs.as_ref().ok_or_else(|| {
|
||||
ApiError::server_error(
|
||||
"raw generate response requested prompt_logprobs but generation returned none"
|
||||
.to_string(),
|
||||
)
|
||||
})?;
|
||||
Some(raw_prompt_logprobs_to_maps(prompt_logprobs))
|
||||
} else {
|
||||
None
|
||||
};
|
||||
|
||||
Ok(GenerateResponse {
|
||||
request_id,
|
||||
choices: vec![GenerateResponseChoice {
|
||||
index: 0,
|
||||
logprobs,
|
||||
finish_reason: Some(collected.finish_reason.as_str().to_string()),
|
||||
token_ids: collected.token_ids,
|
||||
}],
|
||||
prompt_logprobs,
|
||||
kv_transfer_params: collected.kv_transfer_params,
|
||||
})
|
||||
}
|
||||
|
||||
fn raw_logprobs_to_openai_chat(logprobs: &Logprobs) -> Result<ChatLogProbs, ApiError> {
|
||||
let content = logprobs
|
||||
.positions
|
||||
.iter()
|
||||
.map(position_to_chat_logprobs_content)
|
||||
.collect::<Result<Vec<_>, _>>()?;
|
||||
|
||||
Ok(ChatLogProbs {
|
||||
content: Some(content),
|
||||
})
|
||||
}
|
||||
|
||||
fn raw_prompt_logprobs_to_maps(
|
||||
prompt_logprobs: &Logprobs,
|
||||
) -> Vec<Option<HashMap<u32, GenerateLogprob>>> {
|
||||
std::iter::once(None)
|
||||
.chain(
|
||||
prompt_logprobs
|
||||
.positions
|
||||
.iter()
|
||||
.map(|position| Some(position_to_logprob_map(position))),
|
||||
)
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn position_to_chat_logprobs_content(
|
||||
position: &PositionLogprobs,
|
||||
) -> Result<ChatLogProbsContent, ApiError> {
|
||||
let chosen = position.entries.first().ok_or_else(|| {
|
||||
ApiError::server_error(
|
||||
"raw generate logprobs position unexpectedly had no token candidates".to_string(),
|
||||
)
|
||||
})?;
|
||||
let token = format_token_id(chosen.token_id);
|
||||
|
||||
Ok(ChatLogProbsContent {
|
||||
token: token.clone(),
|
||||
logprob: clamp_logprob(chosen.logprob),
|
||||
bytes: Some(token.as_bytes().to_vec()),
|
||||
top_logprobs: position
|
||||
.entries
|
||||
.iter()
|
||||
.map(|entry| {
|
||||
let token = format_token_id(entry.token_id);
|
||||
TopLogProb {
|
||||
token: token.clone(),
|
||||
logprob: clamp_logprob(entry.logprob),
|
||||
bytes: Some(token.into_bytes()),
|
||||
}
|
||||
})
|
||||
.collect(),
|
||||
})
|
||||
}
|
||||
|
||||
fn position_to_logprob_map(position: &PositionLogprobs) -> HashMap<u32, GenerateLogprob> {
|
||||
position
|
||||
.entries
|
||||
.iter()
|
||||
.map(|entry| {
|
||||
(
|
||||
entry.token_id,
|
||||
GenerateLogprob {
|
||||
logprob: clamp_logprob(entry.logprob),
|
||||
rank: Some(entry.rank),
|
||||
decoded_token: Some(format_token_id(entry.token_id)),
|
||||
},
|
||||
)
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn format_token_id(token_id: u32) -> String {
|
||||
format!("token_id:{token_id}")
|
||||
}
|
||||
@@ -0,0 +1,112 @@
|
||||
use vllm_text::{Prompt, TextDecodeOptions, TextRequest};
|
||||
|
||||
use super::types::GenerateRequest;
|
||||
use super::validate;
|
||||
use crate::error::ApiError;
|
||||
use crate::utils::{ResolvedRequestContext, merge_kv_transfer_params};
|
||||
|
||||
/// Lowered generate request plus the response request ID.
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
pub struct PreparedRequest {
|
||||
pub request_id: String,
|
||||
pub text_request: TextRequest,
|
||||
pub include_logprobs: bool,
|
||||
pub include_prompt_logprobs: bool,
|
||||
}
|
||||
|
||||
/// Validate and lower one raw generate request into the internal
|
||||
/// text-generation format.
|
||||
pub fn prepare_generate_request(
|
||||
request: GenerateRequest,
|
||||
served_model_names: &[String],
|
||||
ctx: ResolvedRequestContext,
|
||||
) -> Result<PreparedRequest, ApiError> {
|
||||
validate::validate_request_compat(&request, served_model_names)?;
|
||||
|
||||
let include_logprobs = request.sampling_params.logprobs.is_some();
|
||||
let include_prompt_logprobs = request.sampling_params.prompt_logprobs.is_some();
|
||||
let mut sampling_params = request.sampling_params;
|
||||
sampling_params.vllm_xargs = merge_kv_transfer_params(
|
||||
sampling_params.vllm_xargs,
|
||||
request.kv_transfer_params.as_ref(),
|
||||
);
|
||||
|
||||
let text_request = TextRequest {
|
||||
request_id: ctx.request_id.clone(),
|
||||
prompt: Prompt::TokenIds(request.token_ids),
|
||||
mm_features: None,
|
||||
sampling_params,
|
||||
decode_options: TextDecodeOptions::default(),
|
||||
intermediate: false,
|
||||
priority: request.priority,
|
||||
cache_salt: request.cache_salt,
|
||||
add_special_tokens: false,
|
||||
data_parallel_rank: ctx.data_parallel_rank,
|
||||
};
|
||||
|
||||
Ok(PreparedRequest {
|
||||
request_id: ctx.request_id,
|
||||
text_request,
|
||||
include_logprobs,
|
||||
include_prompt_logprobs,
|
||||
})
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use serde_json::json;
|
||||
use vllm_text::Prompt;
|
||||
|
||||
use super::prepare_generate_request;
|
||||
use crate::routes::inference::generate::types::GenerateRequest;
|
||||
use crate::utils::ResolvedRequestContext;
|
||||
|
||||
#[test]
|
||||
fn prepare_generate_request_maps_token_prompt_and_sampling_params() {
|
||||
let request: GenerateRequest = serde_json::from_value(json!({
|
||||
"model": "Qwen/Qwen1.5-0.5B-Chat",
|
||||
"token_ids": [11, 22, 33],
|
||||
"priority": -3,
|
||||
"cache_salt": "salt",
|
||||
"sampling_params": {
|
||||
"max_tokens": 7,
|
||||
"logprobs": 2,
|
||||
"prompt_logprobs": 1,
|
||||
"ignore_eos": true
|
||||
},
|
||||
"kv_transfer_params": {
|
||||
"connector": "x"
|
||||
}
|
||||
}))
|
||||
.expect("parse request");
|
||||
|
||||
let prepared = prepare_generate_request(
|
||||
request,
|
||||
&["Qwen/Qwen1.5-0.5B-Chat".to_string()],
|
||||
ResolvedRequestContext::default(),
|
||||
)
|
||||
.expect("prepare");
|
||||
|
||||
assert_eq!(
|
||||
prepared.text_request.prompt,
|
||||
Prompt::TokenIds(vec![11, 22, 33])
|
||||
);
|
||||
assert_eq!(prepared.text_request.sampling_params.max_tokens, Some(7));
|
||||
assert_eq!(prepared.text_request.sampling_params.logprobs, Some(2));
|
||||
assert_eq!(
|
||||
prepared.text_request.sampling_params.prompt_logprobs,
|
||||
Some(1)
|
||||
);
|
||||
assert!(prepared.text_request.sampling_params.ignore_eos);
|
||||
assert_eq!(prepared.text_request.priority, -3);
|
||||
assert_eq!(prepared.text_request.cache_salt.as_deref(), Some("salt"));
|
||||
assert_eq!(
|
||||
prepared
|
||||
.text_request
|
||||
.sampling_params
|
||||
.vllm_xargs
|
||||
.and_then(|mut xargs| xargs.remove("kv_transfer_params")),
|
||||
Some(json!({"connector": "x"}))
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,57 @@
|
||||
use std::collections::HashMap;
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
use serde_json::{Map, Value};
|
||||
use validator::Validate;
|
||||
use vllm_text::SamplingParams;
|
||||
|
||||
use crate::routes::openai::utils::types::{ChatLogProbs, Normalizable};
|
||||
|
||||
/// vLLM-compatible request type for the token-in/token-out generate API.
|
||||
#[serde_with::skip_serializing_none]
|
||||
#[derive(Debug, Clone, Deserialize, Serialize, Validate)]
|
||||
pub struct GenerateRequest {
|
||||
pub request_id: Option<String>,
|
||||
pub model: Option<String>,
|
||||
pub token_ids: Vec<u32>,
|
||||
pub sampling_params: SamplingParams,
|
||||
#[serde(default)]
|
||||
pub stream: bool,
|
||||
pub cache_salt: Option<String>,
|
||||
#[serde(default)]
|
||||
pub priority: i32,
|
||||
pub kv_transfer_params: Option<HashMap<String, Value>>,
|
||||
#[serde(flatten)]
|
||||
pub other: Map<String, Value>,
|
||||
}
|
||||
|
||||
impl Normalizable for GenerateRequest {}
|
||||
|
||||
/// Mirrors the Python vLLM `GenerateResponseChoice` class.
|
||||
#[serde_with::skip_serializing_none]
|
||||
#[derive(Debug, Clone, Serialize)]
|
||||
pub(super) struct GenerateResponseChoice {
|
||||
pub index: u32,
|
||||
pub logprobs: Option<ChatLogProbs>,
|
||||
pub finish_reason: Option<String>,
|
||||
pub token_ids: Vec<u32>,
|
||||
}
|
||||
|
||||
/// Mirrors the Python vLLM `GenerateResponse` class.
|
||||
#[serde_with::skip_serializing_none]
|
||||
#[derive(Debug, Clone, Serialize)]
|
||||
pub(super) struct GenerateResponse {
|
||||
pub request_id: String,
|
||||
pub choices: Vec<GenerateResponseChoice>,
|
||||
pub prompt_logprobs: Option<Vec<Option<HashMap<u32, GenerateLogprob>>>>,
|
||||
pub kv_transfer_params: Option<Value>,
|
||||
}
|
||||
|
||||
/// Mirrors the Python vLLM `Logprob` class used in prompt-logprobs payloads.
|
||||
#[serde_with::skip_serializing_none]
|
||||
#[derive(Debug, Clone, Serialize)]
|
||||
pub(super) struct GenerateLogprob {
|
||||
pub logprob: f32,
|
||||
pub rank: Option<u32>,
|
||||
pub decoded_token: Option<String>,
|
||||
}
|
||||
@@ -0,0 +1,84 @@
|
||||
use super::types::GenerateRequest;
|
||||
use crate::error::{ApiError, bail_invalid_request};
|
||||
|
||||
/// Enforce the minimal compatibility contract for the Rust token generate
|
||||
/// route.
|
||||
pub(super) fn validate_request_compat(
|
||||
request: &GenerateRequest,
|
||||
served_model_names: &[String],
|
||||
) -> Result<(), ApiError> {
|
||||
if let Some(model) = request.model.as_ref()
|
||||
&& !served_model_names.iter().any(|n| n == model)
|
||||
{
|
||||
return Err(ApiError::model_not_found(model.clone()));
|
||||
}
|
||||
|
||||
if request.stream {
|
||||
bail_invalid_request!(param = "stream", "stream=true is not supported.");
|
||||
}
|
||||
|
||||
if request.token_ids.is_empty() {
|
||||
bail_invalid_request!(
|
||||
param = "token_ids",
|
||||
"token_ids must contain at least one token ID."
|
||||
);
|
||||
}
|
||||
|
||||
if request.sampling_params.max_tokens == Some(0) {
|
||||
bail_invalid_request!(
|
||||
param = "sampling_params",
|
||||
"max_tokens must be greater than 0."
|
||||
);
|
||||
}
|
||||
|
||||
if let Some(prompt_logprobs) = request.sampling_params.prompt_logprobs
|
||||
&& prompt_logprobs < 0
|
||||
&& prompt_logprobs != -1
|
||||
{
|
||||
bail_invalid_request!(
|
||||
param = "sampling_params",
|
||||
"`prompt_logprobs` must be a non-negative value or -1."
|
||||
);
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use serde_json::json;
|
||||
|
||||
use super::validate_request_compat;
|
||||
use crate::routes::inference::generate::types::GenerateRequest;
|
||||
|
||||
fn base_request() -> GenerateRequest {
|
||||
serde_json::from_value(json!({
|
||||
"model": "Qwen/Qwen1.5-0.5B-Chat",
|
||||
"token_ids": [11, 22],
|
||||
"sampling_params": {}
|
||||
}))
|
||||
.expect("parse request")
|
||||
}
|
||||
|
||||
fn served(names: &[&str]) -> Vec<String> {
|
||||
names.iter().map(|s| s.to_string()).collect()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn validate_request_compat_rejects_streaming() {
|
||||
let request = GenerateRequest {
|
||||
stream: true,
|
||||
..base_request()
|
||||
};
|
||||
assert!(validate_request_compat(&request, &served(&["Qwen/Qwen1.5-0.5B-Chat"])).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn validate_request_compat_rejects_empty_token_ids() {
|
||||
let request = GenerateRequest {
|
||||
token_ids: Vec::new(),
|
||||
..base_request()
|
||||
};
|
||||
assert!(validate_request_compat(&request, &served(&["Qwen/Qwen1.5-0.5B-Chat"])).is_err());
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,3 @@
|
||||
pub mod generate;
|
||||
|
||||
pub use generate::generate;
|
||||
@@ -0,0 +1,18 @@
|
||||
use std::sync::Arc;
|
||||
|
||||
use axum::Json;
|
||||
use axum::extract::State;
|
||||
use serde::Serialize;
|
||||
|
||||
use crate::state::AppState;
|
||||
|
||||
#[derive(Serialize)]
|
||||
pub(crate) struct ServerLoadResponse {
|
||||
server_load: u64,
|
||||
}
|
||||
|
||||
pub async fn load(State(state): State<Arc<AppState>>) -> Json<ServerLoadResponse> {
|
||||
Json(ServerLoadResponse {
|
||||
server_load: state.server_load(),
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,26 @@
|
||||
use axum::http::header::CONTENT_TYPE;
|
||||
use axum::http::{HeaderValue, StatusCode};
|
||||
use axum::response::{IntoResponse, Response};
|
||||
use thiserror_ext::AsReport;
|
||||
use vllm_metrics::METRICS;
|
||||
|
||||
const OPENMETRICS_CONTENT_TYPE: &str = "application/openmetrics-text; version=1.0.0; charset=utf-8";
|
||||
|
||||
pub async fn scrape() -> Response {
|
||||
match METRICS.render() {
|
||||
Ok(body) => (
|
||||
[(
|
||||
CONTENT_TYPE,
|
||||
HeaderValue::from_static(OPENMETRICS_CONTENT_TYPE),
|
||||
)],
|
||||
body,
|
||||
)
|
||||
.into_response(),
|
||||
|
||||
Err(error) => (
|
||||
StatusCode::INTERNAL_SERVER_ERROR,
|
||||
format!("failed to render metrics: {}", error.as_report()),
|
||||
)
|
||||
.into_response(),
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,992 @@
|
||||
use itertools::Itertools as _;
|
||||
use vllm_chat::{
|
||||
AssistantContentBlock, AssistantToolCall, ChatContent, ChatContentPart,
|
||||
ChatMessage as VllmChatMessage, ChatOptions, ChatRequest, ChatTool, ChatToolChoice,
|
||||
GenerationPromptMode, SamplingParams,
|
||||
};
|
||||
|
||||
use super::types::ChatCompletionRequest;
|
||||
use super::validate;
|
||||
use crate::error::{ApiError, bail_invalid_request};
|
||||
use crate::routes::openai::utils::structured_outputs::convert_from_response_format;
|
||||
use crate::routes::openai::utils::types::{
|
||||
ChatMessage, ContentPart, MessageContent, Tool, ToolChoice, ToolChoiceValue,
|
||||
};
|
||||
use crate::utils::{ResolvedRequestContext, convert_logit_bias, merge_kv_transfer_params};
|
||||
|
||||
/// Lowered chat request plus the public response metadata carried by every SSE
|
||||
/// chunk.
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
pub struct PreparedRequest {
|
||||
/// Stable OpenAI-style request ID, reused as the external chat request ID.
|
||||
pub request_id: String,
|
||||
/// Public model ID echoed back to the client.
|
||||
pub response_model: String,
|
||||
/// Whether the caller asked for the final streamed usage chunk.
|
||||
pub include_usage: bool,
|
||||
/// Whether the caller requested output logprobs on chat choices.
|
||||
pub requested_logprobs: bool,
|
||||
/// Whether the caller requested top-level prompt logprobs.
|
||||
pub include_prompt_logprobs: bool,
|
||||
/// Lowered chat request for `vllm-chat`.
|
||||
pub chat_request: ChatRequest,
|
||||
/// Last assistant-role message content to echo back when `echo=true`.
|
||||
pub echo: Option<String>,
|
||||
/// Whether to include token IDs alongside generated text.
|
||||
pub return_token_ids: bool,
|
||||
/// Whether to format logprob tokens as `token_id:{id}`.
|
||||
pub return_tokens_as_token_ids: bool,
|
||||
}
|
||||
|
||||
/// Validate and lower one OpenAI chat completion request into the internal chat
|
||||
/// format.
|
||||
///
|
||||
/// `served_model_names` must be non-empty; the first entry is used as the
|
||||
/// `model` field in responses.
|
||||
pub(crate) fn prepare_chat_request(
|
||||
request: ChatCompletionRequest,
|
||||
served_model_names: &[String],
|
||||
ctx: ResolvedRequestContext,
|
||||
) -> Result<PreparedRequest, ApiError> {
|
||||
validate::validate_request_compat(&request, served_model_names)?;
|
||||
|
||||
let request_id = format!("chatcmpl-{}", ctx.request_id);
|
||||
let echo = request
|
||||
.echo
|
||||
.then(|| extract_last_assistant_content(&request.messages))
|
||||
.flatten();
|
||||
let messages: Vec<_> = request.messages.into_iter().map(convert_message).try_collect()?;
|
||||
let generation_prompt_mode = normalize_generation_prompt_mode(
|
||||
request.add_generation_prompt,
|
||||
request.continue_final_message,
|
||||
&messages,
|
||||
)?;
|
||||
|
||||
let template_kwargs = request.chat_template_kwargs.unwrap_or_default();
|
||||
|
||||
let include_usage = (request.stream_options.as_ref())
|
||||
.and_then(|options| options.include_usage)
|
||||
.unwrap_or(false);
|
||||
let requested_logprobs = request.logprobs;
|
||||
|
||||
// Auto-enable prompt logprobs for non-streaming echo, matching Python vLLM's
|
||||
// behavior.
|
||||
let top_logprobs = request.top_logprobs.unwrap_or(0);
|
||||
let prompt_logprobs = request
|
||||
.prompt_logprobs
|
||||
.or((request.echo && !request.stream).then_some(top_logprobs));
|
||||
let include_prompt_logprobs = prompt_logprobs.is_some();
|
||||
|
||||
let structured_outputs = convert_from_response_format(
|
||||
request.response_format.as_ref(),
|
||||
&request.structured_outputs,
|
||||
)?;
|
||||
|
||||
let chat_request = ChatRequest {
|
||||
request_id: request_id.clone(),
|
||||
messages,
|
||||
sampling_params: SamplingParams {
|
||||
temperature: request.temperature,
|
||||
top_p: request.top_p,
|
||||
top_k: request.top_k,
|
||||
seed: request.seed,
|
||||
max_tokens: request.max_completion_tokens,
|
||||
min_tokens: request.min_tokens,
|
||||
logprobs: request.logprobs.then_some(top_logprobs),
|
||||
prompt_logprobs,
|
||||
min_p: request.min_p,
|
||||
frequency_penalty: request.frequency_penalty,
|
||||
presence_penalty: request.presence_penalty,
|
||||
repetition_penalty: request.repetition_penalty,
|
||||
stop_token_ids: request.stop_token_ids,
|
||||
ignore_eos: request.ignore_eos,
|
||||
logit_bias: convert_logit_bias(request.logit_bias)?,
|
||||
allowed_token_ids: request.allowed_token_ids,
|
||||
bad_words: request.bad_words,
|
||||
logprob_token_ids: None,
|
||||
structured_outputs,
|
||||
skip_reading_prefix_cache: None,
|
||||
vllm_xargs: merge_kv_transfer_params(
|
||||
request.vllm_xargs,
|
||||
request.kv_transfer_params.as_ref(),
|
||||
),
|
||||
},
|
||||
chat_options: ChatOptions {
|
||||
generation_prompt_mode,
|
||||
chat_template: request.chat_template,
|
||||
reasoning_effort: request.reasoning_effort,
|
||||
template_kwargs,
|
||||
},
|
||||
tools: convert_tools(request.tools)?,
|
||||
tool_choice: convert_tool_choice(request.tool_choice.as_ref())?,
|
||||
decode_options: vllm_text::output::TextDecodeOptions {
|
||||
skip_special_tokens: request.skip_special_tokens,
|
||||
include_stop_str_in_output: request.include_stop_str_in_output,
|
||||
stop_strings: request.stop.map(|stop| stop.into_vec()),
|
||||
min_tokens: request.min_tokens.unwrap_or(0),
|
||||
},
|
||||
intermediate: request.stream,
|
||||
priority: request.priority.unwrap_or(0),
|
||||
documents: request.documents,
|
||||
cache_salt: request.cache_salt,
|
||||
add_special_tokens: request.add_special_tokens,
|
||||
data_parallel_rank: ctx.data_parallel_rank,
|
||||
};
|
||||
|
||||
Ok(PreparedRequest {
|
||||
request_id,
|
||||
response_model: served_model_names.first().cloned().unwrap_or_default(),
|
||||
include_usage,
|
||||
requested_logprobs,
|
||||
include_prompt_logprobs,
|
||||
chat_request,
|
||||
echo,
|
||||
return_token_ids: request.return_token_ids.unwrap_or(false),
|
||||
return_tokens_as_token_ids: request.return_tokens_as_token_ids.unwrap_or(false),
|
||||
})
|
||||
}
|
||||
|
||||
fn normalize_generation_prompt_mode(
|
||||
add_generation_prompt: Option<bool>,
|
||||
continue_final_message: bool,
|
||||
messages: &[VllmChatMessage],
|
||||
) -> Result<GenerationPromptMode, ApiError> {
|
||||
if add_generation_prompt == Some(true) && continue_final_message {
|
||||
bail_invalid_request!(
|
||||
"Cannot set both `continue_final_message` and `add_generation_prompt` to True."
|
||||
);
|
||||
}
|
||||
|
||||
let last_role = messages.last().map(VllmChatMessage::role);
|
||||
match (add_generation_prompt, continue_final_message, last_role) {
|
||||
(Some(true), true, _) => unreachable!("rejected above"),
|
||||
(_, true, Some(vllm_chat::ChatRole::Assistant)) => {
|
||||
Ok(GenerationPromptMode::ContinueFinalAssistant)
|
||||
}
|
||||
(_, true, _) => {
|
||||
bail_invalid_request!(
|
||||
"Cannot set `continue_final_message` to True when the last message is not from the assistant."
|
||||
);
|
||||
}
|
||||
(Some(false), false, _) => Ok(GenerationPromptMode::NoGenerationPrompt),
|
||||
(None | Some(true), false, _) => Ok(GenerationPromptMode::StartNewAssistant),
|
||||
}
|
||||
}
|
||||
|
||||
/// Extract the text content of the last message if it has the assistant role.
|
||||
fn extract_last_assistant_content(messages: &[ChatMessage]) -> Option<String> {
|
||||
let ChatMessage::Assistant { content, .. } = messages.last()? else {
|
||||
return None;
|
||||
};
|
||||
let text = match content.as_ref()? {
|
||||
MessageContent::Text(text) => text.clone(),
|
||||
MessageContent::Parts(parts) => parts
|
||||
.iter()
|
||||
.filter_map(|p| match p {
|
||||
ContentPart::Text { text } => Some(text.as_str()),
|
||||
_ => None,
|
||||
})
|
||||
.collect::<Vec<_>>()
|
||||
.join("\n"),
|
||||
};
|
||||
(!text.is_empty()).then_some(text)
|
||||
}
|
||||
|
||||
/// Lower one OpenAI chat message into the `vllm-chat` message shape.
|
||||
fn convert_message(message: ChatMessage) -> Result<VllmChatMessage, ApiError> {
|
||||
match message {
|
||||
ChatMessage::System { content, .. } => {
|
||||
Ok(VllmChatMessage::system(convert_content(content)?))
|
||||
}
|
||||
ChatMessage::User { content, .. } => Ok(VllmChatMessage::user(convert_content(content)?)),
|
||||
ChatMessage::Assistant {
|
||||
content,
|
||||
tool_calls,
|
||||
reasoning,
|
||||
name: _,
|
||||
} => {
|
||||
let mut blocks = Vec::new();
|
||||
if let Some(reasoning) = reasoning
|
||||
&& !reasoning.is_empty()
|
||||
{
|
||||
blocks.push(AssistantContentBlock::Reasoning { text: reasoning });
|
||||
}
|
||||
if let Some(content) = content {
|
||||
blocks.extend(convert_assistant_text_blocks(content)?);
|
||||
}
|
||||
if let Some(tool_calls) = tool_calls {
|
||||
blocks.extend(convert_assistant_tool_calls(tool_calls)?);
|
||||
}
|
||||
if blocks.is_empty() {
|
||||
bail_invalid_request!(
|
||||
"Assistant messages must contain text, reasoning content, or tool_calls."
|
||||
);
|
||||
}
|
||||
|
||||
Ok(VllmChatMessage::assistant_blocks(blocks))
|
||||
}
|
||||
ChatMessage::Tool {
|
||||
content,
|
||||
tool_call_id,
|
||||
} => Ok(VllmChatMessage::tool_response(
|
||||
convert_content(content)?,
|
||||
tool_call_id,
|
||||
)),
|
||||
ChatMessage::Function { .. } => {
|
||||
bail_invalid_request!("Function messages are not supported.")
|
||||
}
|
||||
ChatMessage::Developer {
|
||||
content,
|
||||
tools,
|
||||
name: _,
|
||||
} => Ok(VllmChatMessage::developer(
|
||||
convert_content(content)?,
|
||||
convert_message_tools(tools)?,
|
||||
)),
|
||||
}
|
||||
}
|
||||
|
||||
/// Convert the given OpenAI message content value into the internal format in
|
||||
/// `vllm-chat`.
|
||||
fn convert_content(content: MessageContent) -> Result<ChatContent, ApiError> {
|
||||
match content {
|
||||
MessageContent::Text(text) => Ok(ChatContent::Text(text)),
|
||||
MessageContent::Parts(parts) => parts
|
||||
.into_iter()
|
||||
.map(|part| match part {
|
||||
ContentPart::Text { text } => Ok(ChatContentPart::text(text)),
|
||||
ContentPart::ImageUrl { image_url, uuid } => Ok(ChatContentPart::ImageUrl {
|
||||
image_url: image_url.url,
|
||||
detail: image_url.detail,
|
||||
uuid,
|
||||
}),
|
||||
_ => bail_invalid_request!("Only text and image_url content parts are supported."),
|
||||
})
|
||||
.try_collect()
|
||||
.map(ChatContent::Parts),
|
||||
}
|
||||
}
|
||||
|
||||
/// Convert the given OpenAI assistant message content into the internal format
|
||||
/// in `vllm-chat`.
|
||||
fn convert_assistant_text_blocks(
|
||||
content: MessageContent,
|
||||
) -> Result<Vec<AssistantContentBlock>, ApiError> {
|
||||
match content {
|
||||
MessageContent::Text(text) => Ok(vec![AssistantContentBlock::Text { text }]),
|
||||
MessageContent::Parts(parts) => parts
|
||||
.into_iter()
|
||||
.map(|part| match part {
|
||||
ContentPart::Text { text } => Ok(AssistantContentBlock::Text { text }),
|
||||
_ => bail_invalid_request!(
|
||||
"Only text content parts are supported for assistant messages."
|
||||
),
|
||||
})
|
||||
.try_collect(),
|
||||
}
|
||||
}
|
||||
|
||||
fn convert_assistant_tool_calls(
|
||||
tool_calls: Vec<crate::routes::openai::utils::types::ToolCall>,
|
||||
) -> Result<Vec<AssistantContentBlock>, ApiError> {
|
||||
tool_calls
|
||||
.into_iter()
|
||||
.map(|tool_call| {
|
||||
if tool_call.tool_type != "function" {
|
||||
bail_invalid_request!("Only function tool calls are supported.");
|
||||
}
|
||||
|
||||
Ok(AssistantContentBlock::ToolCall(AssistantToolCall {
|
||||
id: tool_call.id,
|
||||
name: tool_call.function.name,
|
||||
arguments: tool_call.function.arguments.unwrap_or_else(|| "{}".to_string()),
|
||||
}))
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn convert_tools(tools: Option<Vec<Tool>>) -> Result<Vec<ChatTool>, ApiError> {
|
||||
tools
|
||||
.unwrap_or_default()
|
||||
.into_iter()
|
||||
.map(|tool| {
|
||||
if tool.tool_type != "function" {
|
||||
bail_invalid_request!("Only function tools are supported.");
|
||||
}
|
||||
Ok(ChatTool {
|
||||
name: tool.function.name,
|
||||
description: tool.function.description,
|
||||
parameters: tool.function.parameters,
|
||||
strict: tool.function.strict,
|
||||
})
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn convert_message_tools(tools: Option<Vec<Tool>>) -> Result<Option<Vec<ChatTool>>, ApiError> {
|
||||
let tools = convert_tools(tools)?;
|
||||
Ok((!tools.is_empty()).then_some(tools))
|
||||
}
|
||||
|
||||
fn convert_tool_choice(tool_choice: Option<&ToolChoice>) -> Result<ChatToolChoice, ApiError> {
|
||||
match tool_choice {
|
||||
None | Some(ToolChoice::Value(ToolChoiceValue::Auto)) => Ok(ChatToolChoice::Auto),
|
||||
Some(ToolChoice::Value(ToolChoiceValue::None)) => Ok(ChatToolChoice::None),
|
||||
_ => bail_invalid_request!("tool_choice={:?} is not supported yet.", tool_choice),
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use std::collections::HashMap;
|
||||
|
||||
use axum::http::HeaderMap;
|
||||
use expect_test::expect;
|
||||
use llm_multimodal::ImageDetail;
|
||||
use serde_json::json;
|
||||
use vllm_chat::{
|
||||
AssistantContentBlock, AssistantToolCall, ChatContentPart, ChatMessage as VllmChatMessage,
|
||||
ChatTool as VllmChatTool, ChatToolChoice, GenerationPromptMode,
|
||||
SamplingParams as VllmSamplingParams,
|
||||
};
|
||||
use vllm_text::output::TextDecodeOptions;
|
||||
|
||||
use super::prepare_chat_request;
|
||||
use crate::routes::openai::chat_completions::types::{
|
||||
AssistantRole, ChatCompletionMessage, ChatCompletionRequest,
|
||||
};
|
||||
use crate::routes::openai::utils::types::{
|
||||
ChatMessage, ContentPart, Function, FunctionCallResponse, ImageUrl, MessageContent, Tool,
|
||||
ToolCall, ToolChoice, ToolChoiceValue, VideoUrl,
|
||||
};
|
||||
use crate::utils::{ResolvedRequestContext, resolve_request_context};
|
||||
|
||||
fn request_context(headers: &HeaderMap, request_id: Option<&str>) -> ResolvedRequestContext {
|
||||
resolve_request_context(headers, request_id)
|
||||
}
|
||||
|
||||
fn served(names: &[&str]) -> Vec<String> {
|
||||
names.iter().map(|s| s.to_string()).collect()
|
||||
}
|
||||
|
||||
fn base_request() -> ChatCompletionRequest {
|
||||
ChatCompletionRequest {
|
||||
model: "Qwen/Qwen1.5-0.5B-Chat".to_string(),
|
||||
messages: vec![ChatMessage::User {
|
||||
content: MessageContent::Text("hello".to_string()),
|
||||
name: None,
|
||||
}],
|
||||
stream: true,
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn prepare_chat_request_maps_text_parts() {
|
||||
let mut request = base_request();
|
||||
request.messages = vec![ChatMessage::Assistant {
|
||||
content: Some(MessageContent::Parts(vec![ContentPart::Text {
|
||||
text: "hello".to_string(),
|
||||
}])),
|
||||
name: None,
|
||||
tool_calls: None,
|
||||
reasoning: None,
|
||||
}];
|
||||
request.add_generation_prompt = Some(false);
|
||||
request.continue_final_message = true;
|
||||
request.skip_special_tokens = false;
|
||||
request.chat_template_kwargs = Some(HashMap::from([("foo".to_string(), json!("bar"))]));
|
||||
|
||||
let prepared = prepare_chat_request(
|
||||
request,
|
||||
&served(&["Qwen/Qwen1.5-0.5B-Chat"]),
|
||||
ResolvedRequestContext::default(),
|
||||
)
|
||||
.expect("request is valid");
|
||||
|
||||
assert!(prepared.request_id.starts_with("chatcmpl-"));
|
||||
assert_eq!(
|
||||
prepared.chat_request.messages,
|
||||
vec![VllmChatMessage::assistant_text("hello")]
|
||||
);
|
||||
assert_eq!(
|
||||
prepared.chat_request.sampling_params,
|
||||
VllmSamplingParams::default()
|
||||
);
|
||||
assert_eq!(
|
||||
prepared.chat_request.chat_options.generation_prompt_mode,
|
||||
GenerationPromptMode::ContinueFinalAssistant
|
||||
);
|
||||
assert_eq!(
|
||||
prepared.chat_request.chat_options.template_kwargs,
|
||||
HashMap::from([("foo".to_string(), json!("bar"))])
|
||||
);
|
||||
assert_eq!(
|
||||
prepared.chat_request.decode_options,
|
||||
TextDecodeOptions {
|
||||
skip_special_tokens: false,
|
||||
include_stop_str_in_output: false,
|
||||
stop_strings: None,
|
||||
min_tokens: 0,
|
||||
}
|
||||
);
|
||||
assert!(prepared.chat_request.tools.is_empty());
|
||||
assert_eq!(prepared.chat_request.tool_choice, ChatToolChoice::Auto);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn prepare_chat_request_keeps_optional_sampling_fields_unset() {
|
||||
let prepared = prepare_chat_request(
|
||||
base_request(),
|
||||
&served(&["Qwen/Qwen1.5-0.5B-Chat"]),
|
||||
ResolvedRequestContext::default(),
|
||||
)
|
||||
.expect("request is valid");
|
||||
|
||||
assert!(prepared.request_id.starts_with("chatcmpl-"));
|
||||
assert_eq!(
|
||||
prepared.chat_request.messages,
|
||||
vec![VllmChatMessage::user("hello")]
|
||||
);
|
||||
assert_eq!(
|
||||
prepared.chat_request.sampling_params,
|
||||
VllmSamplingParams::default()
|
||||
);
|
||||
assert_eq!(
|
||||
prepared.chat_request.chat_options.generation_prompt_mode,
|
||||
GenerationPromptMode::StartNewAssistant
|
||||
);
|
||||
assert_eq!(
|
||||
prepared.chat_request.decode_options,
|
||||
TextDecodeOptions {
|
||||
skip_special_tokens: true,
|
||||
include_stop_str_in_output: false,
|
||||
stop_strings: None,
|
||||
min_tokens: 0,
|
||||
}
|
||||
);
|
||||
assert!(prepared.chat_request.tools.is_empty());
|
||||
assert_eq!(prepared.chat_request.tool_choice, ChatToolChoice::Auto);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn prepare_chat_request_preserves_sampling_passthrough_fields() {
|
||||
let request = ChatCompletionRequest {
|
||||
seed: Some(42),
|
||||
min_p: Some(0.2),
|
||||
frequency_penalty: Some(0.3),
|
||||
presence_penalty: Some(0.4),
|
||||
repetition_penalty: Some(1.1),
|
||||
..base_request()
|
||||
};
|
||||
|
||||
let prepared = prepare_chat_request(
|
||||
request,
|
||||
&served(&["Qwen/Qwen1.5-0.5B-Chat"]),
|
||||
ResolvedRequestContext::default(),
|
||||
)
|
||||
.expect("request is valid");
|
||||
let expected = VllmSamplingParams {
|
||||
seed: Some(42),
|
||||
min_p: Some(0.2),
|
||||
frequency_penalty: Some(0.3),
|
||||
presence_penalty: Some(0.4),
|
||||
repetition_penalty: Some(1.1),
|
||||
..VllmSamplingParams::default()
|
||||
};
|
||||
assert_eq!(prepared.chat_request.sampling_params, expected);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn prepare_chat_request_accepts_developer_messages() {
|
||||
let request = ChatCompletionRequest {
|
||||
messages: vec![ChatMessage::Developer {
|
||||
content: MessageContent::Text("hello".to_string()),
|
||||
tools: Some(vec![Tool {
|
||||
tool_type: "function".to_string(),
|
||||
function: Function {
|
||||
name: "get_weather".to_string(),
|
||||
description: Some("Get weather".to_string()),
|
||||
parameters: json!({
|
||||
"type": "object",
|
||||
"properties": {"city": {"type": "string"}},
|
||||
}),
|
||||
strict: Some(true),
|
||||
},
|
||||
}]),
|
||||
name: None,
|
||||
}],
|
||||
..base_request()
|
||||
};
|
||||
|
||||
let prepared = prepare_chat_request(
|
||||
request,
|
||||
&served(&["Qwen/Qwen1.5-0.5B-Chat"]),
|
||||
ResolvedRequestContext::default(),
|
||||
)
|
||||
.expect("request is valid");
|
||||
|
||||
assert_eq!(
|
||||
prepared.chat_request.messages,
|
||||
vec![VllmChatMessage::developer(
|
||||
"hello",
|
||||
Some(vec![VllmChatTool {
|
||||
name: "get_weather".to_string(),
|
||||
description: Some("Get weather".to_string()),
|
||||
parameters: json!({
|
||||
"type": "object",
|
||||
"properties": {"city": {"type": "string"}},
|
||||
}),
|
||||
strict: Some(true),
|
||||
}]),
|
||||
)]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn prepare_chat_request_maps_image_url_content_parts() {
|
||||
let request = ChatCompletionRequest {
|
||||
messages: vec![ChatMessage::User {
|
||||
content: MessageContent::Parts(vec![
|
||||
ContentPart::Text {
|
||||
text: "describe ".to_string(),
|
||||
},
|
||||
ContentPart::ImageUrl {
|
||||
image_url: ImageUrl {
|
||||
url: "https://example.com/image.png".to_string(),
|
||||
detail: Some(ImageDetail::Low),
|
||||
},
|
||||
uuid: Some("image-1".to_string()),
|
||||
},
|
||||
ContentPart::Text {
|
||||
text: " briefly".to_string(),
|
||||
},
|
||||
]),
|
||||
name: None,
|
||||
}],
|
||||
..base_request()
|
||||
};
|
||||
|
||||
let prepared = prepare_chat_request(
|
||||
request,
|
||||
&served(&["Qwen/Qwen1.5-0.5B-Chat"]),
|
||||
ResolvedRequestContext::default(),
|
||||
)
|
||||
.expect("request is valid");
|
||||
|
||||
assert_eq!(
|
||||
prepared.chat_request.messages,
|
||||
vec![VllmChatMessage::user(vec![
|
||||
ChatContentPart::text("describe "),
|
||||
ChatContentPart::ImageUrl {
|
||||
image_url: "https://example.com/image.png".to_string(),
|
||||
detail: Some(ImageDetail::Low),
|
||||
uuid: Some("image-1".to_string()),
|
||||
},
|
||||
ChatContentPart::text(" briefly"),
|
||||
])]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn prepare_chat_request_maps_developer_image_url_content_parts() {
|
||||
let request = ChatCompletionRequest {
|
||||
messages: vec![ChatMessage::Developer {
|
||||
content: MessageContent::Parts(vec![ContentPart::ImageUrl {
|
||||
image_url: ImageUrl {
|
||||
url: "https://example.com/image.png".to_string(),
|
||||
detail: None,
|
||||
},
|
||||
uuid: None,
|
||||
}]),
|
||||
tools: None,
|
||||
name: None,
|
||||
}],
|
||||
..base_request()
|
||||
};
|
||||
|
||||
let prepared = prepare_chat_request(
|
||||
request,
|
||||
&served(&["Qwen/Qwen1.5-0.5B-Chat"]),
|
||||
ResolvedRequestContext::default(),
|
||||
)
|
||||
.expect("request is valid");
|
||||
|
||||
assert_eq!(
|
||||
prepared.chat_request.messages,
|
||||
vec![VllmChatMessage::developer(
|
||||
vec![ChatContentPart::image_url("https://example.com/image.png")],
|
||||
None,
|
||||
)]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn prepare_chat_request_rejects_video_content_parts() {
|
||||
let request = ChatCompletionRequest {
|
||||
messages: vec![ChatMessage::User {
|
||||
content: MessageContent::Parts(vec![ContentPart::VideoUrl {
|
||||
video_url: VideoUrl {
|
||||
url: "https://example.com/video.mp4".to_string(),
|
||||
},
|
||||
}]),
|
||||
name: None,
|
||||
}],
|
||||
..base_request()
|
||||
};
|
||||
|
||||
let error = prepare_chat_request(
|
||||
request,
|
||||
&served(&["Qwen/Qwen1.5-0.5B-Chat"]),
|
||||
ResolvedRequestContext::default(),
|
||||
)
|
||||
.unwrap_err();
|
||||
|
||||
expect!["Only text and image_url content parts are supported."]
|
||||
.assert_eq(&error.to_error_response().error.message);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn prepare_chat_request_rejects_assistant_image_url_content_parts() {
|
||||
let request = ChatCompletionRequest {
|
||||
messages: vec![ChatMessage::Assistant {
|
||||
content: Some(MessageContent::Parts(vec![ContentPart::ImageUrl {
|
||||
image_url: ImageUrl {
|
||||
url: "https://example.com/image.png".to_string(),
|
||||
detail: None,
|
||||
},
|
||||
uuid: None,
|
||||
}])),
|
||||
name: None,
|
||||
tool_calls: None,
|
||||
reasoning: None,
|
||||
}],
|
||||
..base_request()
|
||||
};
|
||||
|
||||
let error = prepare_chat_request(
|
||||
request,
|
||||
&served(&["Qwen/Qwen1.5-0.5B-Chat"]),
|
||||
ResolvedRequestContext::default(),
|
||||
)
|
||||
.unwrap_err();
|
||||
|
||||
expect!["Only text content parts are supported for assistant messages."]
|
||||
.assert_eq(&error.to_error_response().error.message);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn prepare_chat_request_accepts_assistant_reasoning_history() {
|
||||
let message = ChatCompletionMessage {
|
||||
role: AssistantRole,
|
||||
content: Some("answer".to_string()),
|
||||
tool_calls: None,
|
||||
reasoning: Some("inner".to_string()),
|
||||
};
|
||||
let message_json = serde_json::to_value(message).expect("message serializes");
|
||||
|
||||
let request = ChatCompletionRequest {
|
||||
messages: vec![
|
||||
serde_json::from_value(message_json).expect("response message is valid history"),
|
||||
],
|
||||
add_generation_prompt: Some(false),
|
||||
..base_request()
|
||||
};
|
||||
|
||||
let prepared = prepare_chat_request(
|
||||
request,
|
||||
&served(&["Qwen/Qwen1.5-0.5B-Chat"]),
|
||||
ResolvedRequestContext::default(),
|
||||
)
|
||||
.expect("request is valid");
|
||||
assert_eq!(
|
||||
prepared.chat_request.messages,
|
||||
vec![VllmChatMessage::assistant_blocks(vec![
|
||||
AssistantContentBlock::Reasoning {
|
||||
text: "inner".to_string(),
|
||||
},
|
||||
AssistantContentBlock::Text {
|
||||
text: "answer".to_string(),
|
||||
},
|
||||
])]
|
||||
);
|
||||
assert!(prepared.chat_request.tools.is_empty());
|
||||
assert_eq!(prepared.chat_request.tool_choice, ChatToolChoice::Auto);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn prepare_chat_request_accepts_legacy_reasoning_content_alias() {
|
||||
let request = ChatCompletionRequest {
|
||||
messages: vec![
|
||||
serde_json::from_value(json!({
|
||||
"role": "assistant",
|
||||
"content": "answer",
|
||||
"reasoning_content": "inner",
|
||||
}))
|
||||
.expect("legacy reasoning_content alias is accepted"),
|
||||
],
|
||||
add_generation_prompt: Some(false),
|
||||
..base_request()
|
||||
};
|
||||
|
||||
let prepared = prepare_chat_request(
|
||||
request,
|
||||
&served(&["Qwen/Qwen1.5-0.5B-Chat"]),
|
||||
ResolvedRequestContext::default(),
|
||||
)
|
||||
.expect("request is valid");
|
||||
assert_eq!(
|
||||
prepared.chat_request.messages,
|
||||
vec![VllmChatMessage::assistant_blocks(vec![
|
||||
AssistantContentBlock::Reasoning {
|
||||
text: "inner".to_string(),
|
||||
},
|
||||
AssistantContentBlock::Text {
|
||||
text: "answer".to_string(),
|
||||
},
|
||||
])]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn prepare_chat_request_accepts_tools_and_tool_history() {
|
||||
let request = ChatCompletionRequest {
|
||||
messages: vec![
|
||||
ChatMessage::Assistant {
|
||||
content: None,
|
||||
name: None,
|
||||
tool_calls: Some(vec![ToolCall {
|
||||
id: "call_1".to_string(),
|
||||
tool_type: "function".to_string(),
|
||||
function: FunctionCallResponse {
|
||||
name: "get_weather".to_string(),
|
||||
arguments: Some(r#"{"city":"Paris"}"#.to_string()),
|
||||
},
|
||||
}]),
|
||||
reasoning: None,
|
||||
},
|
||||
ChatMessage::Tool {
|
||||
content: MessageContent::Text("Sunny".to_string()),
|
||||
tool_call_id: "call_1".to_string(),
|
||||
},
|
||||
],
|
||||
tools: Some(vec![Tool {
|
||||
tool_type: "function".to_string(),
|
||||
function: Function {
|
||||
name: "get_weather".to_string(),
|
||||
description: Some("Get weather".to_string()),
|
||||
parameters: json!({
|
||||
"type": "object",
|
||||
"properties": {"city": {"type": "string"}},
|
||||
}),
|
||||
strict: None,
|
||||
},
|
||||
}]),
|
||||
tool_choice: Some(ToolChoice::Value(ToolChoiceValue::None)),
|
||||
..base_request()
|
||||
};
|
||||
|
||||
let prepared = prepare_chat_request(
|
||||
request,
|
||||
&served(&["Qwen/Qwen1.5-0.5B-Chat"]),
|
||||
ResolvedRequestContext::default(),
|
||||
)
|
||||
.expect("request is valid");
|
||||
assert_eq!(
|
||||
prepared.chat_request.messages,
|
||||
vec![
|
||||
VllmChatMessage::assistant_blocks(vec![AssistantContentBlock::ToolCall(
|
||||
AssistantToolCall {
|
||||
id: "call_1".to_string(),
|
||||
name: "get_weather".to_string(),
|
||||
arguments: r#"{"city":"Paris"}"#.to_string(),
|
||||
},
|
||||
)]),
|
||||
VllmChatMessage::tool_response("Sunny", "call_1"),
|
||||
]
|
||||
);
|
||||
assert_eq!(
|
||||
prepared.chat_request.tools,
|
||||
vec![VllmChatTool {
|
||||
name: "get_weather".to_string(),
|
||||
description: Some("Get weather".to_string()),
|
||||
parameters: json!({
|
||||
"type": "object",
|
||||
"properties": {"city": {"type": "string"}},
|
||||
}),
|
||||
strict: None,
|
||||
}]
|
||||
);
|
||||
assert_eq!(prepared.chat_request.tool_choice, ChatToolChoice::None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn prepare_chat_request_lowers_logprobs_fields() {
|
||||
let request = ChatCompletionRequest {
|
||||
stream: false,
|
||||
logprobs: true,
|
||||
prompt_logprobs: Some(2),
|
||||
..base_request()
|
||||
};
|
||||
|
||||
let prepared = prepare_chat_request(
|
||||
request,
|
||||
&served(&["Qwen/Qwen1.5-0.5B-Chat"]),
|
||||
ResolvedRequestContext::default(),
|
||||
)
|
||||
.expect("request is valid");
|
||||
|
||||
assert!(prepared.requested_logprobs);
|
||||
assert!(prepared.include_prompt_logprobs);
|
||||
assert_eq!(prepared.chat_request.sampling_params.logprobs, Some(0));
|
||||
assert_eq!(
|
||||
prepared.chat_request.sampling_params.prompt_logprobs,
|
||||
Some(2)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn prepare_chat_request_keeps_prompt_logprobs_independent_from_echo() {
|
||||
let request = ChatCompletionRequest {
|
||||
logprobs: true,
|
||||
top_logprobs: Some(3),
|
||||
echo: true,
|
||||
..base_request()
|
||||
};
|
||||
|
||||
let prepared = prepare_chat_request(
|
||||
request,
|
||||
&served(&["Qwen/Qwen1.5-0.5B-Chat"]),
|
||||
ResolvedRequestContext::default(),
|
||||
)
|
||||
.expect("request is valid");
|
||||
|
||||
assert_eq!(prepared.chat_request.sampling_params.logprobs, Some(3));
|
||||
assert_eq!(prepared.chat_request.sampling_params.prompt_logprobs, None);
|
||||
assert!(!prepared.include_prompt_logprobs);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn prepare_chat_request_threads_data_parallel_rank() {
|
||||
let mut headers = HeaderMap::new();
|
||||
headers.insert("X-data-parallel-rank", "7".parse().unwrap());
|
||||
let prepared = prepare_chat_request(
|
||||
base_request(),
|
||||
&served(&["Qwen/Qwen1.5-0.5B-Chat"]),
|
||||
request_context(&headers, None),
|
||||
)
|
||||
.expect("request is valid");
|
||||
assert_eq!(prepared.chat_request.data_parallel_rank, Some(7));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn prepare_chat_request_leaves_data_parallel_rank_none_when_absent() {
|
||||
let prepared = prepare_chat_request(
|
||||
base_request(),
|
||||
&served(&["Qwen/Qwen1.5-0.5B-Chat"]),
|
||||
ResolvedRequestContext::default(),
|
||||
)
|
||||
.expect("request is valid");
|
||||
assert_eq!(prepared.chat_request.data_parallel_rank, None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn prepare_chat_request_maps_no_generation_prompt_mode() {
|
||||
let mut request = base_request();
|
||||
request.add_generation_prompt = Some(false);
|
||||
|
||||
let prepared = prepare_chat_request(
|
||||
request,
|
||||
&served(&["Qwen/Qwen1.5-0.5B-Chat"]),
|
||||
ResolvedRequestContext::default(),
|
||||
)
|
||||
.expect("request is valid");
|
||||
|
||||
assert_eq!(
|
||||
prepared.chat_request.chat_options.generation_prompt_mode,
|
||||
GenerationPromptMode::NoGenerationPrompt
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn prepare_chat_request_rejects_conflicting_explicit_generation_prompt_flags() {
|
||||
let mut request = base_request();
|
||||
request.add_generation_prompt = Some(true);
|
||||
request.continue_final_message = true;
|
||||
|
||||
let error = prepare_chat_request(
|
||||
request,
|
||||
&served(&["Qwen/Qwen1.5-0.5B-Chat"]),
|
||||
ResolvedRequestContext::default(),
|
||||
)
|
||||
.unwrap_err();
|
||||
|
||||
expect!["Cannot set both `continue_final_message` and `add_generation_prompt` to True."]
|
||||
.assert_eq(&error.to_error_response().error.message);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn prepare_chat_request_accepts_continue_final_message_with_implicit_add_generation_prompt() {
|
||||
let mut request = base_request();
|
||||
request.messages = vec![ChatMessage::Assistant {
|
||||
content: Some(MessageContent::Text("hello".to_string())),
|
||||
name: None,
|
||||
tool_calls: None,
|
||||
reasoning: None,
|
||||
}];
|
||||
request.continue_final_message = true;
|
||||
|
||||
let prepared = prepare_chat_request(
|
||||
request,
|
||||
&served(&["Qwen/Qwen1.5-0.5B-Chat"]),
|
||||
ResolvedRequestContext::default(),
|
||||
)
|
||||
.expect("request is valid");
|
||||
|
||||
assert_eq!(
|
||||
prepared.chat_request.chat_options.generation_prompt_mode,
|
||||
GenerationPromptMode::ContinueFinalAssistant
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn prepare_chat_request_rejects_continue_final_message_without_final_assistant() {
|
||||
let mut request = base_request();
|
||||
request.continue_final_message = true;
|
||||
|
||||
let error = prepare_chat_request(
|
||||
request,
|
||||
&served(&["Qwen/Qwen1.5-0.5B-Chat"]),
|
||||
ResolvedRequestContext::default(),
|
||||
)
|
||||
.unwrap_err();
|
||||
|
||||
expect!["Cannot set `continue_final_message` to True when the last message is not from the assistant."]
|
||||
.assert_eq(&error.to_error_response().error.message);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn prepare_chat_request_allows_new_assistant_mode_after_final_assistant() {
|
||||
let request = ChatCompletionRequest {
|
||||
messages: vec![ChatMessage::Assistant {
|
||||
content: Some(MessageContent::Text("hello".to_string())),
|
||||
name: None,
|
||||
tool_calls: None,
|
||||
reasoning: None,
|
||||
}],
|
||||
..base_request()
|
||||
};
|
||||
|
||||
let prepared = prepare_chat_request(
|
||||
request,
|
||||
&served(&["Qwen/Qwen1.5-0.5B-Chat"]),
|
||||
ResolvedRequestContext::default(),
|
||||
)
|
||||
.expect("request is valid");
|
||||
|
||||
assert_eq!(
|
||||
prepared.chat_request.chat_options.generation_prompt_mode,
|
||||
GenerationPromptMode::StartNewAssistant
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,600 @@
|
||||
use std::collections::HashMap;
|
||||
use std::fmt;
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
use serde_json::Value;
|
||||
use serde_with::SerializeDisplay;
|
||||
use validator::Validate;
|
||||
use vllm_chat::ReasoningEffort;
|
||||
|
||||
use crate::routes::openai::utils::structured_outputs::ResponseFormat;
|
||||
use crate::routes::openai::utils::types::{
|
||||
ChatLogProbs, ChatMessage, MessageContent, Normalizable, StreamOptions, StringOrArray, Tool,
|
||||
ToolCall, ToolCallDelta, ToolChoice, ToolChoiceValue, ToolReference, UNKNOWN_MODEL_ID, Usage,
|
||||
default_true, validate_stop, validate_top_p_value,
|
||||
};
|
||||
|
||||
/// vLLM-compatible request type for the Chat Completions API.
|
||||
///
|
||||
/// Mirrors the Python vLLM `ChatCompletionRequest` class. The local copy keeps
|
||||
/// the request type route-owned so we can add vLLM-only fields directly instead
|
||||
/// of layering wrapper deserializers on top.
|
||||
#[serde_with::skip_serializing_none]
|
||||
#[derive(Debug, Clone, Deserialize, Serialize, Validate)]
|
||||
#[validate(schema(function = "validate_chat_cross_parameters"))]
|
||||
pub struct ChatCompletionRequest {
|
||||
// -------- Standard OpenAI API Parameters --------
|
||||
/// A list of messages comprising the conversation so far
|
||||
#[validate(custom(function = "validate_messages"))]
|
||||
pub messages: Vec<ChatMessage>,
|
||||
|
||||
/// ID of the model to use
|
||||
#[serde(default = "default_model")]
|
||||
pub model: String,
|
||||
|
||||
/// Number between -2.0 and 2.0. Positive values penalize new tokens based
|
||||
/// on their existing frequency in the text so far
|
||||
#[validate(range(min = -2.0, max = 2.0))]
|
||||
pub frequency_penalty: Option<f32>,
|
||||
|
||||
/// Modify the likelihood of specified tokens appearing in the completion
|
||||
pub logit_bias: Option<HashMap<String, f32>>,
|
||||
|
||||
/// Whether to return log probabilities of the output tokens
|
||||
#[serde(default)]
|
||||
pub logprobs: bool,
|
||||
|
||||
/// An integer specifying the number of most likely tokens to return
|
||||
/// -1 means return all
|
||||
#[validate(range(min = -1))]
|
||||
pub top_logprobs: Option<i32>,
|
||||
|
||||
/// Deprecated: Replaced by max_completion_tokens
|
||||
#[deprecated(note = "Use max_completion_tokens instead")]
|
||||
#[validate(range(min = 1))]
|
||||
pub max_tokens: Option<u32>,
|
||||
|
||||
/// An upper bound for the number of tokens that can be generated for a
|
||||
/// completion
|
||||
#[validate(range(min = 1))]
|
||||
pub max_completion_tokens: Option<u32>,
|
||||
|
||||
/// How many chat completion choices to generate for each input message
|
||||
#[validate(range(min = 1, max = 10))]
|
||||
pub n: Option<u32>,
|
||||
|
||||
/// Number between -2.0 and 2.0. Positive values penalize new tokens based
|
||||
/// on whether they appear in the text so far
|
||||
#[validate(range(min = -2.0, max = 2.0))]
|
||||
pub presence_penalty: Option<f32>,
|
||||
|
||||
/// An object specifying the format that the model must output
|
||||
pub response_format: Option<ResponseFormat>,
|
||||
|
||||
/// If specified, our system will make a best effort to sample
|
||||
/// deterministically
|
||||
pub seed: Option<i64>,
|
||||
|
||||
/// Up to 4 sequences where the API will stop generating further tokens
|
||||
#[validate(custom(function = "validate_stop"))]
|
||||
pub stop: Option<StringOrArray>,
|
||||
|
||||
/// If set, partial message deltas will be sent
|
||||
#[serde(default)]
|
||||
pub stream: bool,
|
||||
|
||||
/// Options for streaming response
|
||||
pub stream_options: Option<StreamOptions>,
|
||||
|
||||
/// What sampling temperature to use, between 0 and 2
|
||||
#[validate(range(min = 0.0, max = 2.0))]
|
||||
pub temperature: Option<f32>,
|
||||
|
||||
/// An alternative to sampling with temperature
|
||||
#[validate(custom(function = "validate_top_p_value"))]
|
||||
pub top_p: Option<f32>,
|
||||
|
||||
/// A list of tools the model may call
|
||||
pub tools: Option<Vec<Tool>>,
|
||||
|
||||
/// Controls which (if any) tool is called by the model
|
||||
pub tool_choice: Option<ToolChoice>,
|
||||
|
||||
/// Effort level for reasoning models (none, minimal, low, medium, high,
|
||||
/// xhigh, max)
|
||||
pub reasoning_effort: Option<ReasoningEffort>,
|
||||
|
||||
/// Whether to enable parallel function calling during tool use
|
||||
pub parallel_tool_calls: Option<bool>,
|
||||
|
||||
/// A unique identifier representing your end-user
|
||||
pub user: Option<String>,
|
||||
|
||||
// -------- vLLM Sampling Parameters --------
|
||||
/// Use beam search instead of sampling
|
||||
#[serde(default)]
|
||||
pub use_beam_search: bool,
|
||||
|
||||
/// Top-k sampling parameter
|
||||
pub top_k: Option<u32>,
|
||||
|
||||
/// Min-p nucleus sampling parameter
|
||||
#[validate(range(min = 0.0, max = 1.0))]
|
||||
pub min_p: Option<f32>,
|
||||
|
||||
/// Repetition penalty for reducing repetitive text
|
||||
#[validate(range(min = 0.0, max = 2.0))]
|
||||
pub repetition_penalty: Option<f32>,
|
||||
|
||||
/// Length penalty for beam search
|
||||
pub length_penalty: Option<f32>,
|
||||
|
||||
/// Specific token IDs to use as stop conditions
|
||||
pub stop_token_ids: Option<Vec<u32>>,
|
||||
|
||||
/// Include stop string in output
|
||||
#[serde(default)]
|
||||
pub include_stop_str_in_output: bool,
|
||||
|
||||
/// Ignore end-of-sequence tokens during generation
|
||||
#[serde(default)]
|
||||
pub ignore_eos: bool,
|
||||
|
||||
/// Minimum number of tokens to generate
|
||||
#[validate(range(min = 1))]
|
||||
pub min_tokens: Option<u32>,
|
||||
|
||||
/// Skip special tokens during detokenization
|
||||
#[serde(default = "default_true")]
|
||||
pub skip_special_tokens: bool,
|
||||
|
||||
/// Add spaces between special tokens during detokenization
|
||||
#[serde(default = "default_true")]
|
||||
pub spaces_between_special_tokens: bool,
|
||||
|
||||
/// Truncate prompt tokens to this length
|
||||
pub truncate_prompt_tokens: Option<i64>,
|
||||
|
||||
/// Number of prompt logprobs to return
|
||||
pub prompt_logprobs: Option<i32>,
|
||||
|
||||
/// Restrict output to these token IDs only
|
||||
pub allowed_token_ids: Option<Vec<u32>>,
|
||||
|
||||
/// List of bad words to avoid during generation
|
||||
pub bad_words: Option<Vec<String>>,
|
||||
|
||||
// -------- Extra vLLM Parameters --------
|
||||
/// Token budget for reasoning/thinking
|
||||
pub thinking_token_budget: Option<u32>,
|
||||
|
||||
/// Whether to include reasoning content in the response
|
||||
#[serde(default = "default_true")]
|
||||
pub include_reasoning: bool,
|
||||
|
||||
/// If true, the new message will be prepended with the last message if they
|
||||
/// belong to the same role.
|
||||
#[serde(default)]
|
||||
pub echo: bool,
|
||||
|
||||
/// Whether to add the generation prompt to the chat template.
|
||||
///
|
||||
/// When omitted, the request follows the API default behavior, which is
|
||||
/// equivalent to `true` unless `continue_final_message=true` selects
|
||||
/// final assistant continuation instead.
|
||||
pub add_generation_prompt: Option<bool>,
|
||||
|
||||
/// Continue generating from final assistant message
|
||||
#[serde(default)]
|
||||
pub continue_final_message: bool,
|
||||
|
||||
/// Whether to add special tokens (e.g. BOS) to the prompt
|
||||
#[serde(default)]
|
||||
pub add_special_tokens: bool,
|
||||
|
||||
/// Documents for RAG (retrieval-augmented generation)
|
||||
pub documents: Option<Vec<Value>>,
|
||||
|
||||
/// Jinja chat template override
|
||||
pub chat_template: Option<String>,
|
||||
|
||||
/// Additional keyword args passed to the chat template renderer
|
||||
pub chat_template_kwargs: Option<HashMap<String, Value>>,
|
||||
|
||||
/// Additional kwargs for media IO connectors, keyed by modality
|
||||
pub media_io_kwargs: Option<HashMap<String, Value>>,
|
||||
|
||||
/// Additional kwargs for the HF processor
|
||||
pub mm_processor_kwargs: Option<HashMap<String, Value>>,
|
||||
|
||||
/// Additional kwargs for structured outputs
|
||||
pub structured_outputs: Option<Value>,
|
||||
|
||||
/// Request scheduling priority (lower means earlier; default 0)
|
||||
pub priority: Option<i32>,
|
||||
|
||||
/// External request ID used for response correlation.
|
||||
pub request_id: Option<String>,
|
||||
|
||||
/// Tokens represented as strings of the form 'token_id:{token_id}' in
|
||||
/// logprobs
|
||||
pub return_tokens_as_token_ids: Option<bool>,
|
||||
|
||||
/// Include token IDs alongside generated text
|
||||
pub return_token_ids: Option<bool>,
|
||||
|
||||
/// Salt for prefix cache isolation in multi-user environments
|
||||
pub cache_salt: Option<String>,
|
||||
|
||||
/// KV transfer parameters for disaggregated serving
|
||||
pub kv_transfer_params: Option<HashMap<String, Value>>,
|
||||
|
||||
/// Additional request parameters with string or numeric values for custom
|
||||
/// extensions
|
||||
pub vllm_xargs: Option<HashMap<String, Value>>,
|
||||
|
||||
/// Parameters for detecting repetitive N-gram patterns in output tokens
|
||||
pub repetition_detection: Option<Value>,
|
||||
}
|
||||
|
||||
impl Default for ChatCompletionRequest {
|
||||
#[expect(deprecated)]
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
messages: Vec::new(),
|
||||
model: default_model(),
|
||||
frequency_penalty: None,
|
||||
logit_bias: None,
|
||||
logprobs: false,
|
||||
top_logprobs: None,
|
||||
max_tokens: None,
|
||||
max_completion_tokens: None,
|
||||
n: None,
|
||||
presence_penalty: None,
|
||||
response_format: None,
|
||||
seed: None,
|
||||
stop: None,
|
||||
stream: false,
|
||||
stream_options: None,
|
||||
temperature: None,
|
||||
top_p: None,
|
||||
tools: None,
|
||||
tool_choice: None,
|
||||
reasoning_effort: None,
|
||||
thinking_token_budget: None,
|
||||
include_reasoning: true,
|
||||
parallel_tool_calls: None,
|
||||
user: None,
|
||||
use_beam_search: false,
|
||||
top_k: None,
|
||||
min_p: None,
|
||||
repetition_penalty: None,
|
||||
length_penalty: None,
|
||||
stop_token_ids: None,
|
||||
include_stop_str_in_output: false,
|
||||
ignore_eos: false,
|
||||
min_tokens: None,
|
||||
skip_special_tokens: true,
|
||||
spaces_between_special_tokens: true,
|
||||
truncate_prompt_tokens: None,
|
||||
prompt_logprobs: None,
|
||||
allowed_token_ids: None,
|
||||
bad_words: None,
|
||||
echo: false,
|
||||
add_generation_prompt: None,
|
||||
continue_final_message: false,
|
||||
add_special_tokens: false,
|
||||
documents: None,
|
||||
chat_template: None,
|
||||
chat_template_kwargs: None,
|
||||
media_io_kwargs: None,
|
||||
mm_processor_kwargs: None,
|
||||
structured_outputs: None,
|
||||
priority: None,
|
||||
request_id: None,
|
||||
return_tokens_as_token_ids: None,
|
||||
return_token_ids: None,
|
||||
cache_salt: None,
|
||||
kv_transfer_params: None,
|
||||
vllm_xargs: None,
|
||||
repetition_detection: None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Normalizable for ChatCompletionRequest {
|
||||
/// Normalize the request by applying migrations and defaults.
|
||||
fn normalize(&mut self) {
|
||||
// Migrate deprecated max_tokens → max_completion_tokens
|
||||
#[expect(deprecated)]
|
||||
if self.max_completion_tokens.is_none() && self.max_tokens.is_some() {
|
||||
self.max_completion_tokens = self.max_tokens;
|
||||
self.max_tokens = None;
|
||||
}
|
||||
|
||||
// Apply tool_choice defaults
|
||||
// If tools is None, leave tool_choice as None (don't set it)
|
||||
if self.tool_choice.is_none()
|
||||
&& let Some(tools) = &self.tools
|
||||
{
|
||||
let choice_value = if tools.is_empty() {
|
||||
ToolChoiceValue::None
|
||||
} else {
|
||||
ToolChoiceValue::Auto
|
||||
};
|
||||
self.tool_choice = Some(ToolChoice::Value(choice_value));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Mirrors the Python vLLM `ChatCompletionResponse` class.
|
||||
#[serde_with::skip_serializing_none]
|
||||
#[derive(Debug, Clone, Serialize)]
|
||||
pub(super) struct ChatCompletionResponse {
|
||||
pub id: String,
|
||||
pub object: String,
|
||||
pub created: u64,
|
||||
pub model: String,
|
||||
pub choices: Vec<ChatCompletionChoice>,
|
||||
pub usage: Option<Usage>,
|
||||
pub system_fingerprint: Option<String>,
|
||||
pub prompt_logprobs: Option<Vec<Option<HashMap<String, f32>>>>,
|
||||
pub prompt_token_ids: Option<Vec<u32>>,
|
||||
pub kv_transfer_params: Option<Value>,
|
||||
}
|
||||
|
||||
/// Mirrors the Python vLLM `ChatCompletionResponseChoice` class.
|
||||
#[serde_with::skip_serializing_none]
|
||||
#[derive(Debug, Clone, Serialize)]
|
||||
pub(super) struct ChatCompletionChoice {
|
||||
pub index: u32,
|
||||
pub message: ChatCompletionMessage,
|
||||
pub logprobs: Option<ChatLogProbs>,
|
||||
pub finish_reason: Option<String>,
|
||||
pub stop_reason: Option<Value>,
|
||||
pub token_ids: Option<Vec<u32>>,
|
||||
}
|
||||
|
||||
/// A literal type for the "assistant" role, since the API only allows that
|
||||
/// specific value in responses.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, SerializeDisplay)]
|
||||
pub(super) struct AssistantRole;
|
||||
|
||||
impl fmt::Display for AssistantRole {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
f.write_str("assistant")
|
||||
}
|
||||
}
|
||||
|
||||
/// Mirrors the Python vLLM response `ChatMessage` class.
|
||||
#[serde_with::skip_serializing_none]
|
||||
#[derive(Debug, Clone, Serialize)]
|
||||
pub(super) struct ChatCompletionMessage {
|
||||
pub role: AssistantRole,
|
||||
pub content: Option<String>,
|
||||
pub tool_calls: Option<Vec<ToolCall>>,
|
||||
pub reasoning: Option<String>,
|
||||
}
|
||||
|
||||
/// Mirrors the Python vLLM `ChatCompletionStreamResponse` class.
|
||||
#[serde_with::skip_serializing_none]
|
||||
#[derive(Debug, Clone, Serialize)]
|
||||
pub(super) struct ChatCompletionStreamResponse {
|
||||
pub id: String,
|
||||
pub object: String,
|
||||
pub created: u64,
|
||||
pub model: String,
|
||||
pub choices: Vec<ChatCompletionStreamChoice>,
|
||||
pub usage: Option<Usage>,
|
||||
pub prompt_token_ids: Option<Vec<u32>>,
|
||||
}
|
||||
|
||||
impl ChatCompletionStreamResponse {
|
||||
/// Create a stream response with the standard envelope fields pre-filled.
|
||||
pub fn new(id: &str, model: &str, created: u64) -> Self {
|
||||
Self {
|
||||
id: id.to_string(),
|
||||
object: "chat.completion.chunk".to_string(),
|
||||
created,
|
||||
model: model.to_string(),
|
||||
choices: Vec::new(),
|
||||
usage: None,
|
||||
prompt_token_ids: None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Mirrors the Python vLLM `ChatCompletionResponseStreamChoice` class.
|
||||
#[serde_with::skip_serializing_none]
|
||||
#[derive(Debug, Clone, Default, Serialize)]
|
||||
pub(super) struct ChatCompletionStreamChoice {
|
||||
pub index: u32,
|
||||
pub delta: ChatMessageDelta,
|
||||
pub logprobs: Option<ChatLogProbs>,
|
||||
pub finish_reason: Option<String>,
|
||||
pub stop_reason: Option<Value>,
|
||||
pub token_ids: Option<Vec<u32>>,
|
||||
}
|
||||
|
||||
/// Mirrors the Python vLLM `DeltaMessage` class.
|
||||
#[serde_with::skip_serializing_none]
|
||||
#[derive(Debug, Clone, Default, Serialize)]
|
||||
pub(super) struct ChatMessageDelta {
|
||||
pub role: Option<AssistantRole>,
|
||||
pub content: Option<String>,
|
||||
pub tool_calls: Option<Vec<ToolCallDelta>>,
|
||||
pub reasoning: Option<String>,
|
||||
}
|
||||
|
||||
fn default_model() -> String {
|
||||
UNKNOWN_MODEL_ID.to_string()
|
||||
}
|
||||
|
||||
/// Validates messages array is not empty and has valid content
|
||||
fn validate_messages(messages: &[ChatMessage]) -> Result<(), validator::ValidationError> {
|
||||
if messages.is_empty() {
|
||||
return Err(validator::ValidationError::new("messages cannot be empty"));
|
||||
}
|
||||
|
||||
for msg in messages {
|
||||
if let ChatMessage::User { content, .. } = msg {
|
||||
match content {
|
||||
MessageContent::Text(text) if text.is_empty() => {
|
||||
return Err(validator::ValidationError::new(
|
||||
"message content cannot be empty",
|
||||
));
|
||||
}
|
||||
MessageContent::Parts(parts) if parts.is_empty() => {
|
||||
return Err(validator::ValidationError::new(
|
||||
"message content parts cannot be empty",
|
||||
));
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Schema-level validation for cross-field dependencies
|
||||
fn validate_chat_cross_parameters(
|
||||
req: &ChatCompletionRequest,
|
||||
) -> Result<(), validator::ValidationError> {
|
||||
// 1. Validate logprobs dependency
|
||||
if req.top_logprobs.is_some() && !req.logprobs {
|
||||
let mut e = validator::ValidationError::new("top_logprobs_requires_logprobs");
|
||||
e.message = Some("top_logprobs is only allowed when logprobs is enabled".into());
|
||||
return Err(e);
|
||||
}
|
||||
|
||||
// 2. Validate stream_options dependency
|
||||
if req.stream_options.is_some() && !req.stream {
|
||||
let mut e = validator::ValidationError::new("stream_options_requires_stream");
|
||||
e.message = Some("stream_options can only be used when stream is true".into());
|
||||
return Err(e);
|
||||
}
|
||||
|
||||
// 3. Validate token limits - min <= max
|
||||
if let (Some(min_tokens), Some(max_completion_tokens)) =
|
||||
(req.min_tokens, req.max_completion_tokens)
|
||||
&& min_tokens > max_completion_tokens
|
||||
{
|
||||
let mut e = validator::ValidationError::new("min_tokens_exceeds_max_completion_tokens");
|
||||
e.message = Some("min_tokens cannot be greater than max_completion_tokens".into());
|
||||
return Err(e);
|
||||
}
|
||||
|
||||
#[expect(deprecated, reason = "Local type still mirrors legacy upstream field")]
|
||||
if let (Some(min_tokens), Some(max_tokens)) = (req.min_tokens, req.max_tokens)
|
||||
&& min_tokens > max_tokens
|
||||
{
|
||||
let mut e = validator::ValidationError::new("min_tokens_exceeds_max_tokens");
|
||||
e.message = Some("min_tokens cannot be greater than max_tokens".into());
|
||||
return Err(e);
|
||||
}
|
||||
|
||||
// 4. Validate response format JSON schema name
|
||||
if let Some(ResponseFormat::JsonSchema { json_schema }) = &req.response_format
|
||||
&& json_schema.name.is_empty()
|
||||
{
|
||||
let mut e = validator::ValidationError::new("json_schema_name_empty");
|
||||
e.message = Some("JSON schema name cannot be empty".into());
|
||||
return Err(e);
|
||||
}
|
||||
|
||||
// 5. Validate tool_choice requires tools (except for "none")
|
||||
if let Some(ref tool_choice) = req.tool_choice {
|
||||
let has_tools = req.tools.as_ref().is_some_and(|t| !t.is_empty());
|
||||
|
||||
// Check if tool_choice is anything other than "none"
|
||||
let is_some_choice = !matches!(tool_choice, ToolChoice::Value(ToolChoiceValue::None));
|
||||
|
||||
if is_some_choice && !has_tools {
|
||||
let mut e = validator::ValidationError::new("tool_choice_requires_tools");
|
||||
e.message = Some("Invalid value for 'tool_choice': 'tool_choice' is only allowed when 'tools' are specified.".into());
|
||||
return Err(e);
|
||||
}
|
||||
|
||||
// Additional validation when tools are present
|
||||
if let Some(tools) = req.tools.as_ref().filter(|t| !t.is_empty()) {
|
||||
match tool_choice {
|
||||
ToolChoice::Function { function, .. } => {
|
||||
// Validate that the specified function name exists in tools
|
||||
let function_exists = tools.iter().any(|tool| {
|
||||
tool.tool_type == "function" && tool.function.name == function.name
|
||||
});
|
||||
|
||||
if !function_exists {
|
||||
let mut e =
|
||||
validator::ValidationError::new("tool_choice_function_not_found");
|
||||
e.message = Some(
|
||||
format!(
|
||||
"Invalid value for 'tool_choice': function '{}' not found in 'tools'.",
|
||||
function.name
|
||||
)
|
||||
.into(),
|
||||
);
|
||||
return Err(e);
|
||||
}
|
||||
}
|
||||
ToolChoice::AllowedTools {
|
||||
mode,
|
||||
tools: allowed_tools,
|
||||
..
|
||||
} => {
|
||||
// Validate mode is "auto" or "required"
|
||||
if mode != "auto" && mode != "required" {
|
||||
let mut e = validator::ValidationError::new("tool_choice_invalid_mode");
|
||||
e.message = Some(format!(
|
||||
"Invalid value for 'tool_choice.mode': must be 'auto' or 'required', got '{mode}'."
|
||||
).into());
|
||||
return Err(e);
|
||||
}
|
||||
|
||||
// Validate that all ToolReferences are Function type (Chat API only supports
|
||||
// function tools)
|
||||
for tool_ref in allowed_tools {
|
||||
match tool_ref {
|
||||
ToolReference::Function { name } => {
|
||||
// Validate that the function exists in tools array
|
||||
let tool_exists = tools.iter().any(|tool| {
|
||||
tool.tool_type == "function" && tool.function.name == *name
|
||||
});
|
||||
|
||||
if !tool_exists {
|
||||
let mut e = validator::ValidationError::new(
|
||||
"tool_choice_tool_not_found",
|
||||
);
|
||||
e.message = Some(
|
||||
format!(
|
||||
"Invalid value for 'tool_choice.tools': tool '{name}' not found in 'tools'."
|
||||
)
|
||||
.into(),
|
||||
);
|
||||
return Err(e);
|
||||
}
|
||||
}
|
||||
_ => {
|
||||
// Chat Completion API only supports function tools in tool_choice
|
||||
let mut e = validator::ValidationError::new(
|
||||
"tool_choice_invalid_tool_type",
|
||||
);
|
||||
e.message = Some(
|
||||
format!(
|
||||
"Invalid value for 'tool_choice.tools': Chat Completion API only supports function tools, got '{}'.",
|
||||
tool_ref.identifier()
|
||||
)
|
||||
.into(),
|
||||
);
|
||||
return Err(e);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
ToolChoice::Value(_) => {}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
@@ -0,0 +1,410 @@
|
||||
use super::types::ChatCompletionRequest;
|
||||
use crate::error::{ApiError, bail_invalid_request};
|
||||
use crate::routes::openai::utils::types::{ChatMessage, Tool, ToolChoice, ToolChoiceValue};
|
||||
|
||||
/// Enforce the minimal compatibility contract for the Rust OpenAI server.
|
||||
pub(super) fn validate_request_compat(
|
||||
request: &ChatCompletionRequest,
|
||||
served_model_names: &[String],
|
||||
) -> Result<(), ApiError> {
|
||||
if !served_model_names.iter().any(|n| n == &request.model) {
|
||||
return Err(ApiError::model_not_found(request.model.clone()));
|
||||
}
|
||||
|
||||
if request.stream_options.is_some() && !request.stream {
|
||||
bail_invalid_request!(
|
||||
param = "stream_options",
|
||||
"stream_options are only supported when stream=true."
|
||||
);
|
||||
}
|
||||
|
||||
if request.n.unwrap_or(1) > 1 {
|
||||
bail_invalid_request!(param = "n", "Only n=1 is supported.");
|
||||
}
|
||||
|
||||
if request.top_logprobs.is_some() && !request.logprobs {
|
||||
bail_invalid_request!(
|
||||
param = "top_logprobs",
|
||||
"top_logprobs can only be used when logprobs=true."
|
||||
);
|
||||
}
|
||||
|
||||
if let Some(prompt_logprobs) = request.prompt_logprobs {
|
||||
if prompt_logprobs < 0 && prompt_logprobs != -1 {
|
||||
bail_invalid_request!(
|
||||
param = "prompt_logprobs",
|
||||
"prompt_logprobs must be a non-negative value or -1."
|
||||
);
|
||||
}
|
||||
|
||||
if request.stream && (prompt_logprobs > 0 || prompt_logprobs == -1) {
|
||||
bail_invalid_request!(
|
||||
param = "prompt_logprobs",
|
||||
"prompt_logprobs are not available when stream=true."
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
if let Some(tools) = request.tools.as_ref() {
|
||||
validate_function_tools(tools, "tools")?;
|
||||
}
|
||||
|
||||
for message in &request.messages {
|
||||
if let ChatMessage::Developer {
|
||||
tools: Some(tools), ..
|
||||
} = message
|
||||
{
|
||||
validate_function_tools(tools, "messages[].tools")?;
|
||||
}
|
||||
}
|
||||
|
||||
if let Some(tool_choice) = &request.tool_choice {
|
||||
match tool_choice {
|
||||
ToolChoice::Value(ToolChoiceValue::Auto | ToolChoiceValue::None) => {}
|
||||
ToolChoice::Value(ToolChoiceValue::Required) => {
|
||||
bail_invalid_request!(
|
||||
param = "tool_choice",
|
||||
"tool_choice=required is not supported yet."
|
||||
);
|
||||
}
|
||||
ToolChoice::Function { .. } => {
|
||||
bail_invalid_request!(
|
||||
param = "tool_choice",
|
||||
"Named function tool_choice is not supported yet."
|
||||
);
|
||||
}
|
||||
ToolChoice::AllowedTools { .. } => {
|
||||
bail_invalid_request!(
|
||||
param = "tool_choice",
|
||||
"allowed_tools tool_choice is not supported yet."
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if request.use_beam_search {
|
||||
bail_invalid_request!(
|
||||
param = "use_beam_search",
|
||||
"use_beam_search is not supported."
|
||||
);
|
||||
}
|
||||
|
||||
// ---- Reject parameters that are accepted for deserialization but not yet
|
||||
// implemented ----
|
||||
|
||||
if request.parallel_tool_calls.is_some() {
|
||||
bail_invalid_request!(
|
||||
param = "parallel_tool_calls",
|
||||
"parallel_tool_calls is not supported."
|
||||
);
|
||||
}
|
||||
|
||||
reject_non_default(
|
||||
request.length_penalty.as_ref(),
|
||||
"length_penalty",
|
||||
"length_penalty is not supported.",
|
||||
)?;
|
||||
if !request.spaces_between_special_tokens {
|
||||
bail_invalid_request!(
|
||||
param = "spaces_between_special_tokens",
|
||||
"spaces_between_special_tokens is not supported."
|
||||
);
|
||||
}
|
||||
reject_non_default(
|
||||
request.truncate_prompt_tokens.as_ref(),
|
||||
"truncate_prompt_tokens",
|
||||
"truncate_prompt_tokens is not supported.",
|
||||
)?;
|
||||
reject_non_default(
|
||||
request.thinking_token_budget.as_ref(),
|
||||
"thinking_token_budget",
|
||||
"thinking_token_budget is not supported.",
|
||||
)?;
|
||||
if !request.include_reasoning {
|
||||
bail_invalid_request!(
|
||||
param = "include_reasoning",
|
||||
"include_reasoning is not supported."
|
||||
);
|
||||
}
|
||||
reject_non_default(
|
||||
request.media_io_kwargs.as_ref(),
|
||||
"media_io_kwargs",
|
||||
"media_io_kwargs is not supported.",
|
||||
)?;
|
||||
reject_non_default(
|
||||
request.mm_processor_kwargs.as_ref(),
|
||||
"mm_processor_kwargs",
|
||||
"mm_processor_kwargs is not supported.",
|
||||
)?;
|
||||
reject_non_default(
|
||||
request.repetition_detection.as_ref(),
|
||||
"repetition_detection",
|
||||
"repetition_detection is not supported.",
|
||||
)?;
|
||||
|
||||
if let Some(options) = &request.stream_options
|
||||
&& options.continuous_usage_stats.is_some()
|
||||
{
|
||||
bail_invalid_request!(
|
||||
param = "stream_options",
|
||||
"continuous_usage_stats is not supported."
|
||||
);
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Reject one option unless it is entirely absent.
|
||||
fn reject_non_default<T>(
|
||||
value: Option<&T>,
|
||||
param: &'static str,
|
||||
message: &str,
|
||||
) -> Result<(), ApiError> {
|
||||
if value.is_some() {
|
||||
bail_invalid_request!(param = param, "{}", message);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn validate_function_tools(tools: &[Tool], param: &'static str) -> Result<(), ApiError> {
|
||||
for tool in tools {
|
||||
if tool.tool_type != "function" {
|
||||
bail_invalid_request!(param = param, "Only function tools are supported.");
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use std::collections::HashMap;
|
||||
|
||||
use serde_json::json;
|
||||
use vllm_chat::ReasoningEffort;
|
||||
|
||||
use super::validate_request_compat;
|
||||
use crate::routes::openai::chat_completions::types::ChatCompletionRequest;
|
||||
use crate::routes::openai::utils::structured_outputs::ResponseFormat;
|
||||
use crate::routes::openai::utils::types::{
|
||||
ChatMessage, Function, FunctionChoice, MessageContent, StringOrArray, Tool, ToolChoice,
|
||||
ToolChoiceValue, ToolReference,
|
||||
};
|
||||
|
||||
fn served(names: &[&str]) -> Vec<String> {
|
||||
names.iter().map(|s| s.to_string()).collect()
|
||||
}
|
||||
|
||||
fn base_request() -> ChatCompletionRequest {
|
||||
ChatCompletionRequest {
|
||||
model: "Qwen/Qwen1.5-0.5B-Chat".to_string(),
|
||||
messages: vec![ChatMessage::User {
|
||||
content: MessageContent::Text("hello".to_string()),
|
||||
name: None,
|
||||
}],
|
||||
stream: true,
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn validate_request_compat_accepts_stop() {
|
||||
let request = ChatCompletionRequest {
|
||||
stop: Some(StringOrArray::String("stop".to_string())),
|
||||
..base_request()
|
||||
};
|
||||
|
||||
validate_request_compat(&request, &served(&["Qwen/Qwen1.5-0.5B-Chat"]))
|
||||
.expect("stop strings should be accepted");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn validate_request_compat_accepts_non_zero_penalties_and_function_tools() {
|
||||
let request = ChatCompletionRequest {
|
||||
frequency_penalty: Some(0.5),
|
||||
presence_penalty: Some(0.25),
|
||||
min_p: Some(0.2),
|
||||
repetition_penalty: Some(1.1),
|
||||
seed: Some(7),
|
||||
..base_request()
|
||||
};
|
||||
validate_request_compat(&request, &served(&["Qwen/Qwen1.5-0.5B-Chat"]))
|
||||
.expect("sampling fields should be accepted");
|
||||
|
||||
let request = ChatCompletionRequest {
|
||||
tools: Some(vec![Tool {
|
||||
tool_type: "function".to_string(),
|
||||
function: Function {
|
||||
name: "tool".to_string(),
|
||||
description: None,
|
||||
parameters: json!({}),
|
||||
strict: None,
|
||||
},
|
||||
}]),
|
||||
..base_request()
|
||||
};
|
||||
validate_request_compat(&request, &served(&["Qwen/Qwen1.5-0.5B-Chat"]))
|
||||
.expect("function tools should be accepted");
|
||||
|
||||
let request = ChatCompletionRequest {
|
||||
messages: vec![ChatMessage::Developer {
|
||||
content: MessageContent::Text("policy".to_string()),
|
||||
tools: Some(vec![Tool {
|
||||
tool_type: "function".to_string(),
|
||||
function: Function {
|
||||
name: "tool".to_string(),
|
||||
description: None,
|
||||
parameters: json!({}),
|
||||
strict: None,
|
||||
},
|
||||
}]),
|
||||
name: None,
|
||||
}],
|
||||
..base_request()
|
||||
};
|
||||
validate_request_compat(&request, &served(&["Qwen/Qwen1.5-0.5B-Chat"]))
|
||||
.expect("developer function tools should be accepted");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn validate_request_compat_rejects_non_function_developer_tools() {
|
||||
let request = ChatCompletionRequest {
|
||||
messages: vec![ChatMessage::Developer {
|
||||
content: MessageContent::Text("policy".to_string()),
|
||||
tools: Some(vec![Tool {
|
||||
tool_type: "mcp".to_string(),
|
||||
function: Function {
|
||||
name: "tool".to_string(),
|
||||
description: None,
|
||||
parameters: json!({}),
|
||||
strict: None,
|
||||
},
|
||||
}]),
|
||||
name: None,
|
||||
}],
|
||||
..base_request()
|
||||
};
|
||||
|
||||
assert!(validate_request_compat(&request, &served(&["Qwen/Qwen1.5-0.5B-Chat"])).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn validate_request_compat_accepts_output_logprobs() {
|
||||
let request = ChatCompletionRequest {
|
||||
logprobs: true,
|
||||
..base_request()
|
||||
};
|
||||
validate_request_compat(&request, &served(&["Qwen/Qwen1.5-0.5B-Chat"]))
|
||||
.expect("logprobs should be accepted");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn validate_request_compat_accepts_reasoning_effort() {
|
||||
let request = ChatCompletionRequest {
|
||||
reasoning_effort: Some(ReasoningEffort::Max),
|
||||
chat_template_kwargs: Some(HashMap::from([(
|
||||
"reasoning_effort".to_string(),
|
||||
json!("low"),
|
||||
)])),
|
||||
..base_request()
|
||||
};
|
||||
|
||||
validate_request_compat(&request, &served(&["Qwen/Qwen1.5-0.5B-Chat"]))
|
||||
.expect("reasoning_effort should be accepted");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn validate_request_compat_rejects_top_logprobs_without_logprobs() {
|
||||
let request = ChatCompletionRequest {
|
||||
top_logprobs: Some(0),
|
||||
..base_request()
|
||||
};
|
||||
assert!(validate_request_compat(&request, &served(&["Qwen/Qwen1.5-0.5B-Chat"])).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn validate_request_compat_rejects_streaming_prompt_logprobs_requests() {
|
||||
let request = ChatCompletionRequest {
|
||||
prompt_logprobs: Some(1),
|
||||
..base_request()
|
||||
};
|
||||
assert!(validate_request_compat(&request, &served(&["Qwen/Qwen1.5-0.5B-Chat"])).is_err());
|
||||
|
||||
let request = ChatCompletionRequest {
|
||||
prompt_logprobs: Some(-1),
|
||||
..base_request()
|
||||
};
|
||||
assert!(validate_request_compat(&request, &served(&["Qwen/Qwen1.5-0.5B-Chat"])).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn validate_request_compat_rejects_invalid_prompt_logprobs_value() {
|
||||
let request = ChatCompletionRequest {
|
||||
stream: false,
|
||||
prompt_logprobs: Some(-2),
|
||||
..base_request()
|
||||
};
|
||||
assert!(validate_request_compat(&request, &served(&["Qwen/Qwen1.5-0.5B-Chat"])).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn validate_request_compat_accepts_response_format() {
|
||||
let request = ChatCompletionRequest {
|
||||
response_format: Some(ResponseFormat::Text),
|
||||
..base_request()
|
||||
};
|
||||
validate_request_compat(&request, &served(&["Qwen/Qwen1.5-0.5B-Chat"]))
|
||||
.expect("response_format=text should be accepted");
|
||||
|
||||
let request = ChatCompletionRequest {
|
||||
response_format: Some(ResponseFormat::JsonObject),
|
||||
..base_request()
|
||||
};
|
||||
validate_request_compat(&request, &served(&["Qwen/Qwen1.5-0.5B-Chat"]))
|
||||
.expect("response_format=json_object should be accepted");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn validate_request_compat_accepts_noop_tool_choice_none() {
|
||||
let request = ChatCompletionRequest {
|
||||
tool_choice: Some(ToolChoice::Value(ToolChoiceValue::None)),
|
||||
..base_request()
|
||||
};
|
||||
|
||||
validate_request_compat(&request, &served(&["Qwen/Qwen1.5-0.5B-Chat"]))
|
||||
.expect("tool_choice=none is ok");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn validate_request_compat_rejects_required_and_named_tool_choices() {
|
||||
let required = ChatCompletionRequest {
|
||||
tool_choice: Some(ToolChoice::Value(ToolChoiceValue::Required)),
|
||||
..base_request()
|
||||
};
|
||||
assert!(validate_request_compat(&required, &served(&["Qwen/Qwen1.5-0.5B-Chat"])).is_err());
|
||||
|
||||
let named = ChatCompletionRequest {
|
||||
tool_choice: Some(ToolChoice::Function {
|
||||
tool_type: "function".to_string(),
|
||||
function: FunctionChoice {
|
||||
name: "tool".to_string(),
|
||||
},
|
||||
}),
|
||||
..base_request()
|
||||
};
|
||||
assert!(validate_request_compat(&named, &served(&["Qwen/Qwen1.5-0.5B-Chat"])).is_err());
|
||||
|
||||
let allowed_tools = ChatCompletionRequest {
|
||||
tool_choice: Some(ToolChoice::AllowedTools {
|
||||
tool_type: "allowed_tools".to_string(),
|
||||
mode: "auto".to_string(),
|
||||
tools: vec![ToolReference::Function {
|
||||
name: "tool".to_string(),
|
||||
}],
|
||||
}),
|
||||
..base_request()
|
||||
};
|
||||
assert!(
|
||||
validate_request_compat(&allowed_tools, &served(&["Qwen/Qwen1.5-0.5B-Chat"])).is_err()
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,571 @@
|
||||
mod convert;
|
||||
mod types;
|
||||
mod validate;
|
||||
|
||||
use std::convert::Infallible;
|
||||
use std::result::Result;
|
||||
use std::sync::Arc;
|
||||
|
||||
use asynk_strim_attr::{TryYielder, try_stream};
|
||||
use axum::Json;
|
||||
use axum::extract::State;
|
||||
use axum::http::HeaderMap;
|
||||
use axum::response::sse::{Event, Sse};
|
||||
use axum::response::{IntoResponse, Response};
|
||||
use futures::{Stream, StreamExt as _, pin_mut};
|
||||
use thiserror_ext::AsReport as _;
|
||||
use tracing::{debug, error, info, trace};
|
||||
use tracing_futures::Instrument as _;
|
||||
use vllm_text::{DecodedTextEvent, FinishReason, TextOutputStream, TextOutputStreamExt as _};
|
||||
|
||||
use super::utils::logprobs::{
|
||||
collected_logprobs_to_openai, decoded_logprobs_to_openai, decoded_prompt_logprobs_to_maps,
|
||||
text_len,
|
||||
};
|
||||
use super::utils::types::Usage;
|
||||
use crate::error::{ApiError, bail_server_error, server_error};
|
||||
use crate::routes::openai::completions::convert::prepare_completion_request;
|
||||
use crate::routes::openai::completions::types::{
|
||||
CompletionChoice, CompletionRequest, CompletionResponse, CompletionSseChunk,
|
||||
CompletionStreamChoice, CompletionStreamResponse,
|
||||
};
|
||||
use crate::routes::openai::utils::types::LogProbs;
|
||||
use crate::routes::openai::utils::validated_json::ValidatedJson;
|
||||
use crate::state::AppState;
|
||||
use crate::utils::{resolve_request_context, unix_timestamp};
|
||||
|
||||
/// Validate one completions request and proxy it into the shared `vllm-text`
|
||||
/// stack.
|
||||
pub async fn completions(
|
||||
State(state): State<Arc<AppState>>,
|
||||
headers: HeaderMap,
|
||||
ValidatedJson(body): ValidatedJson<CompletionRequest>,
|
||||
) -> Response {
|
||||
let stream = body.stream;
|
||||
let logprobs = body.logprobs;
|
||||
let request_context = resolve_request_context(&headers, body.request_id.as_deref());
|
||||
|
||||
let prepared =
|
||||
match prepare_completion_request(body, state.served_model_names(), request_context) {
|
||||
Ok(prepared) => prepared,
|
||||
Err(error) => return error.into_response(),
|
||||
};
|
||||
let request_span = tracing::info_span!(
|
||||
"completions",
|
||||
request_id = %prepared.request_id,
|
||||
engine_request_id = tracing::field::Empty,
|
||||
);
|
||||
|
||||
let created = unix_timestamp();
|
||||
let include_prompt_logprobs = prepared.text_request.sampling_params.prompt_logprobs.is_some();
|
||||
let log_request = state.enable_log_requests;
|
||||
|
||||
let text_stream = match state
|
||||
.chat
|
||||
.text()
|
||||
.generate(prepared.text_request)
|
||||
.instrument(request_span.clone())
|
||||
.await
|
||||
{
|
||||
Ok(stream) => stream,
|
||||
Err(error) => {
|
||||
return server_error!(
|
||||
"failed to submit completion request: {}",
|
||||
error.to_report_string()
|
||||
)
|
||||
.into_response();
|
||||
}
|
||||
};
|
||||
|
||||
if stream {
|
||||
let chunk_stream = completion_chunk_stream(
|
||||
text_stream,
|
||||
prepared.request_id,
|
||||
prepared.response_model,
|
||||
created,
|
||||
log_request,
|
||||
prepared.include_usage,
|
||||
prepared.echo,
|
||||
logprobs,
|
||||
prepared.return_token_ids,
|
||||
prepared.return_tokens_as_token_ids,
|
||||
);
|
||||
let sse_stream = completion_sse_stream(chunk_stream).instrument(request_span);
|
||||
|
||||
Sse::new(sse_stream).into_response()
|
||||
} else {
|
||||
let response = match collect_completion(
|
||||
text_stream,
|
||||
prepared.request_id,
|
||||
prepared.response_model,
|
||||
created,
|
||||
prepared.echo,
|
||||
logprobs,
|
||||
include_prompt_logprobs,
|
||||
prepared.return_token_ids,
|
||||
prepared.return_tokens_as_token_ids,
|
||||
)
|
||||
.instrument(request_span.clone())
|
||||
.await
|
||||
{
|
||||
Ok(response) => response,
|
||||
Err(error) => return error.into_response(),
|
||||
};
|
||||
|
||||
if log_request {
|
||||
let usage = response.usage.as_ref();
|
||||
info!(
|
||||
parent: &request_span,
|
||||
model = %response.model,
|
||||
prompt_tokens = usage.map_or(0, |u| u.prompt_tokens),
|
||||
output_tokens = usage.and_then(|u| u.completion_tokens).unwrap_or(0),
|
||||
finish_reason = response.choices.first().and_then(|c| c.finish_reason.as_deref()).unwrap_or("unknown"),
|
||||
"completion finished"
|
||||
);
|
||||
}
|
||||
|
||||
Json(response).into_response()
|
||||
}
|
||||
}
|
||||
|
||||
async fn collect_completion(
|
||||
stream: impl TextOutputStream,
|
||||
request_id: String,
|
||||
response_model: String,
|
||||
created: u64,
|
||||
echo: Option<String>,
|
||||
requested_logprobs: Option<u32>,
|
||||
include_prompt_logprobs: bool,
|
||||
return_token_ids: bool,
|
||||
return_tokens_as_token_ids: bool,
|
||||
) -> Result<CompletionResponse, ApiError> {
|
||||
let collected = stream
|
||||
.collect_output()
|
||||
.await
|
||||
.map_err(|error| server_error!("completion stream failed: {}", error.to_report_string()))?;
|
||||
let finish_reason = collected.finish_reason.clone();
|
||||
let stop_reason = finish_reason
|
||||
.as_stop_reason()
|
||||
.map(|sr| serde_json::to_value(sr).expect("StopReason must serialize to JSON"));
|
||||
|
||||
let prompt_char_count = echo.as_ref().map(|prompt| text_len(prompt)).unwrap_or_default();
|
||||
let prompt_logprobs = if include_prompt_logprobs {
|
||||
let prompt_logprobs = collected.prompt_logprobs.as_ref().ok_or_else(|| {
|
||||
server_error!(
|
||||
"completion response requested prompt_logprobs but generation returned none"
|
||||
)
|
||||
})?;
|
||||
Some(prompt_logprobs)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let logprobs = if requested_logprobs.is_some() {
|
||||
Some(collected_logprobs_to_openai(
|
||||
&collected,
|
||||
echo.is_some(),
|
||||
prompt_char_count,
|
||||
return_tokens_as_token_ids,
|
||||
)?)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let prompt_logprobs =
|
||||
prompt_logprobs.map(|lp| decoded_prompt_logprobs_to_maps(lp, return_tokens_as_token_ids));
|
||||
let text = match &echo {
|
||||
None => collected.text,
|
||||
Some(prompt) => format!("{prompt}{}", collected.text),
|
||||
};
|
||||
|
||||
Ok(CompletionResponse {
|
||||
id: request_id,
|
||||
object: "text_completion".to_string(),
|
||||
created,
|
||||
model: response_model,
|
||||
choices: vec![CompletionChoice {
|
||||
index: 0,
|
||||
text,
|
||||
logprobs,
|
||||
finish_reason: Some(completion_finish_reason_to_openai(finish_reason)?.into()),
|
||||
stop_reason,
|
||||
prompt_logprobs,
|
||||
token_ids: return_token_ids.then(|| collected.token_ids.clone()),
|
||||
prompt_token_ids: return_token_ids.then(|| collected.prompt_token_ids.to_vec()),
|
||||
}],
|
||||
usage: Some(Usage::from_counts(
|
||||
collected.prompt_token_ids.len() as u32,
|
||||
collected.token_ids.len() as u32,
|
||||
)),
|
||||
system_fingerprint: None,
|
||||
kv_transfer_params: collected.kv_transfer_params,
|
||||
})
|
||||
}
|
||||
|
||||
/// Convert one internal decoded-text stream into OpenAI completions chunks.
|
||||
#[try_stream]
|
||||
async fn completion_chunk_stream(
|
||||
stream: impl TextOutputStream,
|
||||
request_id: String,
|
||||
response_model: String,
|
||||
created: u64,
|
||||
log_request: bool,
|
||||
include_usage: bool,
|
||||
echo: Option<String>,
|
||||
requested_logprobs: Option<u32>,
|
||||
return_token_ids: bool,
|
||||
return_tokens_as_token_ids: bool,
|
||||
mut y: TryYielder<CompletionSseChunk, ApiError>,
|
||||
) -> Result<(), ApiError> {
|
||||
pin_mut!(stream);
|
||||
let mut visible_text_len = 0_u32;
|
||||
let mut first_chunk = true;
|
||||
|
||||
while let Some(next) = stream.next().await {
|
||||
match next {
|
||||
Ok(DecodedTextEvent::Start {
|
||||
prompt_token_ids, ..
|
||||
}) => {
|
||||
debug!("completion stream started");
|
||||
if let Some(prompt) = echo.as_ref() {
|
||||
visible_text_len = text_len(prompt);
|
||||
let mut chunk =
|
||||
delta_chunk(&request_id, &response_model, created, prompt.clone(), None);
|
||||
if return_token_ids && first_chunk {
|
||||
if let Some(choice) = chunk.choices.first_mut() {
|
||||
choice.prompt_token_ids = Some(prompt_token_ids.to_vec());
|
||||
}
|
||||
first_chunk = false;
|
||||
}
|
||||
y.yield_ok(CompletionSseChunk::Chunk(chunk)).await;
|
||||
} else if return_token_ids {
|
||||
// Emit a chunk with prompt_token_ids in the first streaming response
|
||||
let mut chunk =
|
||||
delta_chunk(&request_id, &response_model, created, String::new(), None);
|
||||
if let Some(choice) = chunk.choices.first_mut() {
|
||||
choice.prompt_token_ids = Some(prompt_token_ids.to_vec());
|
||||
}
|
||||
first_chunk = false;
|
||||
y.yield_ok(CompletionSseChunk::Chunk(chunk)).await;
|
||||
}
|
||||
}
|
||||
Ok(DecodedTextEvent::TextDelta {
|
||||
delta,
|
||||
token_ids,
|
||||
logprobs,
|
||||
finished,
|
||||
}) => {
|
||||
let delta_text_len = text_len(&delta);
|
||||
let logprobs = if requested_logprobs.is_some() {
|
||||
let decoded_logprobs = logprobs.as_ref().ok_or_else(|| {
|
||||
server_error!(
|
||||
"completion stream requested logprobs but generation returned none"
|
||||
)
|
||||
})?;
|
||||
Some(decoded_logprobs_to_openai(
|
||||
decoded_logprobs,
|
||||
visible_text_len,
|
||||
return_tokens_as_token_ids,
|
||||
)?)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let mut chunk = delta_chunk(&request_id, &response_model, created, delta, logprobs);
|
||||
if return_token_ids && let Some(choice) = chunk.choices.first_mut() {
|
||||
choice.token_ids = Some(token_ids);
|
||||
}
|
||||
y.yield_ok(CompletionSseChunk::Chunk(chunk)).await;
|
||||
visible_text_len = visible_text_len.saturating_add(delta_text_len);
|
||||
|
||||
if let Some(finished) = finished {
|
||||
if log_request {
|
||||
info!(
|
||||
stream = true,
|
||||
model = %response_model,
|
||||
prompt_tokens = finished.prompt_token_count,
|
||||
output_tokens = finished.output_token_count,
|
||||
finish_reason = finished.finish_reason.as_str(),
|
||||
"completion finished"
|
||||
);
|
||||
}
|
||||
y.yield_ok(CompletionSseChunk::Chunk(final_chunk(
|
||||
&request_id,
|
||||
&response_model,
|
||||
created,
|
||||
finished.finish_reason,
|
||||
)?))
|
||||
.await;
|
||||
|
||||
if include_usage {
|
||||
y.yield_ok(CompletionSseChunk::Usage(usage_chunk(
|
||||
&request_id,
|
||||
&response_model,
|
||||
created,
|
||||
Usage::from_counts(
|
||||
finished.prompt_token_count as u32,
|
||||
finished.output_token_count as u32,
|
||||
),
|
||||
)))
|
||||
.await;
|
||||
}
|
||||
}
|
||||
}
|
||||
Err(error) => {
|
||||
error!(
|
||||
error = %error.as_report(),
|
||||
"completion stream failed"
|
||||
);
|
||||
bail_server_error!("{}", error.to_report_string());
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn delta_chunk(
|
||||
request_id: &str,
|
||||
response_model: &str,
|
||||
created: u64,
|
||||
text: String,
|
||||
logprobs: Option<LogProbs>,
|
||||
) -> CompletionStreamResponse {
|
||||
let mut chunk = CompletionStreamResponse::new(request_id, response_model, created);
|
||||
chunk.choices.push(CompletionStreamChoice {
|
||||
text,
|
||||
logprobs,
|
||||
..Default::default()
|
||||
});
|
||||
chunk
|
||||
}
|
||||
|
||||
fn final_chunk(
|
||||
request_id: &str,
|
||||
response_model: &str,
|
||||
created: u64,
|
||||
finish_reason: FinishReason,
|
||||
) -> Result<CompletionStreamResponse, ApiError> {
|
||||
let finish_reason = completion_finish_reason_to_openai(finish_reason)?;
|
||||
|
||||
let mut chunk = CompletionStreamResponse::new(request_id, response_model, created);
|
||||
chunk.choices.push(CompletionStreamChoice {
|
||||
finish_reason: Some(finish_reason.to_string()),
|
||||
..Default::default()
|
||||
});
|
||||
Ok(chunk)
|
||||
}
|
||||
|
||||
fn completion_finish_reason_to_openai(
|
||||
finish_reason: FinishReason,
|
||||
) -> Result<&'static str, ApiError> {
|
||||
match finish_reason {
|
||||
FinishReason::Stop(_) | FinishReason::Repetition => Ok("stop"),
|
||||
FinishReason::Length => Ok("length"),
|
||||
FinishReason::Abort => Ok("abort"),
|
||||
FinishReason::Error => {
|
||||
bail_server_error!("Internal server error");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn usage_chunk(
|
||||
request_id: &str,
|
||||
response_model: &str,
|
||||
created: u64,
|
||||
usage: Usage,
|
||||
) -> CompletionStreamResponse {
|
||||
let mut chunk = CompletionStreamResponse::new(request_id, response_model, created);
|
||||
chunk.usage = Some(usage);
|
||||
chunk
|
||||
}
|
||||
|
||||
/// Convert one chunk stream into OpenAI-style SSE events.
|
||||
///
|
||||
/// OpenAI-style streaming errors are encoded as ordinary `data: {"error": ...}`
|
||||
/// events followed by `data: [DONE]`, so the transport stream itself stays
|
||||
/// infallible even when generation fails after the HTTP response has started.
|
||||
#[try_stream]
|
||||
async fn completion_sse_stream(
|
||||
stream: impl Stream<Item = Result<CompletionSseChunk, ApiError>>,
|
||||
mut y: TryYielder<Event, Infallible>,
|
||||
) -> Result<(), Infallible> {
|
||||
pin_mut!(stream);
|
||||
|
||||
while let Some(next) = stream.next().await {
|
||||
match next {
|
||||
Ok(chunk) => y.yield_ok(to_sse_event(&chunk)).await,
|
||||
Err(error) => {
|
||||
y.yield_ok(to_error_sse_event(&error)).await;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
y.yield_ok(done_sse_event()).await;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Serialize one OpenAI chunk payload into one SSE `data:` event.
|
||||
fn to_sse_event(chunk: &CompletionSseChunk) -> Event {
|
||||
let payload = serde_json::to_string(chunk).expect("completion chunk must serialize to JSON");
|
||||
trace!(payload, "completion emitting chunk");
|
||||
Event::default().data(payload)
|
||||
}
|
||||
|
||||
/// Serialize one OpenAI error payload into one SSE `data:` event.
|
||||
fn to_error_sse_event(error: &ApiError) -> Event {
|
||||
let payload = serde_json::to_string(&error.to_error_response())
|
||||
.expect("ErrorResponse must serialize to JSON");
|
||||
trace!(payload, "completion emitting error");
|
||||
Event::default().data(payload)
|
||||
}
|
||||
|
||||
/// Build the terminal OpenAI SSE sentinel event.
|
||||
fn done_sse_event() -> Event {
|
||||
trace!("completion emitting done");
|
||||
Event::default().data("[DONE]")
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use futures::{StreamExt as _, stream};
|
||||
use itertools::Itertools as _;
|
||||
use vllm_text::{
|
||||
DecodedLogprobs, DecodedPositionLogprobs, DecodedTextEvent, DecodedTokenLogprob,
|
||||
FinishReason, Finished,
|
||||
};
|
||||
|
||||
use super::{CompletionSseChunk, completion_chunk_stream, final_chunk};
|
||||
|
||||
#[test]
|
||||
fn final_chunk_maps_stop_finish_reason() {
|
||||
let chunk = final_chunk("cmpl-1", "model", 1, FinishReason::stop_eos())
|
||||
.expect("finish reason valid");
|
||||
assert_eq!(chunk.choices[0].finish_reason.as_deref(), Some("stop"));
|
||||
assert_eq!(chunk.choices[0].text, "");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn final_chunk_maps_length_finish_reason() {
|
||||
let chunk =
|
||||
final_chunk("cmpl-1", "model", 1, FinishReason::Length).expect("finish reason valid");
|
||||
assert_eq!(chunk.choices[0].finish_reason.as_deref(), Some("length"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn final_chunk_maps_abort_finish_reason() {
|
||||
let chunk =
|
||||
final_chunk("cmpl-1", "model", 1, FinishReason::Abort).expect("finish reason valid");
|
||||
assert_eq!(chunk.choices[0].finish_reason.as_deref(), Some("abort"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn final_chunk_rejects_error_finish_reason() {
|
||||
assert!(final_chunk("cmpl-1", "model", 1, FinishReason::Error).is_err());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn completion_chunk_stream_maps_streaming_logprobs() {
|
||||
let stream = stream::iter(vec![
|
||||
Ok(DecodedTextEvent::Start {
|
||||
prompt_token_ids: vec![1, 2, 3, 4, 5].into(),
|
||||
prompt_logprobs: None,
|
||||
}),
|
||||
Ok(DecodedTextEvent::TextDelta {
|
||||
delta: "h".to_string(),
|
||||
token_ids: vec![b'h' as u32],
|
||||
logprobs: Some(DecodedLogprobs {
|
||||
positions: vec![DecodedPositionLogprobs {
|
||||
entries: vec![
|
||||
DecodedTokenLogprob {
|
||||
token_id: 0,
|
||||
token: "h".to_string(),
|
||||
logprob: -0.1,
|
||||
rank: 1,
|
||||
},
|
||||
DecodedTokenLogprob {
|
||||
token_id: 0,
|
||||
token: "H".to_string(),
|
||||
logprob: -0.2,
|
||||
rank: 1,
|
||||
},
|
||||
],
|
||||
}],
|
||||
}),
|
||||
finished: None,
|
||||
}),
|
||||
Ok(DecodedTextEvent::TextDelta {
|
||||
delta: String::new(),
|
||||
token_ids: vec![b'!' as u32],
|
||||
logprobs: Some(DecodedLogprobs {
|
||||
positions: vec![DecodedPositionLogprobs {
|
||||
entries: vec![
|
||||
DecodedTokenLogprob {
|
||||
token_id: 0,
|
||||
token: "!".to_string(),
|
||||
logprob: -0.3,
|
||||
rank: 1,
|
||||
},
|
||||
DecodedTokenLogprob {
|
||||
token_id: 0,
|
||||
token: "?".to_string(),
|
||||
logprob: -0.4,
|
||||
rank: 1,
|
||||
},
|
||||
],
|
||||
}],
|
||||
}),
|
||||
finished: Some(Finished {
|
||||
prompt_token_count: 5,
|
||||
output_token_count: 2,
|
||||
finish_reason: FinishReason::stop_eos(),
|
||||
kv_transfer_params: None,
|
||||
}),
|
||||
}),
|
||||
]);
|
||||
|
||||
let chunks = completion_chunk_stream(
|
||||
stream,
|
||||
"cmpl-1".to_string(),
|
||||
"model".to_string(),
|
||||
1,
|
||||
false,
|
||||
false,
|
||||
None,
|
||||
Some(1),
|
||||
false,
|
||||
false,
|
||||
)
|
||||
.collect::<Vec<_>>()
|
||||
.await;
|
||||
|
||||
let chunks: Vec<_> = chunks.into_iter().try_collect().expect("stream should succeed");
|
||||
|
||||
match &chunks[0] {
|
||||
CompletionSseChunk::Chunk(chunk) => {
|
||||
assert_eq!(chunk.choices[0].text, "h");
|
||||
assert_eq!(
|
||||
chunk.choices[0].logprobs.as_ref().expect("logprobs").tokens,
|
||||
vec!["h".to_string()]
|
||||
);
|
||||
assert_eq!(
|
||||
chunk.choices[0].logprobs.as_ref().expect("logprobs").text_offset,
|
||||
vec![0]
|
||||
);
|
||||
}
|
||||
CompletionSseChunk::Usage(_) => panic!("expected regular chunk"),
|
||||
}
|
||||
|
||||
match &chunks[1] {
|
||||
CompletionSseChunk::Chunk(chunk) => {
|
||||
assert_eq!(chunk.choices[0].text, "");
|
||||
assert_eq!(
|
||||
chunk.choices[0].logprobs.as_ref().expect("logprobs").tokens,
|
||||
vec!["!".to_string()]
|
||||
);
|
||||
assert_eq!(
|
||||
chunk.choices[0].logprobs.as_ref().expect("logprobs").text_offset,
|
||||
vec![1]
|
||||
);
|
||||
}
|
||||
CompletionSseChunk::Usage(_) => panic!("expected regular chunk"),
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,352 @@
|
||||
use vllm_text::{SamplingParams, TextDecodeOptions, TextRequest};
|
||||
|
||||
use super::types::CompletionRequest;
|
||||
use crate::error::ApiError;
|
||||
use crate::routes::openai::completions::validate;
|
||||
use crate::routes::openai::utils::structured_outputs::convert_from_response_format_value;
|
||||
use crate::utils::{ResolvedRequestContext, convert_logit_bias, merge_kv_transfer_params};
|
||||
|
||||
/// Lowered completion request plus the public response metadata carried by
|
||||
/// every SSE chunk.
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
pub struct PreparedRequest {
|
||||
/// Stable OpenAI-style request ID, reused as the external text request ID.
|
||||
pub request_id: String,
|
||||
/// Public model ID echoed back to the client.
|
||||
pub response_model: String,
|
||||
/// Whether the caller asked for the final streamed usage chunk.
|
||||
pub include_usage: bool,
|
||||
/// Lowered text request for the shared `vllm-text` facade.
|
||||
pub text_request: TextRequest,
|
||||
/// Original text prompt that should be echoed back northbound when
|
||||
/// `echo=true`.
|
||||
pub echo: Option<String>,
|
||||
/// Whether to include token IDs alongside generated text.
|
||||
pub return_token_ids: bool,
|
||||
/// Whether to format logprob tokens as `token_id:{id}`.
|
||||
pub return_tokens_as_token_ids: bool,
|
||||
}
|
||||
|
||||
/// Validate and lower one OpenAI completions request into the internal
|
||||
/// text-generation format.
|
||||
///
|
||||
/// `served_model_names` must be non-empty; the first entry is used as the
|
||||
/// `model` field in responses.
|
||||
pub(crate) fn prepare_completion_request(
|
||||
request: CompletionRequest,
|
||||
served_model_names: &[String],
|
||||
ctx: ResolvedRequestContext,
|
||||
) -> Result<PreparedRequest, ApiError> {
|
||||
validate::validate_request_compat(&request, served_model_names)?;
|
||||
|
||||
let request_id = format!("cmpl-{}", ctx.request_id);
|
||||
|
||||
let logprobs = match request.logprobs {
|
||||
Some(logprobs) => Some(i32::try_from(logprobs).map_err(|_| {
|
||||
ApiError::invalid_request(
|
||||
"`logprobs` must fit within a signed 32-bit integer.".to_string(),
|
||||
Some("logprobs"),
|
||||
)
|
||||
})?),
|
||||
None => None,
|
||||
};
|
||||
let prompt_logprobs = request.prompt_logprobs.or(if request.echo && !request.stream {
|
||||
logprobs
|
||||
} else {
|
||||
None
|
||||
});
|
||||
let include_usage = (request.stream_options.as_ref())
|
||||
.and_then(|options| options.include_usage)
|
||||
.unwrap_or(false);
|
||||
let echo = request.echo.then(|| request.prompt.as_text().cloned()).flatten();
|
||||
|
||||
let structured_outputs =
|
||||
convert_from_response_format_value(&request.response_format, &request.structured_outputs)?;
|
||||
|
||||
let text_request = TextRequest {
|
||||
request_id: request_id.clone(),
|
||||
prompt: request.prompt,
|
||||
mm_features: None,
|
||||
sampling_params: SamplingParams {
|
||||
temperature: request.temperature,
|
||||
top_p: request.top_p,
|
||||
top_k: request.top_k,
|
||||
seed: request.seed,
|
||||
max_tokens: request.max_tokens,
|
||||
min_tokens: request.min_tokens,
|
||||
logprobs,
|
||||
prompt_logprobs,
|
||||
min_p: request.min_p,
|
||||
frequency_penalty: request.frequency_penalty,
|
||||
presence_penalty: request.presence_penalty,
|
||||
repetition_penalty: request.repetition_penalty,
|
||||
stop_token_ids: request.stop_token_ids,
|
||||
ignore_eos: request.ignore_eos,
|
||||
logit_bias: convert_logit_bias(request.logit_bias)?,
|
||||
allowed_token_ids: request.allowed_token_ids,
|
||||
bad_words: None,
|
||||
logprob_token_ids: None,
|
||||
structured_outputs,
|
||||
skip_reading_prefix_cache: None,
|
||||
vllm_xargs: merge_kv_transfer_params(
|
||||
request.vllm_xargs,
|
||||
request.kv_transfer_params.as_ref(),
|
||||
),
|
||||
},
|
||||
decode_options: TextDecodeOptions {
|
||||
skip_special_tokens: request.skip_special_tokens,
|
||||
include_stop_str_in_output: request.include_stop_str_in_output,
|
||||
stop_strings: request.stop.map(|stop| stop.into_vec()),
|
||||
min_tokens: request.min_tokens.unwrap_or(0),
|
||||
},
|
||||
intermediate: request.stream,
|
||||
priority: request.priority.unwrap_or(0),
|
||||
cache_salt: request.cache_salt,
|
||||
add_special_tokens: request.add_special_tokens,
|
||||
data_parallel_rank: ctx.data_parallel_rank,
|
||||
};
|
||||
|
||||
Ok(PreparedRequest {
|
||||
request_id,
|
||||
response_model: served_model_names.first().cloned().unwrap_or_default(),
|
||||
include_usage,
|
||||
text_request,
|
||||
echo,
|
||||
return_token_ids: request.return_token_ids.unwrap_or(false),
|
||||
return_tokens_as_token_ids: request.return_tokens_as_token_ids.unwrap_or(false),
|
||||
})
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use axum::http::HeaderMap;
|
||||
use serde_json::json;
|
||||
use vllm_text::Prompt;
|
||||
|
||||
use super::prepare_completion_request;
|
||||
use crate::routes::openai::completions::types::CompletionRequest;
|
||||
use crate::utils::{ResolvedRequestContext, resolve_request_context};
|
||||
|
||||
fn request_context(headers: &HeaderMap, request_id: Option<&str>) -> ResolvedRequestContext {
|
||||
resolve_request_context(headers, request_id)
|
||||
}
|
||||
|
||||
fn served(names: &[&str]) -> Vec<String> {
|
||||
names.iter().map(|s| s.to_string()).collect()
|
||||
}
|
||||
|
||||
fn base_request_json() -> serde_json::Value {
|
||||
json!({
|
||||
"model": "Qwen/Qwen1.5-0.5B-Chat",
|
||||
"prompt": "hello",
|
||||
"stream": true
|
||||
})
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn completion_http_request_deserializes_text_prompt() {
|
||||
let request: CompletionRequest =
|
||||
serde_json::from_value(base_request_json()).expect("parse request");
|
||||
|
||||
assert_eq!(request.prompt, Prompt::Text("hello".to_string()));
|
||||
assert_eq!(request.model, "Qwen/Qwen1.5-0.5B-Chat");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn completion_http_request_deserializes_token_id_prompt() {
|
||||
let request: CompletionRequest = serde_json::from_value(json!({
|
||||
"model": "Qwen/Qwen1.5-0.5B-Chat",
|
||||
"prompt": [11, 22, 33],
|
||||
"stream": true,
|
||||
"ignore_eos": true,
|
||||
"max_tokens": 7
|
||||
}))
|
||||
.expect("parse request");
|
||||
|
||||
assert_eq!(request.prompt, Prompt::TokenIds(vec![11, 22, 33]));
|
||||
assert_eq!(request.max_tokens, Some(7));
|
||||
assert!(request.ignore_eos);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn prepare_completion_request_maps_sampling_fields() {
|
||||
let request: CompletionRequest = serde_json::from_value(json!({
|
||||
"model": "Qwen/Qwen1.5-0.5B-Chat",
|
||||
"prompt": [11, 22, 33],
|
||||
"stream": true,
|
||||
"stream_options": {"include_usage": true},
|
||||
"max_tokens": 7,
|
||||
"logprobs": 2,
|
||||
"top_p": 0.9,
|
||||
"top_k": 42,
|
||||
"min_p": 0.1,
|
||||
"frequency_penalty": 0.2,
|
||||
"presence_penalty": 0.3,
|
||||
"repetition_penalty": 1.1,
|
||||
"ignore_eos": true,
|
||||
"skip_special_tokens": false
|
||||
}))
|
||||
.expect("parse request");
|
||||
|
||||
let prepared = prepare_completion_request(
|
||||
request,
|
||||
&served(&["Qwen/Qwen1.5-0.5B-Chat"]),
|
||||
ResolvedRequestContext::default(),
|
||||
)
|
||||
.expect("prepare");
|
||||
|
||||
assert!(prepared.include_usage);
|
||||
assert_eq!(
|
||||
prepared.text_request.prompt,
|
||||
Prompt::TokenIds(vec![11, 22, 33])
|
||||
);
|
||||
assert_eq!(prepared.text_request.sampling_params.max_tokens, Some(7));
|
||||
assert_eq!(prepared.text_request.sampling_params.logprobs, Some(2));
|
||||
assert_eq!(prepared.text_request.sampling_params.top_p, Some(0.9));
|
||||
assert_eq!(prepared.text_request.sampling_params.top_k, Some(42));
|
||||
assert_eq!(prepared.text_request.sampling_params.min_p, Some(0.1));
|
||||
assert_eq!(
|
||||
prepared.text_request.sampling_params.frequency_penalty,
|
||||
Some(0.2)
|
||||
);
|
||||
assert_eq!(
|
||||
prepared.text_request.sampling_params.presence_penalty,
|
||||
Some(0.3)
|
||||
);
|
||||
assert_eq!(
|
||||
prepared.text_request.sampling_params.repetition_penalty,
|
||||
Some(1.1)
|
||||
);
|
||||
assert!(prepared.text_request.sampling_params.ignore_eos);
|
||||
assert!(!prepared.text_request.decode_options.skip_special_tokens);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn prepare_completion_request_accepts_text_echo() {
|
||||
let request: CompletionRequest = serde_json::from_value(json!({
|
||||
"model": "Qwen/Qwen1.5-0.5B-Chat",
|
||||
"prompt": "hello",
|
||||
"stream": true,
|
||||
"echo": true,
|
||||
"max_tokens": 7
|
||||
}))
|
||||
.expect("parse request");
|
||||
|
||||
let prepared = prepare_completion_request(
|
||||
request,
|
||||
&served(&["Qwen/Qwen1.5-0.5B-Chat"]),
|
||||
ResolvedRequestContext::default(),
|
||||
)
|
||||
.expect("prepare");
|
||||
|
||||
assert_eq!(prepared.echo, Some("hello".to_string()));
|
||||
assert_eq!(prepared.text_request.sampling_params.max_tokens, Some(7));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn prepare_completion_request_enables_prompt_logprobs_for_non_stream_echo() {
|
||||
let request: CompletionRequest = serde_json::from_value(json!({
|
||||
"model": "Qwen/Qwen1.5-0.5B-Chat",
|
||||
"prompt": "hello",
|
||||
"echo": true,
|
||||
"stream": false,
|
||||
"logprobs": 3
|
||||
}))
|
||||
.expect("parse request");
|
||||
|
||||
let prepared = prepare_completion_request(
|
||||
request,
|
||||
&served(&["Qwen/Qwen1.5-0.5B-Chat"]),
|
||||
ResolvedRequestContext::default(),
|
||||
)
|
||||
.expect("prepare");
|
||||
|
||||
assert_eq!(prepared.text_request.sampling_params.logprobs, Some(3));
|
||||
assert_eq!(
|
||||
prepared.text_request.sampling_params.prompt_logprobs,
|
||||
Some(3)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn prepare_completion_request_rejects_token_id_prompt_echo() {
|
||||
let request: CompletionRequest = serde_json::from_value(json!({
|
||||
"model": "Qwen/Qwen1.5-0.5B-Chat",
|
||||
"prompt": [11, 22, 33],
|
||||
"stream": true,
|
||||
"echo": true
|
||||
}))
|
||||
.expect("parse request");
|
||||
|
||||
assert!(
|
||||
prepare_completion_request(
|
||||
request,
|
||||
&served(&["Qwen/Qwen1.5-0.5B-Chat"]),
|
||||
ResolvedRequestContext::default(),
|
||||
)
|
||||
.is_err()
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn prepare_completion_request_accepts_logprobs_fields() {
|
||||
let request: CompletionRequest = serde_json::from_value(json!({
|
||||
"model": "Qwen/Qwen1.5-0.5B-Chat",
|
||||
"prompt": "hello",
|
||||
"stream": false,
|
||||
"logprobs": 1,
|
||||
"prompt_logprobs": 2
|
||||
}))
|
||||
.expect("parse request");
|
||||
|
||||
let prepared = prepare_completion_request(
|
||||
request,
|
||||
&served(&["Qwen/Qwen1.5-0.5B-Chat"]),
|
||||
ResolvedRequestContext::default(),
|
||||
)
|
||||
.expect("prepare");
|
||||
assert_eq!(prepared.text_request.sampling_params.logprobs, Some(1));
|
||||
assert_eq!(
|
||||
prepared.text_request.sampling_params.prompt_logprobs,
|
||||
Some(2)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn prepare_completion_request_threads_data_parallel_rank() {
|
||||
let request: CompletionRequest = serde_json::from_value(json!({
|
||||
"model": "Qwen/Qwen1.5-0.5B-Chat",
|
||||
"prompt": "hello",
|
||||
"stream": false,
|
||||
}))
|
||||
.expect("parse request");
|
||||
|
||||
let mut headers = HeaderMap::new();
|
||||
headers.insert("X-data-parallel-rank", "3".parse().unwrap());
|
||||
let prepared = prepare_completion_request(
|
||||
request,
|
||||
&served(&["Qwen/Qwen1.5-0.5B-Chat"]),
|
||||
request_context(&headers, None),
|
||||
)
|
||||
.expect("prepare");
|
||||
assert_eq!(prepared.text_request.data_parallel_rank, Some(3));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn prepare_completion_request_leaves_data_parallel_rank_none_when_absent() {
|
||||
let request: CompletionRequest = serde_json::from_value(json!({
|
||||
"model": "Qwen/Qwen1.5-0.5B-Chat",
|
||||
"prompt": "hello",
|
||||
"stream": false,
|
||||
}))
|
||||
.expect("parse request");
|
||||
|
||||
let prepared = prepare_completion_request(
|
||||
request,
|
||||
&served(&["Qwen/Qwen1.5-0.5B-Chat"]),
|
||||
ResolvedRequestContext::default(),
|
||||
)
|
||||
.expect("prepare");
|
||||
assert_eq!(prepared.text_request.data_parallel_rank, None);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,253 @@
|
||||
use std::collections::HashMap;
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
use serde_json::{Map, Value};
|
||||
use validator::Validate;
|
||||
use vllm_text::Prompt;
|
||||
|
||||
use crate::routes::openai::utils::types::{
|
||||
LogProbs, Normalizable, StreamOptions, StringOrArray, Usage, default_true, validate_stop,
|
||||
};
|
||||
|
||||
/// Serde default for `CompletionRequest::max_tokens`, matching the Python vLLM
|
||||
/// / OpenAI default.
|
||||
fn default_completion_max_tokens() -> Option<u32> {
|
||||
Some(16)
|
||||
}
|
||||
|
||||
/// vLLM-compatible request type for the Completions API.
|
||||
///
|
||||
/// Mirrors the Python vLLM `CompletionRequest` class. The local copy keeps the
|
||||
/// request type route-owned so we can accept token-id prompts via
|
||||
/// [`vllm_text::Prompt`] and add vLLM-only fields directly instead of layering
|
||||
/// wrapper deserializers on top.
|
||||
#[serde_with::skip_serializing_none]
|
||||
#[derive(Debug, Clone, Deserialize, Serialize, Validate)]
|
||||
pub struct CompletionRequest {
|
||||
// -------- Standard OpenAI API Parameters --------
|
||||
/// ID of the model to use
|
||||
pub model: String,
|
||||
|
||||
/// The prompt(s) to generate completions for.
|
||||
///
|
||||
/// We use [`vllm_text::Prompt`] here to support token-id input.
|
||||
pub prompt: Prompt,
|
||||
|
||||
/// Echo back the prompt in addition to the completion
|
||||
#[serde(default)]
|
||||
pub echo: bool,
|
||||
|
||||
/// Number between -2.0 and 2.0. Positive values penalize new tokens based
|
||||
/// on their existing frequency in the text so far
|
||||
pub frequency_penalty: Option<f32>,
|
||||
|
||||
/// Modify the likelihood of specified tokens appearing in the completion
|
||||
pub logit_bias: Option<HashMap<String, f32>>,
|
||||
|
||||
/// Include the log probabilities on the logprobs most likely tokens
|
||||
pub logprobs: Option<u32>,
|
||||
|
||||
/// The maximum number of tokens to generate (defaults to 16 when absent,
|
||||
/// matching the Python vLLM / OpenAI API convention)
|
||||
#[serde(default = "default_completion_max_tokens")]
|
||||
pub max_tokens: Option<u32>,
|
||||
|
||||
/// How many completions to generate for each prompt
|
||||
pub n: Option<u32>,
|
||||
|
||||
/// Number between -2.0 and 2.0. Positive values penalize new tokens based
|
||||
/// on whether they appear in the text so far
|
||||
pub presence_penalty: Option<f32>,
|
||||
|
||||
/// If specified, our system will make a best effort to sample
|
||||
/// deterministically
|
||||
pub seed: Option<i64>,
|
||||
|
||||
/// Up to 4 sequences where the API will stop generating further tokens
|
||||
#[validate(custom(function = "validate_stop"))]
|
||||
pub stop: Option<StringOrArray>,
|
||||
|
||||
/// Whether to stream back partial progress
|
||||
#[serde(default)]
|
||||
pub stream: bool,
|
||||
|
||||
/// The suffix that comes after a completion of inserted text
|
||||
pub suffix: Option<String>,
|
||||
|
||||
/// What sampling temperature to use, between 0 and 2
|
||||
pub temperature: Option<f32>,
|
||||
|
||||
/// An alternative to sampling with temperature (nucleus sampling)
|
||||
pub top_p: Option<f32>,
|
||||
|
||||
/// A unique identifier representing your end-user
|
||||
pub user: Option<String>,
|
||||
|
||||
// -------- vLLM Sampling Parameters --------
|
||||
/// Options for streaming response
|
||||
pub stream_options: Option<StreamOptions>,
|
||||
|
||||
/// Use beam search instead of sampling
|
||||
#[serde(default)]
|
||||
pub use_beam_search: bool,
|
||||
|
||||
/// Top-k sampling parameter
|
||||
pub top_k: Option<u32>,
|
||||
|
||||
/// Min-p nucleus sampling parameter
|
||||
pub min_p: Option<f32>,
|
||||
|
||||
/// Repetition penalty for reducing repetitive text
|
||||
pub repetition_penalty: Option<f32>,
|
||||
|
||||
/// Length penalty for beam search
|
||||
pub length_penalty: Option<f32>,
|
||||
|
||||
/// Specific token IDs to use as stop conditions
|
||||
pub stop_token_ids: Option<Vec<u32>>,
|
||||
|
||||
/// Include stop string in output
|
||||
#[serde(default)]
|
||||
pub include_stop_str_in_output: bool,
|
||||
|
||||
/// Ignore end-of-sequence tokens during generation
|
||||
#[serde(default)]
|
||||
pub ignore_eos: bool,
|
||||
|
||||
/// Minimum number of tokens to generate
|
||||
pub min_tokens: Option<u32>,
|
||||
|
||||
/// Skip special tokens during detokenization
|
||||
#[serde(default = "default_true")]
|
||||
pub skip_special_tokens: bool,
|
||||
|
||||
/// Add spaces between special tokens during detokenization
|
||||
#[serde(default = "default_true")]
|
||||
pub spaces_between_special_tokens: bool,
|
||||
|
||||
/// Truncate prompt tokens to this length
|
||||
pub truncate_prompt_tokens: Option<i64>,
|
||||
|
||||
/// Restrict output to these token IDs only
|
||||
pub allowed_token_ids: Option<Vec<u32>>,
|
||||
|
||||
/// Number of prompt logprobs to return
|
||||
pub prompt_logprobs: Option<i32>,
|
||||
|
||||
// -------- Extra vLLM Parameters --------
|
||||
/// Whether to add special tokens (e.g. BOS) to the prompt
|
||||
#[serde(default = "default_true")]
|
||||
pub add_special_tokens: bool,
|
||||
|
||||
/// Format specification for structured output (JSON mode, JSON schema,
|
||||
/// etc.)
|
||||
pub response_format: Option<Value>,
|
||||
|
||||
/// Additional kwargs for structured outputs
|
||||
pub structured_outputs: Option<Value>,
|
||||
|
||||
/// Request scheduling priority (lower means earlier; default 0)
|
||||
pub priority: Option<i32>,
|
||||
|
||||
/// External request ID used for response correlation.
|
||||
pub request_id: Option<String>,
|
||||
|
||||
/// Tokens represented as strings of the form 'token_id:{token_id}' in
|
||||
/// logprobs
|
||||
pub return_tokens_as_token_ids: Option<bool>,
|
||||
|
||||
/// Include token IDs alongside generated text
|
||||
pub return_token_ids: Option<bool>,
|
||||
|
||||
/// Salt for prefix cache isolation in multi-user environments
|
||||
pub cache_salt: Option<String>,
|
||||
|
||||
/// KV transfer parameters for disaggregated serving
|
||||
pub kv_transfer_params: Option<HashMap<String, Value>>,
|
||||
|
||||
/// Additional request parameters with string or numeric values for custom
|
||||
/// extensions
|
||||
pub vllm_xargs: Option<HashMap<String, Value>>,
|
||||
|
||||
/// Additional fields
|
||||
#[serde(flatten)]
|
||||
pub other: Map<String, Value>,
|
||||
}
|
||||
|
||||
impl Normalizable for CompletionRequest {}
|
||||
|
||||
/// Mirrors the Python vLLM `CompletionResponse` class.
|
||||
#[serde_with::skip_serializing_none]
|
||||
#[derive(Debug, Clone, Serialize)]
|
||||
pub(super) struct CompletionResponse {
|
||||
pub id: String,
|
||||
pub object: String,
|
||||
pub created: u64,
|
||||
pub model: String,
|
||||
pub choices: Vec<CompletionChoice>,
|
||||
pub usage: Option<Usage>,
|
||||
pub system_fingerprint: Option<String>,
|
||||
pub kv_transfer_params: Option<Value>,
|
||||
}
|
||||
|
||||
/// Mirrors the Python vLLM `CompletionResponseChoice` class.
|
||||
#[serde_with::skip_serializing_none]
|
||||
#[derive(Debug, Clone, Serialize)]
|
||||
pub(super) struct CompletionChoice {
|
||||
pub index: u32,
|
||||
pub text: String,
|
||||
pub logprobs: Option<LogProbs>,
|
||||
pub finish_reason: Option<String>,
|
||||
pub stop_reason: Option<Value>,
|
||||
pub prompt_logprobs: Option<Vec<Option<HashMap<String, f32>>>>,
|
||||
pub token_ids: Option<Vec<u32>>,
|
||||
pub prompt_token_ids: Option<Vec<u32>>,
|
||||
}
|
||||
|
||||
/// Mirrors the Python vLLM `CompletionStreamResponse` class.
|
||||
#[serde_with::skip_serializing_none]
|
||||
#[derive(Debug, Clone, Serialize)]
|
||||
pub(super) struct CompletionStreamResponse {
|
||||
pub id: String,
|
||||
pub object: String,
|
||||
pub created: u64,
|
||||
pub model: String,
|
||||
pub choices: Vec<CompletionStreamChoice>,
|
||||
pub usage: Option<Usage>,
|
||||
}
|
||||
|
||||
impl CompletionStreamResponse {
|
||||
/// Create a stream response with the standard envelope fields pre-filled.
|
||||
pub fn new(id: &str, model: &str, created: u64) -> Self {
|
||||
Self {
|
||||
id: id.to_string(),
|
||||
object: "text_completion".to_string(),
|
||||
created,
|
||||
model: model.to_string(),
|
||||
choices: Vec::new(),
|
||||
usage: None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Mirrors the Python vLLM `CompletionResponseStreamChoice` class.
|
||||
#[serde_with::skip_serializing_none]
|
||||
#[derive(Debug, Clone, Default, Serialize)]
|
||||
pub(super) struct CompletionStreamChoice {
|
||||
pub index: u32,
|
||||
pub text: String,
|
||||
pub logprobs: Option<LogProbs>,
|
||||
pub finish_reason: Option<String>,
|
||||
pub stop_reason: Option<Value>,
|
||||
pub token_ids: Option<Vec<u32>>,
|
||||
pub prompt_token_ids: Option<Vec<u32>>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize)]
|
||||
#[serde(untagged)]
|
||||
pub(super) enum CompletionSseChunk {
|
||||
/// Ordinary OpenAI completions delta/final chunk.
|
||||
Chunk(CompletionStreamResponse),
|
||||
/// Final usage chunk emitted before `[DONE]` when `include_usage=true`.
|
||||
Usage(CompletionStreamResponse),
|
||||
}
|
||||
@@ -0,0 +1,178 @@
|
||||
use vllm_text::Prompt;
|
||||
|
||||
use super::types::CompletionRequest;
|
||||
use crate::error::{ApiError, bail_invalid_request};
|
||||
|
||||
/// Enforce the minimal compatibility contract for the Rust OpenAI server.
|
||||
pub(super) fn validate_request_compat(
|
||||
request: &CompletionRequest,
|
||||
served_model_names: &[String],
|
||||
) -> Result<(), ApiError> {
|
||||
// This path is intentionally scoped to the minimum surface needed by
|
||||
// `vllm-bench` random workload compatibility, so unsupported legacy
|
||||
// completions features fail early here.
|
||||
if !served_model_names.iter().any(|n| n == &request.model) {
|
||||
return Err(ApiError::model_not_found(request.model.clone()));
|
||||
}
|
||||
|
||||
if request.stream_options.is_some() && !request.stream {
|
||||
bail_invalid_request!(
|
||||
param = "stream_options",
|
||||
"stream_options are only supported when stream=true."
|
||||
);
|
||||
}
|
||||
|
||||
if request.n.unwrap_or(1) > 1 {
|
||||
bail_invalid_request!(param = "n", "Only n=1 is supported.");
|
||||
}
|
||||
|
||||
if request.max_tokens == Some(0) {
|
||||
bail_invalid_request!(param = "max_tokens", "max_tokens must be greater than 0.");
|
||||
}
|
||||
|
||||
if request.echo && matches!(request.prompt, Prompt::TokenIds(_)) {
|
||||
bail_invalid_request!(
|
||||
param = "echo",
|
||||
"echo is not supported with token-ID prompts."
|
||||
);
|
||||
}
|
||||
|
||||
if request.suffix.is_some() {
|
||||
bail_invalid_request!(param = "suffix", "suffix is not supported.");
|
||||
}
|
||||
|
||||
if let Some(logprobs) = request.logprobs
|
||||
&& logprobs > i32::MAX as u32
|
||||
{
|
||||
bail_invalid_request!(
|
||||
param = "logprobs",
|
||||
"`logprobs` must fit within a signed 32-bit integer."
|
||||
);
|
||||
}
|
||||
|
||||
if let Some(prompt_logprobs) = request.prompt_logprobs {
|
||||
if request.stream && (prompt_logprobs > 0 || prompt_logprobs == -1) {
|
||||
bail_invalid_request!(
|
||||
param = "prompt_logprobs",
|
||||
"`prompt_logprobs` are not available when `stream=true`."
|
||||
);
|
||||
}
|
||||
|
||||
if prompt_logprobs < 0 && prompt_logprobs != -1 {
|
||||
bail_invalid_request!(
|
||||
param = "prompt_logprobs",
|
||||
"`prompt_logprobs` must be a non-negative value or -1."
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
if request.use_beam_search {
|
||||
bail_invalid_request!(
|
||||
param = "use_beam_search",
|
||||
"use_beam_search is not supported."
|
||||
);
|
||||
}
|
||||
|
||||
// ---- Reject parameters that are accepted for deserialization but not yet
|
||||
// implemented ----
|
||||
|
||||
if request.length_penalty.is_some() {
|
||||
bail_invalid_request!(param = "length_penalty", "length_penalty is not supported.");
|
||||
}
|
||||
if !request.spaces_between_special_tokens {
|
||||
bail_invalid_request!(
|
||||
param = "spaces_between_special_tokens",
|
||||
"spaces_between_special_tokens is not supported."
|
||||
);
|
||||
}
|
||||
if request.truncate_prompt_tokens.is_some() {
|
||||
bail_invalid_request!(
|
||||
param = "truncate_prompt_tokens",
|
||||
"truncate_prompt_tokens is not supported."
|
||||
);
|
||||
}
|
||||
|
||||
if let Some(options) = &request.stream_options
|
||||
&& options.continuous_usage_stats.is_some()
|
||||
{
|
||||
bail_invalid_request!(
|
||||
param = "stream_options",
|
||||
"continuous_usage_stats is not supported."
|
||||
);
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use serde_json::json;
|
||||
|
||||
use super::validate_request_compat;
|
||||
use crate::routes::openai::completions::types::CompletionRequest;
|
||||
|
||||
fn base_request() -> CompletionRequest {
|
||||
serde_json::from_value(json!({
|
||||
"model": "Qwen/Qwen1.5-0.5B-Chat",
|
||||
"prompt": "hello",
|
||||
"stream": true,
|
||||
}))
|
||||
.expect("parse request")
|
||||
}
|
||||
|
||||
fn served_names(names: &[&str]) -> Vec<String> {
|
||||
names.iter().map(|s| s.to_string()).collect()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn validate_request_compat_accepts_logprobs() {
|
||||
let request = CompletionRequest {
|
||||
logprobs: Some(1),
|
||||
..base_request()
|
||||
};
|
||||
assert!(
|
||||
validate_request_compat(&request, &served_names(&["Qwen/Qwen1.5-0.5B-Chat"])).is_ok()
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn validate_request_compat_accepts_any_served_name() {
|
||||
let request = base_request();
|
||||
assert!(
|
||||
validate_request_compat(
|
||||
&request,
|
||||
&served_names(&["other-alias", "Qwen/Qwen1.5-0.5B-Chat"])
|
||||
)
|
||||
.is_ok()
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn validate_request_compat_rejects_unknown_model() {
|
||||
let request = base_request();
|
||||
assert!(validate_request_compat(&request, &served_names(&["other-model"])).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn validate_request_compat_rejects_streaming_prompt_logprobs() {
|
||||
let request = CompletionRequest {
|
||||
prompt_logprobs: Some(1),
|
||||
..base_request()
|
||||
};
|
||||
assert!(
|
||||
validate_request_compat(&request, &served_names(&["Qwen/Qwen1.5-0.5B-Chat"])).is_err()
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn validate_request_compat_accepts_non_stream_prompt_logprobs() {
|
||||
let request = CompletionRequest {
|
||||
stream: false,
|
||||
prompt_logprobs: Some(-1),
|
||||
..base_request()
|
||||
};
|
||||
assert!(
|
||||
validate_request_compat(&request, &served_names(&["Qwen/Qwen1.5-0.5B-Chat"])).is_ok()
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,8 @@
|
||||
pub mod chat_completions;
|
||||
mod completions;
|
||||
mod models;
|
||||
pub(crate) mod utils;
|
||||
|
||||
pub use chat_completions::chat_completions;
|
||||
pub use completions::completions;
|
||||
pub use models::list_models;
|
||||
@@ -0,0 +1,24 @@
|
||||
use std::sync::Arc;
|
||||
|
||||
use axum::Json;
|
||||
use axum::extract::State;
|
||||
|
||||
use crate::routes::openai::utils::types::{ListModelsResponse, ModelObject};
|
||||
use crate::state::AppState;
|
||||
|
||||
/// Return all configured served model names in OpenAI `list models` format.
|
||||
pub async fn list_models(State(state): State<Arc<AppState>>) -> Json<ListModelsResponse> {
|
||||
Json(ListModelsResponse {
|
||||
object: "list".to_string(),
|
||||
data: state
|
||||
.served_model_names()
|
||||
.iter()
|
||||
.map(|name| ModelObject {
|
||||
id: name.clone(),
|
||||
object: "model".to_string(),
|
||||
created: 0,
|
||||
owned_by: "vllm-frontend-rs".to_string(),
|
||||
})
|
||||
.collect(),
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,256 @@
|
||||
use std::collections::HashMap;
|
||||
|
||||
use itertools::Itertools as _;
|
||||
use vllm_text::{
|
||||
CollectedTextOutput, DecodedLogprobs, DecodedPositionLogprobs, DecodedPromptLogprobs,
|
||||
DecodedTokenLogprob,
|
||||
};
|
||||
|
||||
use super::types::{ChatLogProbs, ChatLogProbsContent, LogProbs, TopLogProb};
|
||||
use crate::error::{ApiError, server_error};
|
||||
|
||||
/// Convert decoded token-position logprobs into the OpenAI completions
|
||||
/// `logprobs` shape.
|
||||
pub fn decoded_logprobs_to_openai(
|
||||
logprobs: &DecodedLogprobs,
|
||||
initial_text_offset: u32,
|
||||
return_tokens_as_token_ids: bool,
|
||||
) -> Result<LogProbs, ApiError> {
|
||||
let mut text_offset = Vec::with_capacity(logprobs.positions.len());
|
||||
let mut token_logprobs = Vec::with_capacity(logprobs.positions.len());
|
||||
let mut tokens = Vec::with_capacity(logprobs.positions.len());
|
||||
let mut top_logprobs = Vec::with_capacity(logprobs.positions.len());
|
||||
let mut current_offset = initial_text_offset;
|
||||
|
||||
for position in &logprobs.positions {
|
||||
let chosen = position.entries.first().ok_or_else(|| {
|
||||
server_error!("decoded logprobs position unexpectedly had no token candidates")
|
||||
})?;
|
||||
|
||||
let token_str = format_token(chosen, return_tokens_as_token_ids);
|
||||
text_offset.push(current_offset);
|
||||
token_logprobs.push(Some(clamp_logprob(chosen.logprob)));
|
||||
current_offset = current_offset.saturating_add(text_len(&token_str));
|
||||
tokens.push(token_str);
|
||||
top_logprobs.push(Some(position_top_logprobs_map(
|
||||
position,
|
||||
return_tokens_as_token_ids,
|
||||
)));
|
||||
}
|
||||
|
||||
Ok(LogProbs {
|
||||
tokens,
|
||||
token_logprobs,
|
||||
top_logprobs,
|
||||
text_offset,
|
||||
})
|
||||
}
|
||||
|
||||
/// Convert decoded prompt logprobs into the OpenAI completions `logprobs`
|
||||
/// shape.
|
||||
///
|
||||
/// The first prompt token is included with `None` logprob metadata, matching
|
||||
/// Python vLLM's echoed completions behavior.
|
||||
pub fn decoded_prompt_logprobs_to_openai(
|
||||
prompt_logprobs: &DecodedPromptLogprobs,
|
||||
initial_text_offset: u32,
|
||||
return_tokens_as_token_ids: bool,
|
||||
) -> Result<LogProbs, ApiError> {
|
||||
let mut text_offset = Vec::with_capacity(prompt_logprobs.scored_positions.len() + 1);
|
||||
let mut token_logprobs = Vec::with_capacity(prompt_logprobs.scored_positions.len() + 1);
|
||||
let mut tokens = Vec::with_capacity(prompt_logprobs.scored_positions.len() + 1);
|
||||
let mut top_logprobs = Vec::with_capacity(prompt_logprobs.scored_positions.len() + 1);
|
||||
let mut current_offset = initial_text_offset;
|
||||
|
||||
let first_token_str = if return_tokens_as_token_ids {
|
||||
format!("token_id:{}", prompt_logprobs.first_token_id)
|
||||
} else {
|
||||
prompt_logprobs.first_token.clone()
|
||||
};
|
||||
text_offset.push(current_offset);
|
||||
token_logprobs.push(None);
|
||||
current_offset = current_offset.saturating_add(text_len(&first_token_str));
|
||||
tokens.push(first_token_str);
|
||||
top_logprobs.push(None);
|
||||
|
||||
for position in &prompt_logprobs.scored_positions {
|
||||
let chosen = position.entries.first().ok_or_else(|| {
|
||||
server_error!("decoded prompt logprobs position unexpectedly had no token candidates")
|
||||
})?;
|
||||
|
||||
let token_str = format_token(chosen, return_tokens_as_token_ids);
|
||||
text_offset.push(current_offset);
|
||||
token_logprobs.push(Some(clamp_logprob(chosen.logprob)));
|
||||
current_offset = current_offset.saturating_add(text_len(&token_str));
|
||||
tokens.push(token_str);
|
||||
top_logprobs.push(Some(position_top_logprobs_map(
|
||||
position,
|
||||
return_tokens_as_token_ids,
|
||||
)));
|
||||
}
|
||||
|
||||
Ok(LogProbs {
|
||||
tokens,
|
||||
token_logprobs,
|
||||
top_logprobs,
|
||||
text_offset,
|
||||
})
|
||||
}
|
||||
|
||||
/// Convert decoded prompt logprobs into the vLLM-style prompt-logprobs response
|
||||
/// shape.
|
||||
pub fn decoded_prompt_logprobs_to_maps(
|
||||
prompt_logprobs: &DecodedPromptLogprobs,
|
||||
return_tokens_as_token_ids: bool,
|
||||
) -> Vec<Option<HashMap<String, f32>>> {
|
||||
std::iter::once(None)
|
||||
.chain(prompt_logprobs.scored_positions.iter().map(|position| {
|
||||
Some(position_top_logprobs_map(
|
||||
position,
|
||||
return_tokens_as_token_ids,
|
||||
))
|
||||
}))
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Convert decoded token-position logprobs into the OpenAI chat `logprobs`
|
||||
/// shape.
|
||||
pub fn decoded_logprobs_to_openai_chat(
|
||||
logprobs: &DecodedLogprobs,
|
||||
return_tokens_as_token_ids: bool,
|
||||
) -> Result<ChatLogProbs, ApiError> {
|
||||
let content = logprobs
|
||||
.positions
|
||||
.iter()
|
||||
.map(|pos| position_to_chat_logprobs_content(pos, return_tokens_as_token_ids))
|
||||
.try_collect()?;
|
||||
|
||||
Ok(ChatLogProbs {
|
||||
content: Some(content),
|
||||
})
|
||||
}
|
||||
|
||||
/// Count visible text positions using OpenAI completions' character-offset
|
||||
/// convention.
|
||||
pub fn text_len(text: &str) -> u32 {
|
||||
u32::try_from(text.chars().count()).unwrap_or(u32::MAX)
|
||||
}
|
||||
|
||||
/// Concatenate two OpenAI-style completion logprobs payloads in token order.
|
||||
pub fn append_openai_logprobs(mut prefix: LogProbs, suffix: LogProbs) -> LogProbs {
|
||||
prefix.tokens.extend(suffix.tokens);
|
||||
prefix.token_logprobs.extend(suffix.token_logprobs);
|
||||
prefix.top_logprobs.extend(suffix.top_logprobs);
|
||||
prefix.text_offset.extend(suffix.text_offset);
|
||||
prefix
|
||||
}
|
||||
|
||||
/// Build the non-stream completions `logprobs` payload from collected text
|
||||
/// output.
|
||||
///
|
||||
/// When `echoed_prompt` is true, the returned payload matches Python vLLM's
|
||||
/// echoed completions behavior by concatenating prompt and completion logprobs
|
||||
/// into one OpenAI `LogProbs` object.
|
||||
pub fn collected_logprobs_to_openai(
|
||||
collected: &CollectedTextOutput,
|
||||
echoed_prompt: bool,
|
||||
initial_completion_offset: u32,
|
||||
return_tokens_as_token_ids: bool,
|
||||
) -> Result<LogProbs, ApiError> {
|
||||
if echoed_prompt {
|
||||
let prompt_logprobs = collected.prompt_logprobs.as_ref().ok_or_else(|| {
|
||||
server_error!(
|
||||
"echoed completion logprobs require prompt logprobs but generation returned none"
|
||||
)
|
||||
})?;
|
||||
let prompt_logprobs =
|
||||
decoded_prompt_logprobs_to_openai(prompt_logprobs, 0, return_tokens_as_token_ids)?;
|
||||
let completion_start = prompt_logprobs
|
||||
.text_offset
|
||||
.last()
|
||||
.zip(prompt_logprobs.tokens.last())
|
||||
.map(|(&offset, token)| offset.saturating_add(text_len(token)))
|
||||
.unwrap_or(0);
|
||||
return match collected.logprobs.as_ref() {
|
||||
Some(completion_logprobs) => Ok(append_openai_logprobs(
|
||||
prompt_logprobs,
|
||||
decoded_logprobs_to_openai(
|
||||
completion_logprobs,
|
||||
completion_start,
|
||||
return_tokens_as_token_ids,
|
||||
)?,
|
||||
)),
|
||||
None => Ok(prompt_logprobs),
|
||||
};
|
||||
}
|
||||
|
||||
let completion_logprobs = collected.logprobs.as_ref().ok_or_else(|| {
|
||||
server_error!("completion response requested logprobs but generation returned none")
|
||||
})?;
|
||||
decoded_logprobs_to_openai(
|
||||
completion_logprobs,
|
||||
initial_completion_offset,
|
||||
return_tokens_as_token_ids,
|
||||
)
|
||||
}
|
||||
|
||||
/// Format a token entry as either its decoded string or `token_id:{id}`.
|
||||
fn format_token(entry: &DecodedTokenLogprob, as_token_id: bool) -> String {
|
||||
if as_token_id {
|
||||
format!("token_id:{}", entry.token_id)
|
||||
} else {
|
||||
entry.token.clone()
|
||||
}
|
||||
}
|
||||
|
||||
fn position_top_logprobs_map(
|
||||
position: &DecodedPositionLogprobs,
|
||||
return_tokens_as_token_ids: bool,
|
||||
) -> HashMap<String, f32> {
|
||||
position
|
||||
.entries
|
||||
.iter()
|
||||
.map(|entry| {
|
||||
(
|
||||
format_token(entry, return_tokens_as_token_ids),
|
||||
clamp_logprob(entry.logprob),
|
||||
)
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn position_to_chat_logprobs_content(
|
||||
position: &DecodedPositionLogprobs,
|
||||
return_tokens_as_token_ids: bool,
|
||||
) -> Result<ChatLogProbsContent, ApiError> {
|
||||
let chosen = position.entries.first().ok_or_else(|| {
|
||||
server_error!("decoded chat logprobs position unexpectedly had no token candidates")
|
||||
})?;
|
||||
|
||||
let token_str = format_token(chosen, return_tokens_as_token_ids);
|
||||
Ok(ChatLogProbsContent {
|
||||
token: token_str.clone(),
|
||||
logprob: clamp_logprob(chosen.logprob),
|
||||
bytes: Some(token_bytes(&token_str)),
|
||||
top_logprobs: position
|
||||
.entries
|
||||
.iter()
|
||||
.map(|entry| {
|
||||
let t = format_token(entry, return_tokens_as_token_ids);
|
||||
TopLogProb {
|
||||
logprob: clamp_logprob(entry.logprob),
|
||||
bytes: Some(token_bytes(&t)),
|
||||
token: t,
|
||||
}
|
||||
})
|
||||
.collect(),
|
||||
})
|
||||
}
|
||||
|
||||
fn token_bytes(token: &str) -> Vec<u8> {
|
||||
token.as_bytes().to_vec()
|
||||
}
|
||||
|
||||
pub fn clamp_logprob(logprob: f32) -> f32 {
|
||||
logprob.max(-9999.0)
|
||||
}
|
||||
@@ -0,0 +1,4 @@
|
||||
pub mod logprobs;
|
||||
pub mod structured_outputs;
|
||||
pub mod types;
|
||||
pub mod validated_json;
|
||||
@@ -0,0 +1,132 @@
|
||||
use serde::{Deserialize, Serialize};
|
||||
use serde_json::Value;
|
||||
use vllm_engine_core_client::protocol::StructuredOutputsParams;
|
||||
|
||||
use crate::error::ApiError;
|
||||
|
||||
/// JSON schema specification nested inside a `json_schema` response format.
|
||||
///
|
||||
/// Mirrors the Python vLLM `JsonSchemaResponseFormat` class.
|
||||
#[serde_with::skip_serializing_none]
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
|
||||
pub struct JsonSchemaFormat {
|
||||
pub name: String,
|
||||
#[serde(default)]
|
||||
pub description: Option<String>,
|
||||
/// The actual JSON schema object.
|
||||
#[serde(alias = "json_schema")]
|
||||
pub schema: Value,
|
||||
#[serde(default)]
|
||||
pub strict: Option<bool>,
|
||||
}
|
||||
|
||||
/// Supported `response_format` types for chat and completion requests.
|
||||
///
|
||||
/// This is our own definition (rather than the `openai-protocol` crate's) so
|
||||
/// that we can support the vLLM-specific `structural_tag` variant.
|
||||
///
|
||||
/// Original Python definitions:
|
||||
/// <https://github.com/vllm-project/vllm/blob/f22d6e026/vllm/entrypoints/openai/engine/protocol.py#L116-L157>
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
|
||||
#[serde(tag = "type", rename_all = "snake_case")]
|
||||
pub enum ResponseFormat {
|
||||
Text,
|
||||
JsonObject,
|
||||
JsonSchema {
|
||||
json_schema: JsonSchemaFormat,
|
||||
},
|
||||
/// vLLM-specific structural tag format. The entire object (including the
|
||||
/// `type` field) is JSON-serialized and passed as
|
||||
/// `StructuredOutputsParams.structural_tag`.
|
||||
///
|
||||
/// We capture the payload as a catch-all map so both the legacy
|
||||
/// (`structures`/`triggers`) and current (`format`) shapes are
|
||||
/// preserved without needing typed structs.
|
||||
StructuralTag {
|
||||
#[serde(flatten)]
|
||||
extra: serde_json::Map<String, Value>,
|
||||
},
|
||||
}
|
||||
|
||||
/// Convert an explicit `structured_outputs` JSON blob into
|
||||
/// [`StructuredOutputsParams`].
|
||||
fn deserialize_structured_outputs(
|
||||
raw: &serde_json::Value,
|
||||
) -> Result<StructuredOutputsParams, ApiError> {
|
||||
serde_json::from_value(raw.clone()).map_err(|e| {
|
||||
ApiError::invalid_request(
|
||||
format!("invalid structured_outputs: {e}"),
|
||||
Some("structured_outputs"),
|
||||
)
|
||||
})
|
||||
}
|
||||
|
||||
/// Convert a typed [`ResponseFormat`] and/or raw `structured_outputs` blob into
|
||||
/// engine-core [`StructuredOutputsParams`].
|
||||
///
|
||||
/// Mirrors the Python vLLM conversion in
|
||||
/// `ChatCompletionRequest.to_sampling_params()`: <https://github.com/vllm-project/vllm/blob/f22d6e026/vllm/entrypoints/openai/chat_completion/protocol.py#L457-L487>
|
||||
pub fn convert_from_response_format(
|
||||
response_format: Option<&ResponseFormat>,
|
||||
structured_outputs: &Option<serde_json::Value>,
|
||||
) -> Result<Option<StructuredOutputsParams>, ApiError> {
|
||||
if let Some(raw) = structured_outputs {
|
||||
return Ok(Some(deserialize_structured_outputs(raw)?));
|
||||
}
|
||||
|
||||
let Some(fmt) = response_format else {
|
||||
return Ok(None);
|
||||
};
|
||||
match fmt {
|
||||
ResponseFormat::Text => Ok(None),
|
||||
ResponseFormat::JsonObject => Ok(Some(StructuredOutputsParams {
|
||||
json_object: Some(true),
|
||||
..Default::default()
|
||||
})),
|
||||
ResponseFormat::JsonSchema { json_schema } => Ok(Some(StructuredOutputsParams {
|
||||
json: Some(json_schema.schema.clone()),
|
||||
..Default::default()
|
||||
})),
|
||||
ResponseFormat::StructuralTag { .. } => {
|
||||
// The Python frontend dumps the entire response_format object (including the
|
||||
// `type` field) as a JSON string for the engine-core backend.
|
||||
let tag_json = serde_json::to_string(fmt).map_err(|e| {
|
||||
ApiError::invalid_request(
|
||||
format!("failed to serialize structural_tag: {e}"),
|
||||
Some("response_format"),
|
||||
)
|
||||
})?;
|
||||
Ok(Some(StructuredOutputsParams {
|
||||
structural_tag: Some(tag_json),
|
||||
..Default::default()
|
||||
}))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Convert raw `response_format` and/or `structured_outputs` JSON blobs into
|
||||
/// engine-core [`StructuredOutputsParams`].
|
||||
///
|
||||
/// Used by the completions endpoint which keeps both fields as opaque
|
||||
/// `serde_json::Value`.
|
||||
pub fn convert_from_response_format_value(
|
||||
response_format: &Option<serde_json::Value>,
|
||||
structured_outputs: &Option<serde_json::Value>,
|
||||
) -> Result<Option<StructuredOutputsParams>, ApiError> {
|
||||
if let Some(raw) = structured_outputs {
|
||||
return Ok(Some(deserialize_structured_outputs(raw)?));
|
||||
}
|
||||
|
||||
let Some(raw) = response_format else {
|
||||
return Ok(None);
|
||||
};
|
||||
|
||||
// Deserialize into our typed enum and delegate.
|
||||
let fmt: ResponseFormat = serde_json::from_value(raw.clone()).map_err(|e| {
|
||||
ApiError::invalid_request(
|
||||
format!("invalid response_format: {e}"),
|
||||
Some("response_format"),
|
||||
)
|
||||
})?;
|
||||
convert_from_response_format(Some(&fmt), &None)
|
||||
}
|
||||
@@ -0,0 +1,425 @@
|
||||
use std::collections::HashMap;
|
||||
use std::slice;
|
||||
|
||||
use llm_multimodal::ImageDetail;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use serde_json::Value;
|
||||
|
||||
// ============================================================================
|
||||
// Constants
|
||||
// ============================================================================
|
||||
|
||||
/// Default model identifier used when no model is specified.
|
||||
pub const UNKNOWN_MODEL_ID: &str = "unknown";
|
||||
|
||||
// ============================================================================
|
||||
// Default value helpers
|
||||
// ============================================================================
|
||||
|
||||
/// Helper function for serde default value (returns true).
|
||||
pub fn default_true() -> bool {
|
||||
true
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// String/Array Utilities
|
||||
// ============================================================================
|
||||
|
||||
/// A type that can be either a single string or an array of strings.
|
||||
#[derive(Debug, Clone, PartialEq, Deserialize, Serialize)]
|
||||
#[serde(untagged)]
|
||||
pub enum StringOrArray {
|
||||
String(String),
|
||||
Array(Vec<String>),
|
||||
}
|
||||
|
||||
impl StringOrArray {
|
||||
pub fn as_slice(&self) -> &[String] {
|
||||
match self {
|
||||
StringOrArray::String(s) => slice::from_ref(s),
|
||||
StringOrArray::Array(arr) => arr,
|
||||
}
|
||||
}
|
||||
|
||||
#[allow(unused)]
|
||||
pub fn into_vec(self) -> Vec<String> {
|
||||
match self {
|
||||
StringOrArray::String(s) => vec![s],
|
||||
StringOrArray::Array(arr) => arr,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Validates stop sequences (non-empty strings)
|
||||
pub fn validate_stop(stop: &StringOrArray) -> Result<(), validator::ValidationError> {
|
||||
if stop.as_slice().iter().any(|s| s.is_empty()) {
|
||||
return Err(validator::ValidationError::new(
|
||||
"stop strings cannot be empty",
|
||||
));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Validation helpers
|
||||
// ============================================================================
|
||||
|
||||
/// Validates top_p: 0.0 < top_p <= 1.0.
|
||||
pub fn validate_top_p_value(top_p: f32) -> Result<(), validator::ValidationError> {
|
||||
if !(top_p > 0.0 && top_p <= 1.0) {
|
||||
return Err(validator::ValidationError::new(
|
||||
"top_p must be in (0, 1] - greater than 0.0 and at most 1.0",
|
||||
));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Content Parts (for multimodal messages)
|
||||
// ============================================================================
|
||||
|
||||
#[derive(Debug, Clone, Deserialize, Serialize, PartialEq)]
|
||||
#[serde(tag = "type")]
|
||||
pub enum ContentPart {
|
||||
#[serde(rename = "text")]
|
||||
Text { text: String },
|
||||
#[serde(rename = "image_url")]
|
||||
ImageUrl {
|
||||
image_url: ImageUrl,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
uuid: Option<String>,
|
||||
},
|
||||
#[serde(rename = "video_url")]
|
||||
VideoUrl { video_url: VideoUrl },
|
||||
}
|
||||
|
||||
#[serde_with::skip_serializing_none]
|
||||
#[derive(Debug, Clone, Deserialize, Serialize, PartialEq)]
|
||||
pub struct ImageUrl {
|
||||
pub url: String,
|
||||
pub detail: Option<ImageDetail>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Deserialize, Serialize, PartialEq)]
|
||||
pub struct VideoUrl {
|
||||
pub url: String,
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Streaming
|
||||
// ============================================================================
|
||||
|
||||
/// Mirrors the Python vLLM `StreamOptions` class.
|
||||
#[derive(Debug, Clone, Deserialize, Serialize)]
|
||||
pub struct StreamOptions {
|
||||
pub include_usage: Option<bool>,
|
||||
pub continuous_usage_stats: Option<bool>,
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Tools and Function Calling
|
||||
// ============================================================================
|
||||
|
||||
#[derive(Debug, Clone, Deserialize, Serialize)]
|
||||
pub struct Tool {
|
||||
#[serde(rename = "type")]
|
||||
pub tool_type: String,
|
||||
pub function: Function,
|
||||
}
|
||||
|
||||
#[serde_with::skip_serializing_none]
|
||||
#[derive(Debug, Clone, Deserialize, Serialize)]
|
||||
pub struct Function {
|
||||
pub name: String,
|
||||
pub description: Option<String>,
|
||||
pub parameters: Value,
|
||||
/// Whether to enable strict schema adherence (OpenAI structured outputs).
|
||||
pub strict: Option<bool>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Deserialize, Serialize)]
|
||||
pub struct ToolCall {
|
||||
pub id: String,
|
||||
#[serde(rename = "type")]
|
||||
pub tool_type: String,
|
||||
pub function: FunctionCallResponse,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Deserialize, Serialize)]
|
||||
pub struct FunctionCallResponse {
|
||||
pub name: String,
|
||||
#[serde(default)]
|
||||
pub arguments: Option<String>,
|
||||
}
|
||||
|
||||
#[serde_with::skip_serializing_none]
|
||||
#[derive(Debug, Clone, Deserialize, Serialize)]
|
||||
pub struct ToolCallDelta {
|
||||
pub index: u32,
|
||||
pub id: Option<String>,
|
||||
#[serde(rename = "type")]
|
||||
pub tool_type: Option<String>,
|
||||
pub function: Option<FunctionCallDelta>,
|
||||
}
|
||||
|
||||
#[serde_with::skip_serializing_none]
|
||||
#[derive(Debug, Clone, Deserialize, Serialize)]
|
||||
pub struct FunctionCallDelta {
|
||||
pub name: Option<String>,
|
||||
pub arguments: Option<String>,
|
||||
}
|
||||
|
||||
/// Tool choice value for simple string options.
|
||||
#[derive(Debug, Clone, Deserialize, Serialize)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum ToolChoiceValue {
|
||||
Auto,
|
||||
Required,
|
||||
None,
|
||||
}
|
||||
|
||||
/// Tool choice for the Chat Completion API.
|
||||
#[derive(Debug, Clone, Deserialize, Serialize)]
|
||||
#[serde(untagged)]
|
||||
pub enum ToolChoice {
|
||||
Value(ToolChoiceValue),
|
||||
Function {
|
||||
#[serde(rename = "type")]
|
||||
tool_type: String,
|
||||
function: FunctionChoice,
|
||||
},
|
||||
AllowedTools {
|
||||
#[serde(rename = "type")]
|
||||
tool_type: String,
|
||||
mode: String,
|
||||
tools: Vec<ToolReference>,
|
||||
},
|
||||
}
|
||||
|
||||
impl Default for ToolChoice {
|
||||
fn default() -> Self {
|
||||
Self::Value(ToolChoiceValue::Auto)
|
||||
}
|
||||
}
|
||||
|
||||
/// Function choice specification for `ToolChoice::Function`.
|
||||
#[derive(Debug, Clone, Deserialize, Serialize)]
|
||||
pub struct FunctionChoice {
|
||||
pub name: String,
|
||||
}
|
||||
|
||||
/// Tool reference for `ToolChoice::AllowedTools`.
|
||||
#[derive(Debug, Clone, Deserialize, Serialize)]
|
||||
#[serde(tag = "type")]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum ToolReference {
|
||||
#[serde(rename = "function")]
|
||||
Function { name: String },
|
||||
#[serde(rename = "mcp")]
|
||||
Mcp {
|
||||
server_label: String,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
name: Option<String>,
|
||||
},
|
||||
#[serde(rename = "file_search")]
|
||||
FileSearch,
|
||||
#[serde(rename = "web_search_preview")]
|
||||
WebSearchPreview,
|
||||
#[serde(rename = "computer_use_preview")]
|
||||
ComputerUsePreview,
|
||||
#[serde(rename = "code_interpreter")]
|
||||
CodeInterpreter,
|
||||
#[serde(rename = "image_generation")]
|
||||
ImageGeneration,
|
||||
}
|
||||
|
||||
impl ToolReference {
|
||||
/// Get a unique identifier for this tool reference.
|
||||
pub fn identifier(&self) -> String {
|
||||
match self {
|
||||
ToolReference::Function { name } => format!("function:{name}"),
|
||||
ToolReference::Mcp {
|
||||
server_label,
|
||||
name: Some(n),
|
||||
} => format!("mcp:{server_label}:{n}"),
|
||||
ToolReference::Mcp {
|
||||
server_label,
|
||||
name: _,
|
||||
} => format!("mcp:{server_label}"),
|
||||
ToolReference::FileSearch => "file_search".to_string(),
|
||||
ToolReference::WebSearchPreview => "web_search_preview".to_string(),
|
||||
ToolReference::ComputerUsePreview => "computer_use_preview".to_string(),
|
||||
ToolReference::CodeInterpreter => "code_interpreter".to_string(),
|
||||
ToolReference::ImageGeneration => "image_generation".to_string(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Chat Messages
|
||||
// ============================================================================
|
||||
|
||||
#[serde_with::skip_serializing_none]
|
||||
#[derive(Debug, Clone, Deserialize, Serialize)]
|
||||
#[serde(tag = "role")]
|
||||
pub enum ChatMessage {
|
||||
#[serde(rename = "system")]
|
||||
System {
|
||||
content: MessageContent,
|
||||
name: Option<String>,
|
||||
},
|
||||
#[serde(rename = "user")]
|
||||
User {
|
||||
content: MessageContent,
|
||||
name: Option<String>,
|
||||
},
|
||||
#[serde(rename = "assistant")]
|
||||
Assistant {
|
||||
content: Option<MessageContent>,
|
||||
name: Option<String>,
|
||||
tool_calls: Option<Vec<ToolCall>>,
|
||||
/// Reasoning content for reasoning-capable models.
|
||||
#[serde(alias = "reasoning_content")]
|
||||
#[serde(alias = "thinking")]
|
||||
reasoning: Option<String>,
|
||||
},
|
||||
#[serde(rename = "tool")]
|
||||
Tool {
|
||||
content: MessageContent,
|
||||
tool_call_id: String,
|
||||
},
|
||||
#[serde(rename = "function")]
|
||||
Function { content: String, name: String },
|
||||
#[serde(rename = "developer")]
|
||||
Developer {
|
||||
content: MessageContent,
|
||||
tools: Option<Vec<Tool>>,
|
||||
name: Option<String>,
|
||||
},
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Deserialize, Serialize, PartialEq)]
|
||||
#[serde(untagged)]
|
||||
pub enum MessageContent {
|
||||
Text(String),
|
||||
Parts(Vec<ContentPart>),
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Usage and Logging
|
||||
// ============================================================================
|
||||
|
||||
/// Mirrors the Python vLLM `UsageInfo` class.
|
||||
#[serde_with::skip_serializing_none]
|
||||
#[derive(Debug, Clone, Serialize)]
|
||||
pub struct Usage {
|
||||
pub prompt_tokens: u32,
|
||||
pub total_tokens: u32,
|
||||
pub completion_tokens: Option<u32>,
|
||||
pub prompt_tokens_details: Option<PromptTokenUsageInfo>,
|
||||
}
|
||||
|
||||
impl Usage {
|
||||
/// Create a Usage from prompt and completion token counts.
|
||||
pub fn from_counts(prompt_tokens: u32, completion_tokens: u32) -> Self {
|
||||
Self {
|
||||
prompt_tokens,
|
||||
total_tokens: prompt_tokens + completion_tokens,
|
||||
completion_tokens: Some(completion_tokens),
|
||||
prompt_tokens_details: None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Mirrors the Python vLLM `PromptTokenUsageInfo` class.
|
||||
#[serde_with::skip_serializing_none]
|
||||
#[derive(Debug, Clone, Serialize)]
|
||||
pub struct PromptTokenUsageInfo {
|
||||
pub cached_tokens: Option<u32>,
|
||||
}
|
||||
|
||||
/// OpenAI completions-style logprobs.
|
||||
#[derive(Debug, Clone, Deserialize, Serialize)]
|
||||
pub struct LogProbs {
|
||||
pub tokens: Vec<String>,
|
||||
pub token_logprobs: Vec<Option<f32>>,
|
||||
pub top_logprobs: Vec<Option<HashMap<String, f32>>>,
|
||||
pub text_offset: Vec<u32>,
|
||||
}
|
||||
|
||||
/// Mirrors the Python vLLM `ChatCompletionLogProbs` class.
|
||||
#[serde_with::skip_serializing_none]
|
||||
#[derive(Debug, Clone, Serialize)]
|
||||
pub struct ChatLogProbs {
|
||||
pub content: Option<Vec<ChatLogProbsContent>>,
|
||||
}
|
||||
|
||||
/// Mirrors the Python vLLM `ChatCompletionLogProbsContent` class.
|
||||
#[serde_with::skip_serializing_none]
|
||||
#[derive(Debug, Clone, Serialize)]
|
||||
pub struct ChatLogProbsContent {
|
||||
pub token: String,
|
||||
pub logprob: f32,
|
||||
pub bytes: Option<Vec<u8>>,
|
||||
pub top_logprobs: Vec<TopLogProb>,
|
||||
}
|
||||
|
||||
/// Mirrors the Python vLLM `ChatCompletionLogProb` class.
|
||||
#[serde_with::skip_serializing_none]
|
||||
#[derive(Debug, Clone, Serialize)]
|
||||
pub struct TopLogProb {
|
||||
pub token: String,
|
||||
pub logprob: f32,
|
||||
pub bytes: Option<Vec<u8>>,
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Error Types
|
||||
// ============================================================================
|
||||
|
||||
#[derive(Debug, Clone, Deserialize, Serialize)]
|
||||
pub struct ErrorResponse {
|
||||
pub error: ErrorDetail,
|
||||
}
|
||||
|
||||
#[serde_with::skip_serializing_none]
|
||||
#[derive(Debug, Clone, Deserialize, Serialize)]
|
||||
pub struct ErrorDetail {
|
||||
pub message: String,
|
||||
#[serde(rename = "type")]
|
||||
pub error_type: String,
|
||||
pub param: Option<String>,
|
||||
pub code: Option<String>,
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Model types
|
||||
// ============================================================================
|
||||
|
||||
/// A single model entry in the `/v1/models` response.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct ModelObject {
|
||||
pub id: String,
|
||||
pub object: String,
|
||||
pub created: i64,
|
||||
pub owned_by: String,
|
||||
}
|
||||
|
||||
/// Response body for `GET /v1/models`.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct ListModelsResponse {
|
||||
pub object: String,
|
||||
pub data: Vec<ModelObject>,
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Normalizable trait
|
||||
// ============================================================================
|
||||
|
||||
/// Trait for request types that need post-deserialization normalization.
|
||||
pub trait Normalizable {
|
||||
/// Normalize the request by applying defaults and transformations.
|
||||
fn normalize(&mut self) {
|
||||
// Default: no-op
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,55 @@
|
||||
//! Validated JSON extractor for automatic request validation.
|
||||
//! Variation of https://github.com/lightseekorg/smg/blob/main/crates/protocols/src/validated.rs
|
||||
|
||||
use axum::Json;
|
||||
use axum::extract::rejection::JsonRejection;
|
||||
use axum::extract::{FromRequest, Request};
|
||||
use serde::de::DeserializeOwned;
|
||||
use validator::Validate;
|
||||
|
||||
use super::types::Normalizable;
|
||||
use crate::error::{ApiError, invalid_request};
|
||||
|
||||
/// A JSON extractor that automatically validates and normalizes the request
|
||||
/// body.
|
||||
///
|
||||
/// This extractor deserializes the request body and automatically calls
|
||||
/// `.validate()` on types that implement the `Validate` trait. If validation
|
||||
/// fails, it returns [`ApiError::InvalidRequest`] with details about the
|
||||
/// validation errors.
|
||||
pub struct ValidatedJson<T>(pub T);
|
||||
|
||||
impl<S, T> FromRequest<S> for ValidatedJson<T>
|
||||
where
|
||||
T: DeserializeOwned + Validate + Normalizable + Send,
|
||||
S: Send + Sync,
|
||||
{
|
||||
type Rejection = ApiError;
|
||||
|
||||
async fn from_request(req: Request, state: &S) -> Result<Self, Self::Rejection> {
|
||||
let Json(mut data) = Json::<T>::from_request(req, state)
|
||||
.await
|
||||
.map_err(|err: JsonRejection| ApiError::json_parse_error(err.body_text()))?;
|
||||
|
||||
data.normalize();
|
||||
|
||||
data.validate()
|
||||
.map_err(|validation_errors| invalid_request!("{}", validation_errors))?;
|
||||
|
||||
Ok(ValidatedJson(data))
|
||||
}
|
||||
}
|
||||
|
||||
impl<T> std::ops::Deref for ValidatedJson<T> {
|
||||
type Target = T;
|
||||
|
||||
fn deref(&self) -> &Self::Target {
|
||||
&self.0
|
||||
}
|
||||
}
|
||||
|
||||
impl<T> std::ops::DerefMut for ValidatedJson<T> {
|
||||
fn deref_mut(&mut self) -> &mut Self::Target {
|
||||
&mut self.0
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,78 @@
|
||||
use std::sync::Arc;
|
||||
|
||||
use axum::Json;
|
||||
use axum::extract::{Query, State};
|
||||
use axum::http::StatusCode;
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
use crate::error::ApiError;
|
||||
use crate::state::AppState;
|
||||
use crate::utils::utility_call_error;
|
||||
|
||||
#[derive(Serialize)]
|
||||
pub(crate) struct IsSleepingResponse {
|
||||
is_sleeping: bool,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
pub(crate) struct SleepParams {
|
||||
#[serde(default = "default_sleep_level")]
|
||||
level: u32,
|
||||
#[serde(default = "default_sleep_mode")]
|
||||
mode: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, Default, Deserialize)]
|
||||
pub(crate) struct WakeUpParams {
|
||||
#[serde(default)]
|
||||
tags: Option<Vec<String>>,
|
||||
}
|
||||
|
||||
const fn default_sleep_level() -> u32 {
|
||||
1
|
||||
}
|
||||
|
||||
fn default_sleep_mode() -> String {
|
||||
"abort".to_string()
|
||||
}
|
||||
|
||||
/// Put the engine to sleep.
|
||||
pub async fn sleep(
|
||||
State(state): State<Arc<AppState>>,
|
||||
Query(params): Query<SleepParams>,
|
||||
) -> Result<StatusCode, ApiError> {
|
||||
state
|
||||
.engine_core_client()
|
||||
.sleep(params.level, ¶ms.mode)
|
||||
.await
|
||||
.map_err(|error| utility_call_error("sleep", error))?;
|
||||
|
||||
Ok(StatusCode::OK)
|
||||
}
|
||||
|
||||
/// Wake the engine from sleep mode.
|
||||
pub async fn wake_up(
|
||||
State(state): State<Arc<AppState>>,
|
||||
Query(params): Query<WakeUpParams>,
|
||||
) -> Result<StatusCode, ApiError> {
|
||||
state
|
||||
.engine_core_client()
|
||||
.wake_up(params.tags)
|
||||
.await
|
||||
.map_err(|error| utility_call_error("wake_up", error))?;
|
||||
|
||||
Ok(StatusCode::OK)
|
||||
}
|
||||
|
||||
/// Return whether the engine is currently sleeping at any level.
|
||||
pub async fn is_sleeping(
|
||||
State(state): State<Arc<AppState>>,
|
||||
) -> Result<Json<IsSleepingResponse>, ApiError> {
|
||||
let is_sleeping = state
|
||||
.engine_core_client()
|
||||
.is_sleeping()
|
||||
.await
|
||||
.map_err(|error| utility_call_error("is_sleeping", error))?;
|
||||
|
||||
Ok(Json(IsSleepingResponse { is_sleeping }))
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,120 @@
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::{AtomicU64, Ordering};
|
||||
|
||||
use tokio::time::{Duration, Instant, sleep_until};
|
||||
use tracing::warn;
|
||||
use vllm_chat::ChatLlm;
|
||||
use vllm_engine_core_client::EngineCoreClient;
|
||||
|
||||
const SHUTDOWN_REFCOUNT_POLL_INTERVAL: Duration = Duration::from_millis(100);
|
||||
|
||||
/// Shared router state for the minimal single-model OpenAI server.
|
||||
pub struct AppState {
|
||||
/// All public model IDs served by this frontend. The first entry is the
|
||||
/// primary ID used in responses; all entries are valid in requests.
|
||||
served_model_names: Vec<String>,
|
||||
/// Shared chat facade used by all requests.
|
||||
pub chat: ChatLlm,
|
||||
/// Whether to log a summary line for each completed request.
|
||||
pub enable_log_requests: bool,
|
||||
/// Number of in-flight inference requests currently owned by this frontend.
|
||||
server_load: AtomicU64,
|
||||
}
|
||||
|
||||
impl AppState {
|
||||
/// Construct one application state instance.
|
||||
///
|
||||
/// `served_model_names` must be non-empty; the first entry is the primary
|
||||
/// model ID returned in API responses.
|
||||
///
|
||||
/// # Panics
|
||||
///
|
||||
/// Panics if `served_model_names` is empty.
|
||||
pub fn new(served_model_names: Vec<String>, chat: ChatLlm) -> Self {
|
||||
assert!(
|
||||
!served_model_names.is_empty(),
|
||||
"served_model_names must not be empty"
|
||||
);
|
||||
Self {
|
||||
served_model_names,
|
||||
chat,
|
||||
enable_log_requests: false,
|
||||
server_load: AtomicU64::new(0),
|
||||
}
|
||||
}
|
||||
|
||||
/// Enable per-request completion logging.
|
||||
pub fn with_log_requests(mut self, enabled: bool) -> Self {
|
||||
self.enable_log_requests = enabled;
|
||||
self
|
||||
}
|
||||
|
||||
/// The primary model name echoed back in API responses (the first served
|
||||
/// name).
|
||||
pub fn primary_model_name(&self) -> &str {
|
||||
self.served_model_names.first().map(String::as_str).unwrap_or_default()
|
||||
}
|
||||
|
||||
/// All model names served by this frontend.
|
||||
pub fn served_model_names(&self) -> &[String] {
|
||||
&self.served_model_names
|
||||
}
|
||||
|
||||
/// Return a reference to the underlying engine core client for utility
|
||||
/// calls.
|
||||
pub(crate) fn engine_core_client(&self) -> &EngineCoreClient {
|
||||
self.chat.engine_core_client()
|
||||
}
|
||||
|
||||
/// Return the current in-flight inference request count for the `/load`
|
||||
/// endpoint.
|
||||
pub fn server_load(&self) -> u64 {
|
||||
self.server_load.load(Ordering::Relaxed)
|
||||
}
|
||||
|
||||
/// Increment the in-flight inference request count, called by the load
|
||||
/// tracking middleware.
|
||||
pub(crate) fn increment_server_load(&self) {
|
||||
self.server_load.fetch_add(1, Ordering::Relaxed);
|
||||
}
|
||||
|
||||
/// Decrement the in-flight inference request count, called by the load
|
||||
/// tracking middleware.
|
||||
pub(crate) fn decrement_server_load(&self) {
|
||||
self.server_load.fetch_sub(1, Ordering::Relaxed);
|
||||
}
|
||||
|
||||
/// Wait until all request-owned references are dropped, then shut down the
|
||||
/// engine client.
|
||||
///
|
||||
/// If the deadline elapses while request/connection tasks still hold state
|
||||
/// references, skip the clean engine-client shutdown and let process
|
||||
/// teardown reclaim the remaining resources.
|
||||
pub async fn shutdown(mut self: Arc<Self>, deadline: Instant) -> anyhow::Result<()> {
|
||||
loop {
|
||||
match Arc::try_unwrap(self) {
|
||||
Ok(state) => {
|
||||
state.chat.shutdown().await?;
|
||||
return Ok(());
|
||||
}
|
||||
Err(state) => self = state,
|
||||
}
|
||||
let ref_count = Arc::strong_count(&self);
|
||||
|
||||
let now = Instant::now();
|
||||
if now >= deadline {
|
||||
warn!(
|
||||
ref_count,
|
||||
"shutdown deadline elapsed before app state became idle; skipping engine-client shutdown"
|
||||
);
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
sleep_until(std::cmp::min(
|
||||
deadline,
|
||||
now + SHUTDOWN_REFCOUNT_POLL_INTERVAL,
|
||||
))
|
||||
.await;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,106 @@
|
||||
use std::collections::HashMap;
|
||||
use std::time::{SystemTime, UNIX_EPOCH};
|
||||
|
||||
use axum::http::HeaderMap;
|
||||
use serde_json::Value;
|
||||
use thiserror_ext::AsReport;
|
||||
use uuid::Uuid;
|
||||
|
||||
use crate::error::ApiError;
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Default)]
|
||||
pub struct ResolvedRequestContext {
|
||||
pub request_id: String,
|
||||
pub data_parallel_rank: Option<u32>,
|
||||
}
|
||||
|
||||
/// Return the current Unix timestamp in seconds for OpenAI response objects.
|
||||
pub fn unix_timestamp() -> u64 {
|
||||
SystemTime::now()
|
||||
.duration_since(UNIX_EPOCH)
|
||||
.map(|duration| duration.as_secs())
|
||||
.unwrap_or_default()
|
||||
}
|
||||
|
||||
/// Construct an API error for a failed utility call to the engine core.
|
||||
pub fn utility_call_error(method: &str, error: impl AsReport) -> ApiError {
|
||||
ApiError::server_error(format!("failed to call {method}: {}", error.as_report()))
|
||||
}
|
||||
|
||||
/// Merge `kv_transfer_params` into the `vllm_xargs` map, mirroring the Python
|
||||
/// vLLM behavior where `kv_transfer_params` is injected into `extra_args` for
|
||||
/// engine-core consumption.
|
||||
pub fn merge_kv_transfer_params(
|
||||
mut xargs: Option<HashMap<String, Value>>,
|
||||
kv_transfer_params: Option<&HashMap<String, Value>>,
|
||||
) -> Option<HashMap<String, Value>> {
|
||||
if let Some(kv_params) = kv_transfer_params {
|
||||
let map = xargs.get_or_insert_with(HashMap::new);
|
||||
map.insert(
|
||||
"kv_transfer_params".to_string(),
|
||||
// This is safe because we know that `kv_params` is already valid JSON.
|
||||
serde_json::to_value(kv_params).unwrap(),
|
||||
);
|
||||
}
|
||||
xargs
|
||||
}
|
||||
|
||||
/// Convert OpenAI-style `logit_bias` with string token-ID keys into the
|
||||
/// internal `HashMap<u32, f32>` representation, validating that every key
|
||||
/// parses as a `u32`.
|
||||
pub fn convert_logit_bias(
|
||||
logit_bias: Option<HashMap<String, f32>>,
|
||||
) -> Result<Option<HashMap<u32, f32>>, ApiError> {
|
||||
logit_bias
|
||||
.map(|bias| {
|
||||
bias.into_iter()
|
||||
.map(|(key, value)| {
|
||||
key.parse().map(|k| (k, value)).map_err(|_| {
|
||||
ApiError::invalid_request(
|
||||
format!(
|
||||
"Invalid key in 'logit_bias': '{key}' is not a valid token ID. \
|
||||
Token IDs must be non-negative integers."
|
||||
),
|
||||
Some("logit_bias"),
|
||||
)
|
||||
})
|
||||
})
|
||||
.collect()
|
||||
})
|
||||
.transpose()
|
||||
}
|
||||
|
||||
/// Extract common request metadata from HTTP headers: the external request ID
|
||||
/// and the optional data-parallel rank used for engine routing.
|
||||
pub fn resolve_request_context(
|
||||
headers: &HeaderMap,
|
||||
request_id: Option<&str>,
|
||||
) -> ResolvedRequestContext {
|
||||
// `None` when the header is absent or cannot be parsed as a `u32`.
|
||||
let data_parallel_rank = headers
|
||||
.get("X-data-parallel-rank")
|
||||
.and_then(|v| v.to_str().ok())
|
||||
.and_then(|s| s.trim().parse().ok());
|
||||
|
||||
// Extract request id from header.
|
||||
let request_id_header = headers.get("X-Request-Id").and_then(|value| value.to_str().ok());
|
||||
let request_id = resolve_base_request_id(request_id_header, request_id);
|
||||
|
||||
ResolvedRequestContext {
|
||||
request_id,
|
||||
data_parallel_rank,
|
||||
}
|
||||
}
|
||||
|
||||
/// Resolve the base external request ID before API-specific prefixes such as
|
||||
/// `chatcmpl-`.
|
||||
pub fn resolve_base_request_id(
|
||||
request_id_header: Option<&str>,
|
||||
request_id: Option<&str>,
|
||||
) -> String {
|
||||
request_id_header.or(request_id).map(ToOwned::to_owned).unwrap_or_else(|| {
|
||||
let mut id = Uuid::new_v4().simple().to_string();
|
||||
id.truncate(8);
|
||||
id
|
||||
})
|
||||
}
|
||||
Reference in New Issue
Block a user