[Rust Frontend] Add standalone Rust renderer (#50289)
This commit is contained in:
+39
-1
@@ -15,6 +15,7 @@ The component is organized as a Cargo workspace with several crates, layered bot
|
||||
│ vllm-cmd / vllm-rs │ CLI entrypoint:
|
||||
│ │ Python vLLM frontend subprocess
|
||||
│ │ Rust managed-engine serve mode
|
||||
│ │ Engine-free render mode
|
||||
├─────────────────────────────────┤
|
||||
│ vllm-server │ OpenAI-compatible HTTP API (axum)
|
||||
├─────────────────────────────────┤
|
||||
@@ -74,9 +75,46 @@ To build the `vllm-rs` in isolation:
|
||||
./build_rust.sh
|
||||
```
|
||||
|
||||
### Engine-free renderer
|
||||
|
||||
`vllm-rs render` serves text-only request preprocessing without starting or
|
||||
connecting to a Python inference engine:
|
||||
|
||||
```bash
|
||||
cargo run --manifest-path rust/Cargo.toml -p vllm-cmd --release -- \
|
||||
render Qwen/Qwen3-32B \
|
||||
--host 127.0.0.1 --max-model-len 32768
|
||||
```
|
||||
|
||||
It exposes `/v1/chat/completions/render` and `/v1/completions/render`. Only
|
||||
tokenizer and model configuration files are loaded; model weights, PyTorch,
|
||||
and vLLM kernels are not required.
|
||||
|
||||
The render endpoints return the public token-in `GenerateRequest` consumed by
|
||||
the Rust `/inference/v1/generate` endpoint. A chat render response, or one item
|
||||
from a completion render response, can be submitted to that endpoint without
|
||||
changing its fields.
|
||||
|
||||
The render and inference paths use the same `vllm-chat` and `vllm-text`
|
||||
request-preparation logic; render mode stops before engine submission.
|
||||
Tool-call and reasoning parsers use model-based auto-detection by default. Use
|
||||
`--tool-call-parser` and `--reasoning-parser` to override either selection;
|
||||
unified parsers require the same selection for both options.
|
||||
|
||||
```bash
|
||||
curl http://127.0.0.1:8000/v1/chat/completions/render \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{
|
||||
"model": "Qwen/Qwen3-32B",
|
||||
"messages": [{"role": "user", "content": "Hello"}],
|
||||
"max_completion_tokens": 16
|
||||
}'
|
||||
```
|
||||
|
||||
### Example Request
|
||||
|
||||
After either startup path, you can use any OpenAI-compatible client:
|
||||
After either full-frontend startup path, you can use any OpenAI-compatible
|
||||
client against the inference endpoints:
|
||||
|
||||
```bash
|
||||
curl http://127.0.0.1:8000/v1/chat/completions \
|
||||
|
||||
+31
-22
@@ -97,6 +97,10 @@ pub struct ChatRequestProcessor {
|
||||
/// Effective model dtype reported by the engine.
|
||||
/// Absent for text-only frontends without an engine handshake.
|
||||
model_dtype: Option<ModelDtype>,
|
||||
/// Tool-call parser selection used when preparing generation requests.
|
||||
tool_call_parser: ParserSelection,
|
||||
/// Reasoning parser selection used when preparing generation requests.
|
||||
reasoning_parser: ParserSelection,
|
||||
}
|
||||
|
||||
impl ChatRequestProcessor {
|
||||
@@ -106,6 +110,8 @@ impl ChatRequestProcessor {
|
||||
Self {
|
||||
backend,
|
||||
model_dtype: Some(model_dtype),
|
||||
tool_call_parser: ParserSelection::Auto,
|
||||
reasoning_parser: ParserSelection::Auto,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -114,9 +120,22 @@ impl ChatRequestProcessor {
|
||||
Self {
|
||||
backend,
|
||||
model_dtype: None,
|
||||
tool_call_parser: ParserSelection::Auto,
|
||||
reasoning_parser: ParserSelection::Auto,
|
||||
}
|
||||
}
|
||||
|
||||
/// Configure the parser selections used to prepare generation requests.
|
||||
pub fn with_parser_selections(
|
||||
mut self,
|
||||
tool_call_parser: ParserSelection,
|
||||
reasoning_parser: ParserSelection,
|
||||
) -> Self {
|
||||
self.tool_call_parser = tool_call_parser;
|
||||
self.reasoning_parser = reasoning_parser;
|
||||
self
|
||||
}
|
||||
|
||||
async fn finalize_rendered_prompt(
|
||||
&self,
|
||||
request: &ChatRequest,
|
||||
@@ -196,10 +215,15 @@ impl ChatRequestProcessor {
|
||||
pub async fn prepare(
|
||||
&self,
|
||||
mut request: ChatRequest,
|
||||
options: NewChatOutputProcessorOptions<'_>,
|
||||
) -> Result<(TextRequest, DynChatOutputProcessor)> {
|
||||
request.validate()?;
|
||||
let output_processor = self.backend.new_chat_output_processor(&mut request, options)?;
|
||||
let output_processor = self.backend.new_chat_output_processor(
|
||||
&mut request,
|
||||
NewChatOutputProcessorOptions {
|
||||
tool_call_parser: &self.tool_call_parser,
|
||||
reasoning_parser: &self.reasoning_parser,
|
||||
},
|
||||
)?;
|
||||
let text_request = self.prepare_text_request(request).await?;
|
||||
Ok((text_request, output_processor))
|
||||
}
|
||||
@@ -213,10 +237,6 @@ impl ChatRequestProcessor {
|
||||
pub struct ChatLlm {
|
||||
text: TextLlm,
|
||||
processor: ChatRequestProcessor,
|
||||
/// Tool-call parser selection.
|
||||
tool_call_parser: ParserSelection,
|
||||
/// Reasoning parser selection.
|
||||
reasoning_parser: ParserSelection,
|
||||
}
|
||||
|
||||
impl ChatLlm {
|
||||
@@ -228,8 +248,6 @@ impl ChatLlm {
|
||||
Self {
|
||||
text,
|
||||
processor: ChatRequestProcessor::new(backend, model_dtype),
|
||||
tool_call_parser: ParserSelection::Auto,
|
||||
reasoning_parser: ParserSelection::Auto,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -242,13 +260,13 @@ impl ChatLlm {
|
||||
|
||||
/// Set tool-call parser selection.
|
||||
pub fn with_tool_call_parser(mut self, selection: ParserSelection) -> Self {
|
||||
self.tool_call_parser = selection;
|
||||
self.processor.tool_call_parser = selection;
|
||||
self
|
||||
}
|
||||
|
||||
/// Set reasoning parser selection.
|
||||
pub fn with_reasoning_parser(mut self, selection: ParserSelection) -> Self {
|
||||
self.reasoning_parser = selection;
|
||||
self.processor.reasoning_parser = selection;
|
||||
self
|
||||
}
|
||||
|
||||
@@ -306,7 +324,7 @@ impl ChatLlm {
|
||||
|
||||
/// Effective tool-call parser name for this model, if parsing is enabled.
|
||||
pub fn tool_call_parser_name(&self) -> Option<&str> {
|
||||
match &self.tool_call_parser {
|
||||
match &self.processor.tool_call_parser {
|
||||
ParserSelection::Auto => {
|
||||
ToolParserFactory::global().resolve_name_for_model(self.model_id())
|
||||
}
|
||||
@@ -317,7 +335,7 @@ impl ChatLlm {
|
||||
|
||||
/// Effective reasoning parser name for this model, if parsing is enabled.
|
||||
pub fn reasoning_parser_name(&self) -> Option<&str> {
|
||||
match &self.reasoning_parser {
|
||||
match &self.processor.reasoning_parser {
|
||||
ParserSelection::Auto => {
|
||||
ReasoningParserFactory::global().resolve_name_for_model(self.model_id())
|
||||
}
|
||||
@@ -328,16 +346,7 @@ impl ChatLlm {
|
||||
|
||||
/// Render, tokenize, and submit one chat request.
|
||||
pub async fn chat(&self, request: ChatRequest) -> Result<ChatEventStream> {
|
||||
let (text_request, output_processor) = self
|
||||
.processor
|
||||
.prepare(
|
||||
request,
|
||||
NewChatOutputProcessorOptions {
|
||||
tool_call_parser: &self.tool_call_parser,
|
||||
reasoning_parser: &self.reasoning_parser,
|
||||
},
|
||||
)
|
||||
.await?;
|
||||
let (text_request, output_processor) = self.processor.prepare(request).await?;
|
||||
let request_id = text_request.request_id.clone();
|
||||
let decoded_stream = self.text.generate(text_request).await?.map_err(Error::from).boxed();
|
||||
|
||||
|
||||
+65
-1
@@ -29,7 +29,8 @@ use vllm_managed_engine::ManagedEngineConfig;
|
||||
use vllm_managed_engine::cli::{ManagedEngineArgs, repartition_managed_engine_args};
|
||||
use vllm_server::{
|
||||
ApiServerOptions, ChatTemplateContentFormatOption, Config, CoordinatorMode, CorsConfig,
|
||||
DEFAULT_KEEP_ALIVE_TIMEOUT, HttpListenerMode, ParserSelection, RendererSelection, TlsConfig,
|
||||
DEFAULT_KEEP_ALIVE_TIMEOUT, HttpListenerMode, ParserSelection, RenderConfig, RendererSelection,
|
||||
TlsConfig,
|
||||
};
|
||||
|
||||
use crate::cli::unsupported::UnsupportedArgs;
|
||||
@@ -90,6 +91,8 @@ pub enum Command {
|
||||
/// Run vLLM benchmarks.
|
||||
#[command(subcommand)]
|
||||
Bench(BenchCommand),
|
||||
/// Run engine-free request rendering and preprocessing.
|
||||
Render(RenderArgs),
|
||||
}
|
||||
|
||||
/// Supported benchmark commands.
|
||||
@@ -99,6 +102,67 @@ pub enum BenchCommand {
|
||||
Serve(vllm_bench::BenchServeArgs),
|
||||
}
|
||||
|
||||
/// Arguments for the engine-free text renderer.
|
||||
#[derive(Debug, Args, PartialEq, Eq)]
|
||||
pub struct RenderArgs {
|
||||
/// Model identifier or local model directory containing tokenizer files.
|
||||
model: String,
|
||||
/// HTTP bind host.
|
||||
#[arg(long, default_value = "127.0.0.1")]
|
||||
host: String,
|
||||
/// HTTP bind port.
|
||||
#[arg(long, default_value_t = 8000)]
|
||||
port: u16,
|
||||
/// Public model names accepted by the API.
|
||||
#[arg(long, num_args = 0..)]
|
||||
served_model_name: Vec<String>,
|
||||
/// Select the tool-call parser. Use `auto` to infer from the model or
|
||||
/// `none` to disable parsing.
|
||||
#[arg(long, default_value_t)]
|
||||
tool_call_parser: ParserSelection,
|
||||
/// Select the reasoning parser. Use `auto` to infer from the model or
|
||||
/// `none` to disable parsing.
|
||||
#[arg(long, default_value_t)]
|
||||
reasoning_parser: ParserSelection,
|
||||
/// Select the native chat renderer implementation.
|
||||
#[arg(long = "tokenizer-mode", default_value_t)]
|
||||
renderer: RendererSelection,
|
||||
/// Override the model chat template with a file path or inline template.
|
||||
#[arg(long)]
|
||||
chat_template: Option<String>,
|
||||
/// Default JSON keyword arguments merged into every chat-template render.
|
||||
#[arg(long, value_parser = parse_json::<HashMap<String, Value>>, value_name = "JSON")]
|
||||
default_chat_template_kwargs: Option<HashMap<String, Value>>,
|
||||
/// How message content is exposed to the chat template.
|
||||
#[arg(long, default_value_t)]
|
||||
chat_template_content_format: ChatTemplateContentFormatOption,
|
||||
/// Maximum model context length used for request validation.
|
||||
#[arg(long)]
|
||||
max_model_len: u32,
|
||||
/// Maximum accepted logprobs count; -1 disables the cap.
|
||||
#[arg(long, value_parser = clap::value_parser!(i32).range(-1..), allow_negative_numbers = true)]
|
||||
max_logprobs: Option<i32>,
|
||||
}
|
||||
|
||||
impl RenderArgs {
|
||||
pub(super) fn into_config(self) -> RenderConfig {
|
||||
RenderConfig {
|
||||
model: self.model,
|
||||
served_model_name: self.served_model_name,
|
||||
host: self.host,
|
||||
port: self.port,
|
||||
tool_call_parser: self.tool_call_parser,
|
||||
reasoning_parser: self.reasoning_parser,
|
||||
renderer: self.renderer,
|
||||
chat_template: self.chat_template,
|
||||
default_chat_template_kwargs: self.default_chat_template_kwargs.unwrap_or_default(),
|
||||
chat_template_content_format: self.chat_template_content_format,
|
||||
max_model_len: self.max_model_len,
|
||||
max_logprobs: self.max_logprobs,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// A JSON-encoded list of strings, matching Python's `json.loads` CLI type for
|
||||
/// the CORS list arguments (e.g. `--allowed-origins '["*"]'`). Parsing the whole
|
||||
/// value as one item keeps clap from treating the field as a repeated flag.
|
||||
|
||||
@@ -27,6 +27,41 @@ fn bench_serve_args_parse_without_managed_engine_repartition() {
|
||||
assert!(args.request_rate.is_infinite());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn render_args_build_supported_config() {
|
||||
let cli = Cli::try_parse_from([
|
||||
"vllm-rs",
|
||||
"render",
|
||||
"Qwen/Qwen2.5-0.5B-Instruct",
|
||||
"--port",
|
||||
"8080",
|
||||
"--max-model-len",
|
||||
"32768",
|
||||
"--served-model-name",
|
||||
"qwen",
|
||||
"--tokenizer-mode",
|
||||
"deepseek_v32",
|
||||
"--max-logprobs",
|
||||
"-1",
|
||||
])
|
||||
.unwrap();
|
||||
|
||||
let Command::Render(args) = cli.command else {
|
||||
panic!("expected render args");
|
||||
};
|
||||
let config = args.into_config();
|
||||
|
||||
assert_eq!(config.model, "Qwen/Qwen2.5-0.5B-Instruct");
|
||||
assert_eq!(config.host, "127.0.0.1");
|
||||
assert_eq!(config.port, 8080);
|
||||
assert_eq!(config.max_model_len, 32768);
|
||||
assert_eq!(config.served_model_name, ["qwen"]);
|
||||
assert_eq!(config.tool_call_parser, ParserSelection::Auto);
|
||||
assert_eq!(config.reasoning_parser, ParserSelection::Auto);
|
||||
assert_eq!(config.renderer, RendererSelection::DeepSeekV32);
|
||||
assert_eq!(config.max_logprobs, Some(-1));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn serve_args_forward_python_flags_with_separator() {
|
||||
let cli = Cli::try_parse_from([
|
||||
|
||||
@@ -211,5 +211,8 @@ async fn async_main(cli: Cli) -> Result<()> {
|
||||
)),
|
||||
}
|
||||
}
|
||||
Command::Render(args) => {
|
||||
vllm_server::serve_render(args.into_config(), shutdown_signal()).await
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -9,6 +9,7 @@ mod grpc;
|
||||
mod listener;
|
||||
mod lora;
|
||||
mod middleware;
|
||||
mod render;
|
||||
mod routes;
|
||||
mod runtime;
|
||||
mod server_info;
|
||||
@@ -35,6 +36,7 @@ use hyper::server::conn::http1;
|
||||
use hyper_util::rt::{TokioIo, TokioTimer};
|
||||
use hyper_util::server::graceful::GracefulShutdown;
|
||||
use hyper_util::service::TowerToHyperService;
|
||||
pub use render::{RenderConfig, serve_render};
|
||||
use tokio::net::TcpListener;
|
||||
use tokio::time::{Instant, sleep_until};
|
||||
use tokio_util::sync::CancellationToken;
|
||||
|
||||
@@ -0,0 +1,146 @@
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
// SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::sync::Arc;
|
||||
|
||||
use anyhow::{Context as _, Result, bail};
|
||||
use serde_json::Value;
|
||||
use tokio_util::sync::CancellationToken;
|
||||
use tracing::info;
|
||||
use vllm_chat::{
|
||||
ChatRequestProcessor, ChatTemplateContentFormatOption, LoadModelBackendsOptions,
|
||||
ParserSelection, RendererSelection, load_model_backends,
|
||||
};
|
||||
use vllm_text::TextRequestProcessor;
|
||||
|
||||
use crate::{HttpListenerMode, listener::Listener};
|
||||
|
||||
/// Configuration for the engine-free text rendering server.
|
||||
#[derive(Debug)]
|
||||
pub struct RenderConfig {
|
||||
pub model: String,
|
||||
pub served_model_name: Vec<String>,
|
||||
pub host: String,
|
||||
pub port: u16,
|
||||
pub tool_call_parser: ParserSelection,
|
||||
pub reasoning_parser: ParserSelection,
|
||||
pub renderer: RendererSelection,
|
||||
pub chat_template: Option<String>,
|
||||
pub default_chat_template_kwargs: HashMap<String, Value>,
|
||||
pub chat_template_content_format: ChatTemplateContentFormatOption,
|
||||
pub max_model_len: u32,
|
||||
pub max_logprobs: Option<i32>,
|
||||
}
|
||||
|
||||
impl RenderConfig {
|
||||
/// Validate configuration before initializing renderer/tokenizer backends
|
||||
/// or binding a listener.
|
||||
pub fn validate(&self) -> Result<()> {
|
||||
vllm_chat::validate_parser_overrides(&self.tool_call_parser, &self.reasoning_parser)?;
|
||||
if self.max_logprobs.is_some_and(|value| value < -1) {
|
||||
bail!("max_logprobs must be non-negative or -1");
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) struct RenderState {
|
||||
pub(crate) model: String,
|
||||
pub(crate) served_model_names: Vec<String>,
|
||||
pub(crate) text: TextRequestProcessor,
|
||||
pub(crate) chat: ChatRequestProcessor,
|
||||
}
|
||||
|
||||
async fn build_state(config: &RenderConfig) -> Result<Arc<RenderState>> {
|
||||
let loaded = load_model_backends(
|
||||
&config.model,
|
||||
LoadModelBackendsOptions {
|
||||
renderer: config.renderer,
|
||||
language_model_only: true,
|
||||
chat_template: config.chat_template.clone(),
|
||||
chat_template_content_format: config.chat_template_content_format,
|
||||
default_chat_template_kwargs: config.default_chat_template_kwargs.clone(),
|
||||
limit_mm_per_prompt: Default::default(),
|
||||
},
|
||||
)
|
||||
.await
|
||||
.context("failed to load renderer/tokenizer backends")?;
|
||||
let served_model_names =
|
||||
crate::effective_served_model_names(&config.model, &config.served_model_name);
|
||||
let text = TextRequestProcessor::new(loaded.text_backend, config.max_model_len)
|
||||
.with_max_logprobs(config.max_logprobs);
|
||||
let chat = ChatRequestProcessor::render_only(loaded.chat_backend).with_parser_selections(
|
||||
config.tool_call_parser.clone(),
|
||||
config.reasoning_parser.clone(),
|
||||
);
|
||||
Ok(Arc::new(RenderState {
|
||||
model: config.model.clone(),
|
||||
served_model_names,
|
||||
text,
|
||||
chat,
|
||||
}))
|
||||
}
|
||||
|
||||
/// Run the text-only preprocessing server without starting or connecting
|
||||
/// to an inference engine.
|
||||
pub async fn serve_render(config: RenderConfig, shutdown: CancellationToken) -> Result<()> {
|
||||
config.validate().context("invalid render server configuration")?;
|
||||
let state = tokio::select! {
|
||||
result = build_state(&config) => result?,
|
||||
_ = shutdown.cancelled() => return Ok(()),
|
||||
};
|
||||
let listener_mode = HttpListenerMode::BindTcp {
|
||||
host: config.host.clone(),
|
||||
port: config.port,
|
||||
};
|
||||
let listener = Listener::bind(&listener_mode)
|
||||
.await
|
||||
.with_context(|| format!("failed to bind {}:{}", config.host, config.port))?;
|
||||
let address = listener.local_addr_display()?;
|
||||
info!(
|
||||
%address,
|
||||
model = %config.model,
|
||||
"starting engine-free Rust render server"
|
||||
);
|
||||
axum::serve(listener, crate::routes::render::build_router(state))
|
||||
.with_graceful_shutdown(shutdown.cancelled_owned())
|
||||
.await
|
||||
.context("render server failed")
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[tokio::test]
|
||||
async fn serve_render_rejects_unknown_parser_before_startup() {
|
||||
let shutdown = CancellationToken::new();
|
||||
shutdown.cancel();
|
||||
let error = serve_render(
|
||||
RenderConfig {
|
||||
model: "test-model".to_string(),
|
||||
served_model_name: Vec::new(),
|
||||
host: "127.0.0.1".to_string(),
|
||||
port: 8000,
|
||||
tool_call_parser: ParserSelection::Auto,
|
||||
reasoning_parser: ParserSelection::Explicit("typo".to_string()),
|
||||
renderer: RendererSelection::Auto,
|
||||
chat_template: None,
|
||||
default_chat_template_kwargs: HashMap::new(),
|
||||
chat_template_content_format: ChatTemplateContentFormatOption::Auto,
|
||||
max_model_len: 128,
|
||||
max_logprobs: None,
|
||||
},
|
||||
shutdown,
|
||||
)
|
||||
.await
|
||||
.unwrap_err();
|
||||
let report = format!("{error:#}");
|
||||
|
||||
assert!(
|
||||
report.contains("reasoning parser `typo` is not registered"),
|
||||
"{report}"
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -12,6 +12,7 @@ mod metrics;
|
||||
pub(crate) mod openai;
|
||||
mod pause;
|
||||
mod profile;
|
||||
pub(super) mod render;
|
||||
mod server_info;
|
||||
mod sleep;
|
||||
mod tokenize;
|
||||
|
||||
@@ -26,10 +26,12 @@ use vllm_llm::{
|
||||
};
|
||||
|
||||
use self::convert::{ResponseOptions, prepare_generate_request};
|
||||
pub(crate) use self::types::GenerateRequest;
|
||||
use self::types::{
|
||||
GenerateLogprob, GenerateRequest, GenerateResponse, GenerateResponseChoice,
|
||||
GenerateResponseStreamChoice, GenerateStreamResponse,
|
||||
GenerateLogprob, GenerateResponse, GenerateResponseChoice, GenerateResponseStreamChoice,
|
||||
GenerateStreamResponse,
|
||||
};
|
||||
pub(crate) use self::validate::validate_request_compat;
|
||||
use crate::config::ApiServerOptions;
|
||||
use crate::error::{ApiError, bail_server_error, server_error, text_submit_error};
|
||||
use crate::routes::openai::utils::logprobs::clamp_logprob;
|
||||
|
||||
@@ -6,7 +6,7 @@ use crate::error::{ApiError, bail_invalid_request};
|
||||
|
||||
/// Enforce the minimal compatibility contract for the Rust token generate
|
||||
/// route.
|
||||
pub(super) fn validate_request_compat(
|
||||
pub(crate) fn validate_request_compat(
|
||||
request: &GenerateRequest,
|
||||
served_model_names: &[String],
|
||||
) -> Result<(), ApiError> {
|
||||
|
||||
@@ -22,17 +22,18 @@ use tracing::{debug, error, info, trace};
|
||||
use tracing_futures::Instrument as _;
|
||||
use vllm_chat::{
|
||||
AssistantBlockKind, AssistantMessageExt as _, ChatEvent, ChatEventStream, ChatEventStreamTrait,
|
||||
CollectedAssistantMessage, FinishReason,
|
||||
ChatRequest, CollectedAssistantMessage, FinishReason,
|
||||
};
|
||||
use vllm_engine_core_client::protocol::output::StopReason;
|
||||
|
||||
use self::convert::{ResponseOptions, prepare_chat_request};
|
||||
pub(crate) use self::types::ChatCompletionRequest;
|
||||
use crate::config::ApiServerOptions;
|
||||
use crate::error::{ApiError, bail_server_error, chat_submit_error, server_error};
|
||||
use crate::lora::LoraModelResolution;
|
||||
use crate::routes::openai::chat_completions::types::{
|
||||
AssistantRole, ChatCompletionChoice, ChatCompletionMessage, ChatCompletionRequest,
|
||||
ChatCompletionResponse, ChatCompletionStreamChoice, ChatCompletionStreamResponse,
|
||||
ChatMessageDelta,
|
||||
AssistantRole, ChatCompletionChoice, ChatCompletionMessage, ChatCompletionResponse,
|
||||
ChatCompletionStreamChoice, ChatCompletionStreamResponse, ChatMessageDelta,
|
||||
};
|
||||
use crate::routes::openai::utils::logprobs::{
|
||||
decoded_logprobs_to_openai_chat, prompt_logprobs_to_maps,
|
||||
@@ -43,7 +44,15 @@ use crate::routes::openai::utils::types::{
|
||||
use crate::routes::openai::utils::usage::ContinuousUsage;
|
||||
use crate::routes::openai::utils::validated_json::ValidatedJson;
|
||||
use crate::state::AppState;
|
||||
use crate::utils::{resolve_request_context, unix_timestamp};
|
||||
use crate::utils::{ResolvedRequestContext, resolve_request_context, unix_timestamp};
|
||||
|
||||
pub(crate) fn lower_chat_request(
|
||||
request: ChatCompletionRequest,
|
||||
lora_resolution: &LoraModelResolution,
|
||||
ctx: ResolvedRequestContext,
|
||||
) -> Result<ChatRequest, ApiError> {
|
||||
prepare_chat_request(request, lora_resolution, ctx).map(|prepared| prepared.chat_request)
|
||||
}
|
||||
|
||||
/// Validate one chat completion request and proxy it into the shared
|
||||
/// `vllm-chat` stack.
|
||||
|
||||
@@ -21,12 +21,14 @@ use thiserror_ext::AsReport as _;
|
||||
use tracing::{debug, error, info, trace};
|
||||
use tracing_futures::Instrument as _;
|
||||
use vllm_engine_core_client::protocol::output::StopReason;
|
||||
use vllm_text::tokenizer::Tokenizer;
|
||||
use vllm_text::{
|
||||
DecodedPromptLogprobs, DecodedTextEvent, FinishReason, TextOutputStream,
|
||||
TextOutputStreamExt as _,
|
||||
TextOutputStreamExt as _, TextRequest,
|
||||
};
|
||||
|
||||
use self::convert::{ResponseOptions, prepare_completion_request};
|
||||
pub(crate) use self::types::CompletionRequest;
|
||||
use super::utils::logprobs::{
|
||||
collected_logprobs_to_openai, decoded_logprobs_to_openai, decoded_prompt_logprobs_to_openai,
|
||||
prompt_logprobs_to_maps, text_len,
|
||||
@@ -34,15 +36,26 @@ use super::utils::logprobs::{
|
||||
use super::utils::types::Usage;
|
||||
use crate::config::ApiServerOptions;
|
||||
use crate::error::{ApiError, bail_server_error, server_error, text_submit_error};
|
||||
use crate::lora::LoraModelResolution;
|
||||
use crate::routes::openai::completions::types::{
|
||||
CompletionChoice, CompletionRequest, CompletionResponse, CompletionSseChunk,
|
||||
CompletionStreamChoice, CompletionStreamResponse,
|
||||
CompletionChoice, CompletionResponse, CompletionSseChunk, CompletionStreamChoice,
|
||||
CompletionStreamResponse,
|
||||
};
|
||||
use crate::routes::openai::utils::types::LogProbs;
|
||||
use crate::routes::openai::utils::usage::ContinuousUsage;
|
||||
use crate::routes::openai::utils::validated_json::ValidatedJson;
|
||||
use crate::state::AppState;
|
||||
use crate::utils::{resolve_request_context, unix_timestamp};
|
||||
use crate::utils::{ResolvedRequestContext, resolve_request_context, unix_timestamp};
|
||||
|
||||
pub(crate) fn lower_completion_request(
|
||||
request: CompletionRequest,
|
||||
lora_resolution: &LoraModelResolution,
|
||||
ctx: ResolvedRequestContext,
|
||||
tokenizer: &dyn Tokenizer,
|
||||
) -> Result<TextRequest, ApiError> {
|
||||
prepare_completion_request(request, lora_resolution, ctx, tokenizer)
|
||||
.map(|prepared| prepared.text_request)
|
||||
}
|
||||
|
||||
/// Validate one completions request and proxy it into the shared `vllm-text`
|
||||
/// stack.
|
||||
|
||||
@@ -7,5 +7,7 @@ mod models;
|
||||
pub(crate) mod utils;
|
||||
|
||||
pub use chat_completions::chat_completions;
|
||||
pub(crate) use chat_completions::{ChatCompletionRequest, lower_chat_request};
|
||||
pub use completions::completions;
|
||||
pub(crate) use completions::{CompletionRequest, lower_completion_request};
|
||||
pub use models::list_models;
|
||||
|
||||
@@ -0,0 +1,149 @@
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
// SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
use std::sync::Arc;
|
||||
|
||||
use axum::Json;
|
||||
use axum::Router;
|
||||
use axum::extract::DefaultBodyLimit;
|
||||
use axum::extract::State;
|
||||
use axum::http::{HeaderMap, StatusCode};
|
||||
use axum::routing::{get, post};
|
||||
use thiserror_ext::AsReport as _;
|
||||
use vllm_text::TextRequest;
|
||||
|
||||
use crate::DEFAULT_REQUEST_BODY_LIMIT_BYTES;
|
||||
use crate::error::{ApiError, text_submit_error};
|
||||
use crate::lora::LoraModelResolution;
|
||||
use crate::render::RenderState;
|
||||
use crate::routes::inference::generate::{
|
||||
GenerateRequest, validate_request_compat as validate_generate_request,
|
||||
};
|
||||
use crate::routes::openai::utils::types::{ListModelsResponse, ModelObject, StreamOptions};
|
||||
use crate::routes::openai::utils::validated_json::ValidatedJson;
|
||||
use crate::routes::openai::{
|
||||
ChatCompletionRequest, CompletionRequest, lower_chat_request, lower_completion_request,
|
||||
};
|
||||
use crate::utils::{resolve_request_context, unix_timestamp};
|
||||
|
||||
pub(crate) fn build_router(state: Arc<RenderState>) -> Router {
|
||||
Router::new()
|
||||
.route("/health", get(health))
|
||||
.route("/ping", get(health).post(health))
|
||||
.route("/v1/models", get(list_models))
|
||||
.route("/v1/chat/completions/render", post(render_chat))
|
||||
.route("/v1/completions/render", post(render_completion))
|
||||
.with_state(state)
|
||||
.layer(DefaultBodyLimit::max(DEFAULT_REQUEST_BODY_LIMIT_BYTES))
|
||||
}
|
||||
|
||||
async fn health() -> StatusCode {
|
||||
StatusCode::OK
|
||||
}
|
||||
|
||||
async fn list_models(State(state): State<Arc<RenderState>>) -> Json<ListModelsResponse> {
|
||||
let created = unix_timestamp() as i64;
|
||||
Json(ListModelsResponse {
|
||||
object: "list".to_string(),
|
||||
data: state
|
||||
.served_model_names
|
||||
.iter()
|
||||
.map(|model| ModelObject {
|
||||
id: model.clone(),
|
||||
object: "model".to_string(),
|
||||
created,
|
||||
owned_by: "vllm".to_string(),
|
||||
root: Some(state.model.clone()),
|
||||
parent: None,
|
||||
max_model_len: Some(state.text.max_model_len()),
|
||||
})
|
||||
.collect(),
|
||||
})
|
||||
}
|
||||
|
||||
fn model_resolution(state: &RenderState) -> LoraModelResolution {
|
||||
LoraModelResolution {
|
||||
model_names: state.served_model_names.clone(),
|
||||
lora_request: None,
|
||||
}
|
||||
}
|
||||
|
||||
fn lower_render_request(
|
||||
state: &RenderState,
|
||||
text_request: TextRequest,
|
||||
model: String,
|
||||
stream: bool,
|
||||
stream_options: Option<StreamOptions>,
|
||||
) -> Result<GenerateRequest, ApiError> {
|
||||
let prepared = state
|
||||
.text
|
||||
.prepare(text_request)
|
||||
.map_err(|error| text_submit_error("failed to prepare render request", error))?;
|
||||
let token_ids = prepared.generate_request.prompt_token_ids;
|
||||
let text_request = prepared.text_request;
|
||||
|
||||
let request = GenerateRequest {
|
||||
request_id: Some(text_request.request_id),
|
||||
model: Some(model),
|
||||
token_ids,
|
||||
sampling_params: text_request.sampling_params,
|
||||
stream,
|
||||
stream_options,
|
||||
cache_salt: text_request.cache_salt,
|
||||
priority: text_request.priority,
|
||||
kv_transfer_params: None,
|
||||
ec_transfer_params: None,
|
||||
other: Default::default(),
|
||||
};
|
||||
validate_generate_request(&request, &state.served_model_names)?;
|
||||
Ok(request)
|
||||
}
|
||||
|
||||
async fn render_chat(
|
||||
State(state): State<Arc<RenderState>>,
|
||||
headers: HeaderMap,
|
||||
ValidatedJson(body): ValidatedJson<ChatCompletionRequest>,
|
||||
) -> Result<Json<GenerateRequest>, ApiError> {
|
||||
let model = body.model.clone();
|
||||
let stream = body.stream;
|
||||
let stream_options = body.stream_options.clone();
|
||||
let request_context = resolve_request_context(&headers, body.request_id.as_deref());
|
||||
let chat_request = lower_chat_request(body, &model_resolution(&state), request_context)?;
|
||||
let (text_request, _) = state
|
||||
.chat
|
||||
.prepare(chat_request)
|
||||
.await
|
||||
.map_err(|error| ApiError::invalid_request(error.to_report_string(), None))?;
|
||||
Ok(Json(lower_render_request(
|
||||
&state,
|
||||
text_request,
|
||||
model,
|
||||
stream,
|
||||
stream_options,
|
||||
)?))
|
||||
}
|
||||
|
||||
async fn render_completion(
|
||||
State(state): State<Arc<RenderState>>,
|
||||
headers: HeaderMap,
|
||||
ValidatedJson(body): ValidatedJson<CompletionRequest>,
|
||||
) -> Result<Json<Vec<GenerateRequest>>, ApiError> {
|
||||
let model = body.model.clone();
|
||||
let stream = body.stream;
|
||||
let stream_options = body.stream_options.clone();
|
||||
let request_context = resolve_request_context(&headers, body.request_id.as_deref());
|
||||
let tokenizer = state.text.tokenizer();
|
||||
let text_request = lower_completion_request(
|
||||
body,
|
||||
&model_resolution(&state),
|
||||
request_context,
|
||||
tokenizer.as_ref(),
|
||||
)?;
|
||||
Ok(Json(vec![lower_render_request(
|
||||
&state,
|
||||
text_request,
|
||||
model,
|
||||
stream,
|
||||
stream_options,
|
||||
)?]))
|
||||
}
|
||||
@@ -23,8 +23,8 @@ use serial_test::serial;
|
||||
use tower::{Service as _, ServiceExt as _};
|
||||
use vllm_chat::{
|
||||
ChatBackend, ChatContent, ChatContentPart, ChatLlm, ChatMessage, ChatRenderer, ChatRequest,
|
||||
ChatTextBackend, DefaultChatOutputProcessor, DynChatOutputProcessor, DynChatRenderer,
|
||||
NewChatOutputProcessorOptions,
|
||||
ChatRequestProcessor, ChatTextBackend, DefaultChatOutputProcessor, DynChatOutputProcessor,
|
||||
DynChatRenderer, NewChatOutputProcessorOptions, ParserSelection,
|
||||
};
|
||||
use vllm_engine_core_client::mock_engine::default_ready_response;
|
||||
use vllm_engine_core_client::protocol::decode_value;
|
||||
@@ -46,13 +46,17 @@ use vllm_engine_core_client::{
|
||||
use vllm_llm::Llm;
|
||||
use vllm_metrics::METRICS;
|
||||
use vllm_text::tokenizer::DynTokenizer;
|
||||
use vllm_text::{Prompt, TextBackend};
|
||||
use vllm_text::{Prompt, TextBackend, TextRequestProcessor};
|
||||
use vllm_tokenizer::test_utils::TestTokenizer;
|
||||
use zeromq::prelude::{SocketRecv, SocketSend};
|
||||
use zeromq::{DealerSocket, PushSocket, ZmqMessage};
|
||||
|
||||
use super::{build_router, build_router_with_dev_mode, build_router_with_dev_mode_and_lora};
|
||||
use super::{
|
||||
build_router, build_router_with_dev_mode, build_router_with_dev_mode_and_lora,
|
||||
render::build_router as build_render_router,
|
||||
};
|
||||
use crate::config::{ApiServerOptions, CorsConfig};
|
||||
use crate::render::RenderState;
|
||||
use crate::state::AppState;
|
||||
|
||||
fn request_output(
|
||||
@@ -682,6 +686,24 @@ async fn test_app() -> axum::Router {
|
||||
test_app_with_dev_mode(false).await
|
||||
}
|
||||
|
||||
fn test_render_app() -> axum::Router {
|
||||
test_render_app_with_parser_selections(ParserSelection::Auto, ParserSelection::Auto)
|
||||
}
|
||||
|
||||
fn test_render_app_with_parser_selections(
|
||||
tool_call_parser: ParserSelection,
|
||||
reasoning_parser: ParserSelection,
|
||||
) -> axum::Router {
|
||||
let backend = Arc::new(FakeChatBackend::new());
|
||||
build_render_router(Arc::new(RenderState {
|
||||
model: "backend-model".to_string(),
|
||||
served_model_names: vec!["render-model".to_string()],
|
||||
text: TextRequestProcessor::new(backend.clone(), 128),
|
||||
chat: ChatRequestProcessor::render_only(backend)
|
||||
.with_parser_selections(tool_call_parser, reasoning_parser),
|
||||
}))
|
||||
}
|
||||
|
||||
async fn test_app_with_dev_mode(dev_mode_enabled: bool) -> axum::Router {
|
||||
let (chat, _engine_task) = test_models_with_engine_outputs_and_backend(
|
||||
b"engine-openai",
|
||||
@@ -1043,6 +1065,191 @@ fn metric_delta(
|
||||
- metric_value(rendered_before, metric, labels).unwrap_or(0.0)
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn render_chat_returns_generate_request_with_header_request_id() {
|
||||
let mut app = test_render_app();
|
||||
let response = app
|
||||
.call(
|
||||
Request::builder()
|
||||
.method("POST")
|
||||
.uri("/v1/chat/completions/render")
|
||||
.header("content-type", "application/json")
|
||||
.header("X-Request-Id", "header-req")
|
||||
.body(Body::from(
|
||||
json!({
|
||||
"request_id": "body-req",
|
||||
"model": "render-model",
|
||||
"messages": [{"role": "user", "content": "hello"}],
|
||||
"max_completion_tokens": 8,
|
||||
"stream": true,
|
||||
"stream_options": {"include_usage": true},
|
||||
"cache_salt": "render-cache-salt",
|
||||
"priority": -3
|
||||
})
|
||||
.to_string(),
|
||||
))
|
||||
.expect("build request"),
|
||||
)
|
||||
.await
|
||||
.expect("call app");
|
||||
|
||||
assert_eq!(response.status(), StatusCode::OK);
|
||||
let body = to_bytes(response.into_body(), usize::MAX).await.expect("read body");
|
||||
let json: serde_json::Value = serde_json::from_slice(&body).expect("decode json");
|
||||
|
||||
assert_eq!(json["request_id"], "chatcmpl-header-req");
|
||||
assert_eq!(json["model"], "render-model");
|
||||
assert_eq!(json["stream"], true);
|
||||
assert_eq!(json["stream_options"]["include_usage"], true);
|
||||
assert_eq!(json["cache_salt"], "render-cache-salt");
|
||||
assert_eq!(json["priority"], -3);
|
||||
assert_eq!(json["sampling_params"]["max_tokens"], 8);
|
||||
assert!(!json["token_ids"].as_array().unwrap().is_empty());
|
||||
assert!(json.get("prompt_token_ids").is_none());
|
||||
assert!(json.get("mm_features").is_none());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn render_chat_uses_processor_owned_parser_selections() {
|
||||
let mut app = test_render_app_with_parser_selections(
|
||||
ParserSelection::Auto,
|
||||
ParserSelection::Explicit("definitely_missing_reasoning_parser".to_string()),
|
||||
);
|
||||
let response = app
|
||||
.call(
|
||||
Request::builder()
|
||||
.method("POST")
|
||||
.uri("/v1/chat/completions/render")
|
||||
.header("content-type", "application/json")
|
||||
.body(Body::from(
|
||||
json!({
|
||||
"model": "render-model",
|
||||
"messages": [{"role": "user", "content": "hello"}],
|
||||
"max_completion_tokens": 8
|
||||
})
|
||||
.to_string(),
|
||||
))
|
||||
.expect("build request"),
|
||||
)
|
||||
.await
|
||||
.expect("call app");
|
||||
|
||||
assert_eq!(response.status(), StatusCode::BAD_REQUEST);
|
||||
let body = to_bytes(response.into_body(), usize::MAX).await.expect("read body");
|
||||
let body = String::from_utf8(body.to_vec()).expect("decode body");
|
||||
assert!(
|
||||
body.contains("reasoning parser `definitely_missing_reasoning_parser` is not registered")
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn render_completion_returns_generate_request_with_body_request_id() {
|
||||
let mut app = test_render_app();
|
||||
let response = app
|
||||
.call(
|
||||
Request::builder()
|
||||
.method("POST")
|
||||
.uri("/v1/completions/render")
|
||||
.header("content-type", "application/json")
|
||||
.body(Body::from(
|
||||
json!({
|
||||
"request_id": "body-req",
|
||||
"model": "render-model",
|
||||
"prompt": "hello",
|
||||
"max_tokens": 8
|
||||
})
|
||||
.to_string(),
|
||||
))
|
||||
.expect("build request"),
|
||||
)
|
||||
.await
|
||||
.expect("call app");
|
||||
|
||||
assert_eq!(response.status(), StatusCode::OK);
|
||||
let body = to_bytes(response.into_body(), usize::MAX).await.expect("read body");
|
||||
let json: serde_json::Value = serde_json::from_slice(&body).expect("decode json");
|
||||
|
||||
assert_eq!(json[0]["request_id"], "cmpl-body-req");
|
||||
assert_eq!(json[0]["model"], "render-model");
|
||||
assert_eq!(json[0]["stream"], false);
|
||||
assert_eq!(json[0]["sampling_params"]["max_tokens"], 8);
|
||||
assert!(!json[0]["token_ids"].as_array().unwrap().is_empty());
|
||||
assert!(json[0].get("stream_options").is_none());
|
||||
assert!(json[0].get("prompt_token_ids").is_none());
|
||||
assert!(json[0].get("mm_features").is_none());
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
#[serial]
|
||||
async fn render_chat_response_can_be_submitted_to_generate_unchanged() {
|
||||
let mut render_app = test_render_app();
|
||||
let render_response = render_app
|
||||
.call(
|
||||
Request::builder()
|
||||
.method("POST")
|
||||
.uri("/v1/chat/completions/render")
|
||||
.header("content-type", "application/json")
|
||||
.body(Body::from(
|
||||
json!({
|
||||
"request_id": "round-trip",
|
||||
"model": "render-model",
|
||||
"messages": [{"role": "user", "content": "hello"}],
|
||||
"max_completion_tokens": 8,
|
||||
"bad_words": ["blocked"],
|
||||
"vllm_xargs": {"custom": 1}
|
||||
})
|
||||
.to_string(),
|
||||
))
|
||||
.expect("build render request"),
|
||||
)
|
||||
.await
|
||||
.expect("call render app");
|
||||
|
||||
assert_eq!(render_response.status(), StatusCode::OK);
|
||||
let render_body = to_bytes(render_response.into_body(), usize::MAX)
|
||||
.await
|
||||
.expect("read render response");
|
||||
let render_json: serde_json::Value =
|
||||
serde_json::from_slice(&render_body).expect("decode render response");
|
||||
let token_ids: Vec<u32> = serde_json::from_value(render_json["token_ids"].clone())
|
||||
.expect("decode rendered token IDs");
|
||||
|
||||
assert_eq!(
|
||||
render_json["sampling_params"]["bad_words"],
|
||||
json!(["blocked"])
|
||||
);
|
||||
assert_eq!(render_json["sampling_params"]["vllm_xargs"]["custom"], 1);
|
||||
|
||||
let (chat, engine_task) = test_models_with_engine_outputs_and_backend_inner(
|
||||
b"engine-render-generate-round-trip",
|
||||
default_stream_output_specs(),
|
||||
Some(token_ids),
|
||||
Arc::new(FakeChatBackend::new()),
|
||||
)
|
||||
.await;
|
||||
let mut generate_app = build_router(Arc::new(AppState::new(
|
||||
vec!["render-model".to_string()],
|
||||
chat,
|
||||
)));
|
||||
let generate_response = generate_app
|
||||
.call(
|
||||
Request::builder()
|
||||
.method("POST")
|
||||
.uri("/inference/v1/generate")
|
||||
.header("content-type", "application/json")
|
||||
.body(Body::from(render_body))
|
||||
.expect("build generate request"),
|
||||
)
|
||||
.await
|
||||
.expect("call generate app");
|
||||
|
||||
assert_eq!(generate_response.status(), StatusCode::OK);
|
||||
let _ = to_bytes(generate_response.into_body(), usize::MAX)
|
||||
.await
|
||||
.expect("read generate response");
|
||||
engine_task.finish().await;
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
#[serial]
|
||||
async fn list_models_returns_configured_model() {
|
||||
|
||||
Reference in New Issue
Block a user