diff --git a/rust/README.md b/rust/README.md index b14aba3fae..085f70d816 100644 --- a/rust/README.md +++ b/rust/README.md @@ -15,6 +15,7 @@ The component is organized as a Cargo workspace with several crates, layered bot │ vllm-cmd / vllm-rs │ CLI entrypoint: │ │ Python vLLM frontend subprocess │ │ Rust managed-engine serve mode +│ │ Engine-free render mode ├─────────────────────────────────┤ │ vllm-server │ OpenAI-compatible HTTP API (axum) ├─────────────────────────────────┤ @@ -74,9 +75,46 @@ To build the `vllm-rs` in isolation: ./build_rust.sh ``` +### Engine-free renderer + +`vllm-rs render` serves text-only request preprocessing without starting or +connecting to a Python inference engine: + +```bash +cargo run --manifest-path rust/Cargo.toml -p vllm-cmd --release -- \ + render Qwen/Qwen3-32B \ + --host 127.0.0.1 --max-model-len 32768 +``` + +It exposes `/v1/chat/completions/render` and `/v1/completions/render`. Only +tokenizer and model configuration files are loaded; model weights, PyTorch, +and vLLM kernels are not required. + +The render endpoints return the public token-in `GenerateRequest` consumed by +the Rust `/inference/v1/generate` endpoint. A chat render response, or one item +from a completion render response, can be submitted to that endpoint without +changing its fields. + +The render and inference paths use the same `vllm-chat` and `vllm-text` +request-preparation logic; render mode stops before engine submission. +Tool-call and reasoning parsers use model-based auto-detection by default. Use +`--tool-call-parser` and `--reasoning-parser` to override either selection; +unified parsers require the same selection for both options. + +```bash +curl http://127.0.0.1:8000/v1/chat/completions/render \ + -H "Content-Type: application/json" \ + -d '{ + "model": "Qwen/Qwen3-32B", + "messages": [{"role": "user", "content": "Hello"}], + "max_completion_tokens": 16 + }' +``` + ### Example Request -After either startup path, you can use any OpenAI-compatible client: +After either full-frontend startup path, you can use any OpenAI-compatible +client against the inference endpoints: ```bash curl http://127.0.0.1:8000/v1/chat/completions \ diff --git a/rust/src/chat/src/lib.rs b/rust/src/chat/src/lib.rs index e07fb1b657..ca5c7a125a 100644 --- a/rust/src/chat/src/lib.rs +++ b/rust/src/chat/src/lib.rs @@ -97,6 +97,10 @@ pub struct ChatRequestProcessor { /// Effective model dtype reported by the engine. /// Absent for text-only frontends without an engine handshake. model_dtype: Option, + /// Tool-call parser selection used when preparing generation requests. + tool_call_parser: ParserSelection, + /// Reasoning parser selection used when preparing generation requests. + reasoning_parser: ParserSelection, } impl ChatRequestProcessor { @@ -106,6 +110,8 @@ impl ChatRequestProcessor { Self { backend, model_dtype: Some(model_dtype), + tool_call_parser: ParserSelection::Auto, + reasoning_parser: ParserSelection::Auto, } } @@ -114,9 +120,22 @@ impl ChatRequestProcessor { Self { backend, model_dtype: None, + tool_call_parser: ParserSelection::Auto, + reasoning_parser: ParserSelection::Auto, } } + /// Configure the parser selections used to prepare generation requests. + pub fn with_parser_selections( + mut self, + tool_call_parser: ParserSelection, + reasoning_parser: ParserSelection, + ) -> Self { + self.tool_call_parser = tool_call_parser; + self.reasoning_parser = reasoning_parser; + self + } + async fn finalize_rendered_prompt( &self, request: &ChatRequest, @@ -196,10 +215,15 @@ impl ChatRequestProcessor { pub async fn prepare( &self, mut request: ChatRequest, - options: NewChatOutputProcessorOptions<'_>, ) -> Result<(TextRequest, DynChatOutputProcessor)> { request.validate()?; - let output_processor = self.backend.new_chat_output_processor(&mut request, options)?; + let output_processor = self.backend.new_chat_output_processor( + &mut request, + NewChatOutputProcessorOptions { + tool_call_parser: &self.tool_call_parser, + reasoning_parser: &self.reasoning_parser, + }, + )?; let text_request = self.prepare_text_request(request).await?; Ok((text_request, output_processor)) } @@ -213,10 +237,6 @@ impl ChatRequestProcessor { pub struct ChatLlm { text: TextLlm, processor: ChatRequestProcessor, - /// Tool-call parser selection. - tool_call_parser: ParserSelection, - /// Reasoning parser selection. - reasoning_parser: ParserSelection, } impl ChatLlm { @@ -228,8 +248,6 @@ impl ChatLlm { Self { text, processor: ChatRequestProcessor::new(backend, model_dtype), - tool_call_parser: ParserSelection::Auto, - reasoning_parser: ParserSelection::Auto, } } @@ -242,13 +260,13 @@ impl ChatLlm { /// Set tool-call parser selection. pub fn with_tool_call_parser(mut self, selection: ParserSelection) -> Self { - self.tool_call_parser = selection; + self.processor.tool_call_parser = selection; self } /// Set reasoning parser selection. pub fn with_reasoning_parser(mut self, selection: ParserSelection) -> Self { - self.reasoning_parser = selection; + self.processor.reasoning_parser = selection; self } @@ -306,7 +324,7 @@ impl ChatLlm { /// Effective tool-call parser name for this model, if parsing is enabled. pub fn tool_call_parser_name(&self) -> Option<&str> { - match &self.tool_call_parser { + match &self.processor.tool_call_parser { ParserSelection::Auto => { ToolParserFactory::global().resolve_name_for_model(self.model_id()) } @@ -317,7 +335,7 @@ impl ChatLlm { /// Effective reasoning parser name for this model, if parsing is enabled. pub fn reasoning_parser_name(&self) -> Option<&str> { - match &self.reasoning_parser { + match &self.processor.reasoning_parser { ParserSelection::Auto => { ReasoningParserFactory::global().resolve_name_for_model(self.model_id()) } @@ -328,16 +346,7 @@ impl ChatLlm { /// Render, tokenize, and submit one chat request. pub async fn chat(&self, request: ChatRequest) -> Result { - let (text_request, output_processor) = self - .processor - .prepare( - request, - NewChatOutputProcessorOptions { - tool_call_parser: &self.tool_call_parser, - reasoning_parser: &self.reasoning_parser, - }, - ) - .await?; + let (text_request, output_processor) = self.processor.prepare(request).await?; let request_id = text_request.request_id.clone(); let decoded_stream = self.text.generate(text_request).await?.map_err(Error::from).boxed(); diff --git a/rust/src/cmd/src/cli.rs b/rust/src/cmd/src/cli.rs index d5377515cb..b62e75ebac 100644 --- a/rust/src/cmd/src/cli.rs +++ b/rust/src/cmd/src/cli.rs @@ -29,7 +29,8 @@ use vllm_managed_engine::ManagedEngineConfig; use vllm_managed_engine::cli::{ManagedEngineArgs, repartition_managed_engine_args}; use vllm_server::{ ApiServerOptions, ChatTemplateContentFormatOption, Config, CoordinatorMode, CorsConfig, - DEFAULT_KEEP_ALIVE_TIMEOUT, HttpListenerMode, ParserSelection, RendererSelection, TlsConfig, + DEFAULT_KEEP_ALIVE_TIMEOUT, HttpListenerMode, ParserSelection, RenderConfig, RendererSelection, + TlsConfig, }; use crate::cli::unsupported::UnsupportedArgs; @@ -90,6 +91,8 @@ pub enum Command { /// Run vLLM benchmarks. #[command(subcommand)] Bench(BenchCommand), + /// Run engine-free request rendering and preprocessing. + Render(RenderArgs), } /// Supported benchmark commands. @@ -99,6 +102,67 @@ pub enum BenchCommand { Serve(vllm_bench::BenchServeArgs), } +/// Arguments for the engine-free text renderer. +#[derive(Debug, Args, PartialEq, Eq)] +pub struct RenderArgs { + /// Model identifier or local model directory containing tokenizer files. + model: String, + /// HTTP bind host. + #[arg(long, default_value = "127.0.0.1")] + host: String, + /// HTTP bind port. + #[arg(long, default_value_t = 8000)] + port: u16, + /// Public model names accepted by the API. + #[arg(long, num_args = 0..)] + served_model_name: Vec, + /// Select the tool-call parser. Use `auto` to infer from the model or + /// `none` to disable parsing. + #[arg(long, default_value_t)] + tool_call_parser: ParserSelection, + /// Select the reasoning parser. Use `auto` to infer from the model or + /// `none` to disable parsing. + #[arg(long, default_value_t)] + reasoning_parser: ParserSelection, + /// Select the native chat renderer implementation. + #[arg(long = "tokenizer-mode", default_value_t)] + renderer: RendererSelection, + /// Override the model chat template with a file path or inline template. + #[arg(long)] + chat_template: Option, + /// Default JSON keyword arguments merged into every chat-template render. + #[arg(long, value_parser = parse_json::>, value_name = "JSON")] + default_chat_template_kwargs: Option>, + /// How message content is exposed to the chat template. + #[arg(long, default_value_t)] + chat_template_content_format: ChatTemplateContentFormatOption, + /// Maximum model context length used for request validation. + #[arg(long)] + max_model_len: u32, + /// Maximum accepted logprobs count; -1 disables the cap. + #[arg(long, value_parser = clap::value_parser!(i32).range(-1..), allow_negative_numbers = true)] + max_logprobs: Option, +} + +impl RenderArgs { + pub(super) fn into_config(self) -> RenderConfig { + RenderConfig { + model: self.model, + served_model_name: self.served_model_name, + host: self.host, + port: self.port, + tool_call_parser: self.tool_call_parser, + reasoning_parser: self.reasoning_parser, + renderer: self.renderer, + chat_template: self.chat_template, + default_chat_template_kwargs: self.default_chat_template_kwargs.unwrap_or_default(), + chat_template_content_format: self.chat_template_content_format, + max_model_len: self.max_model_len, + max_logprobs: self.max_logprobs, + } + } +} + /// A JSON-encoded list of strings, matching Python's `json.loads` CLI type for /// the CORS list arguments (e.g. `--allowed-origins '["*"]'`). Parsing the whole /// value as one item keeps clap from treating the field as a repeated flag. diff --git a/rust/src/cmd/src/cli/tests.rs b/rust/src/cmd/src/cli/tests.rs index e586477a89..28cea8a371 100644 --- a/rust/src/cmd/src/cli/tests.rs +++ b/rust/src/cmd/src/cli/tests.rs @@ -27,6 +27,41 @@ fn bench_serve_args_parse_without_managed_engine_repartition() { assert!(args.request_rate.is_infinite()); } +#[test] +fn render_args_build_supported_config() { + let cli = Cli::try_parse_from([ + "vllm-rs", + "render", + "Qwen/Qwen2.5-0.5B-Instruct", + "--port", + "8080", + "--max-model-len", + "32768", + "--served-model-name", + "qwen", + "--tokenizer-mode", + "deepseek_v32", + "--max-logprobs", + "-1", + ]) + .unwrap(); + + let Command::Render(args) = cli.command else { + panic!("expected render args"); + }; + let config = args.into_config(); + + assert_eq!(config.model, "Qwen/Qwen2.5-0.5B-Instruct"); + assert_eq!(config.host, "127.0.0.1"); + assert_eq!(config.port, 8080); + assert_eq!(config.max_model_len, 32768); + assert_eq!(config.served_model_name, ["qwen"]); + assert_eq!(config.tool_call_parser, ParserSelection::Auto); + assert_eq!(config.reasoning_parser, ParserSelection::Auto); + assert_eq!(config.renderer, RendererSelection::DeepSeekV32); + assert_eq!(config.max_logprobs, Some(-1)); +} + #[test] fn serve_args_forward_python_flags_with_separator() { let cli = Cli::try_parse_from([ diff --git a/rust/src/cmd/src/main.rs b/rust/src/cmd/src/main.rs index 4e92ece709..b9535561e4 100644 --- a/rust/src/cmd/src/main.rs +++ b/rust/src/cmd/src/main.rs @@ -211,5 +211,8 @@ async fn async_main(cli: Cli) -> Result<()> { )), } } + Command::Render(args) => { + vllm_server::serve_render(args.into_config(), shutdown_signal()).await + } } } diff --git a/rust/src/server/src/lib.rs b/rust/src/server/src/lib.rs index bff47d5b1a..c36aaac2a1 100644 --- a/rust/src/server/src/lib.rs +++ b/rust/src/server/src/lib.rs @@ -9,6 +9,7 @@ mod grpc; mod listener; mod lora; mod middleware; +mod render; mod routes; mod runtime; mod server_info; @@ -35,6 +36,7 @@ use hyper::server::conn::http1; use hyper_util::rt::{TokioIo, TokioTimer}; use hyper_util::server::graceful::GracefulShutdown; use hyper_util::service::TowerToHyperService; +pub use render::{RenderConfig, serve_render}; use tokio::net::TcpListener; use tokio::time::{Instant, sleep_until}; use tokio_util::sync::CancellationToken; diff --git a/rust/src/server/src/render.rs b/rust/src/server/src/render.rs new file mode 100644 index 0000000000..ee8098f8e9 --- /dev/null +++ b/rust/src/server/src/render.rs @@ -0,0 +1,146 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +use std::collections::HashMap; +use std::sync::Arc; + +use anyhow::{Context as _, Result, bail}; +use serde_json::Value; +use tokio_util::sync::CancellationToken; +use tracing::info; +use vllm_chat::{ + ChatRequestProcessor, ChatTemplateContentFormatOption, LoadModelBackendsOptions, + ParserSelection, RendererSelection, load_model_backends, +}; +use vllm_text::TextRequestProcessor; + +use crate::{HttpListenerMode, listener::Listener}; + +/// Configuration for the engine-free text rendering server. +#[derive(Debug)] +pub struct RenderConfig { + pub model: String, + pub served_model_name: Vec, + pub host: String, + pub port: u16, + pub tool_call_parser: ParserSelection, + pub reasoning_parser: ParserSelection, + pub renderer: RendererSelection, + pub chat_template: Option, + pub default_chat_template_kwargs: HashMap, + pub chat_template_content_format: ChatTemplateContentFormatOption, + pub max_model_len: u32, + pub max_logprobs: Option, +} + +impl RenderConfig { + /// Validate configuration before initializing renderer/tokenizer backends + /// or binding a listener. + pub fn validate(&self) -> Result<()> { + vllm_chat::validate_parser_overrides(&self.tool_call_parser, &self.reasoning_parser)?; + if self.max_logprobs.is_some_and(|value| value < -1) { + bail!("max_logprobs must be non-negative or -1"); + } + Ok(()) + } +} + +pub(crate) struct RenderState { + pub(crate) model: String, + pub(crate) served_model_names: Vec, + pub(crate) text: TextRequestProcessor, + pub(crate) chat: ChatRequestProcessor, +} + +async fn build_state(config: &RenderConfig) -> Result> { + let loaded = load_model_backends( + &config.model, + LoadModelBackendsOptions { + renderer: config.renderer, + language_model_only: true, + chat_template: config.chat_template.clone(), + chat_template_content_format: config.chat_template_content_format, + default_chat_template_kwargs: config.default_chat_template_kwargs.clone(), + limit_mm_per_prompt: Default::default(), + }, + ) + .await + .context("failed to load renderer/tokenizer backends")?; + let served_model_names = + crate::effective_served_model_names(&config.model, &config.served_model_name); + let text = TextRequestProcessor::new(loaded.text_backend, config.max_model_len) + .with_max_logprobs(config.max_logprobs); + let chat = ChatRequestProcessor::render_only(loaded.chat_backend).with_parser_selections( + config.tool_call_parser.clone(), + config.reasoning_parser.clone(), + ); + Ok(Arc::new(RenderState { + model: config.model.clone(), + served_model_names, + text, + chat, + })) +} + +/// Run the text-only preprocessing server without starting or connecting +/// to an inference engine. +pub async fn serve_render(config: RenderConfig, shutdown: CancellationToken) -> Result<()> { + config.validate().context("invalid render server configuration")?; + let state = tokio::select! { + result = build_state(&config) => result?, + _ = shutdown.cancelled() => return Ok(()), + }; + let listener_mode = HttpListenerMode::BindTcp { + host: config.host.clone(), + port: config.port, + }; + let listener = Listener::bind(&listener_mode) + .await + .with_context(|| format!("failed to bind {}:{}", config.host, config.port))?; + let address = listener.local_addr_display()?; + info!( + %address, + model = %config.model, + "starting engine-free Rust render server" + ); + axum::serve(listener, crate::routes::render::build_router(state)) + .with_graceful_shutdown(shutdown.cancelled_owned()) + .await + .context("render server failed") +} + +#[cfg(test)] +mod tests { + use super::*; + + #[tokio::test] + async fn serve_render_rejects_unknown_parser_before_startup() { + let shutdown = CancellationToken::new(); + shutdown.cancel(); + let error = serve_render( + RenderConfig { + model: "test-model".to_string(), + served_model_name: Vec::new(), + host: "127.0.0.1".to_string(), + port: 8000, + tool_call_parser: ParserSelection::Auto, + reasoning_parser: ParserSelection::Explicit("typo".to_string()), + renderer: RendererSelection::Auto, + chat_template: None, + default_chat_template_kwargs: HashMap::new(), + chat_template_content_format: ChatTemplateContentFormatOption::Auto, + max_model_len: 128, + max_logprobs: None, + }, + shutdown, + ) + .await + .unwrap_err(); + let report = format!("{error:#}"); + + assert!( + report.contains("reasoning parser `typo` is not registered"), + "{report}" + ); + } +} diff --git a/rust/src/server/src/routes.rs b/rust/src/server/src/routes.rs index 2dec252875..0cf23625e0 100644 --- a/rust/src/server/src/routes.rs +++ b/rust/src/server/src/routes.rs @@ -12,6 +12,7 @@ mod metrics; pub(crate) mod openai; mod pause; mod profile; +pub(super) mod render; mod server_info; mod sleep; mod tokenize; diff --git a/rust/src/server/src/routes/inference/generate.rs b/rust/src/server/src/routes/inference/generate.rs index e176938db4..57d2724cb6 100644 --- a/rust/src/server/src/routes/inference/generate.rs +++ b/rust/src/server/src/routes/inference/generate.rs @@ -26,10 +26,12 @@ use vllm_llm::{ }; use self::convert::{ResponseOptions, prepare_generate_request}; +pub(crate) use self::types::GenerateRequest; use self::types::{ - GenerateLogprob, GenerateRequest, GenerateResponse, GenerateResponseChoice, - GenerateResponseStreamChoice, GenerateStreamResponse, + GenerateLogprob, GenerateResponse, GenerateResponseChoice, GenerateResponseStreamChoice, + GenerateStreamResponse, }; +pub(crate) use self::validate::validate_request_compat; use crate::config::ApiServerOptions; use crate::error::{ApiError, bail_server_error, server_error, text_submit_error}; use crate::routes::openai::utils::logprobs::clamp_logprob; diff --git a/rust/src/server/src/routes/inference/generate/validate.rs b/rust/src/server/src/routes/inference/generate/validate.rs index 772bcaf378..99e2b23158 100644 --- a/rust/src/server/src/routes/inference/generate/validate.rs +++ b/rust/src/server/src/routes/inference/generate/validate.rs @@ -6,7 +6,7 @@ use crate::error::{ApiError, bail_invalid_request}; /// Enforce the minimal compatibility contract for the Rust token generate /// route. -pub(super) fn validate_request_compat( +pub(crate) fn validate_request_compat( request: &GenerateRequest, served_model_names: &[String], ) -> Result<(), ApiError> { diff --git a/rust/src/server/src/routes/openai/chat_completions.rs b/rust/src/server/src/routes/openai/chat_completions.rs index 8a1e9c4383..22f0d1bb4d 100644 --- a/rust/src/server/src/routes/openai/chat_completions.rs +++ b/rust/src/server/src/routes/openai/chat_completions.rs @@ -22,17 +22,18 @@ use tracing::{debug, error, info, trace}; use tracing_futures::Instrument as _; use vllm_chat::{ AssistantBlockKind, AssistantMessageExt as _, ChatEvent, ChatEventStream, ChatEventStreamTrait, - CollectedAssistantMessage, FinishReason, + ChatRequest, CollectedAssistantMessage, FinishReason, }; use vllm_engine_core_client::protocol::output::StopReason; use self::convert::{ResponseOptions, prepare_chat_request}; +pub(crate) use self::types::ChatCompletionRequest; use crate::config::ApiServerOptions; use crate::error::{ApiError, bail_server_error, chat_submit_error, server_error}; +use crate::lora::LoraModelResolution; use crate::routes::openai::chat_completions::types::{ - AssistantRole, ChatCompletionChoice, ChatCompletionMessage, ChatCompletionRequest, - ChatCompletionResponse, ChatCompletionStreamChoice, ChatCompletionStreamResponse, - ChatMessageDelta, + AssistantRole, ChatCompletionChoice, ChatCompletionMessage, ChatCompletionResponse, + ChatCompletionStreamChoice, ChatCompletionStreamResponse, ChatMessageDelta, }; use crate::routes::openai::utils::logprobs::{ decoded_logprobs_to_openai_chat, prompt_logprobs_to_maps, @@ -43,7 +44,15 @@ use crate::routes::openai::utils::types::{ use crate::routes::openai::utils::usage::ContinuousUsage; use crate::routes::openai::utils::validated_json::ValidatedJson; use crate::state::AppState; -use crate::utils::{resolve_request_context, unix_timestamp}; +use crate::utils::{ResolvedRequestContext, resolve_request_context, unix_timestamp}; + +pub(crate) fn lower_chat_request( + request: ChatCompletionRequest, + lora_resolution: &LoraModelResolution, + ctx: ResolvedRequestContext, +) -> Result { + prepare_chat_request(request, lora_resolution, ctx).map(|prepared| prepared.chat_request) +} /// Validate one chat completion request and proxy it into the shared /// `vllm-chat` stack. diff --git a/rust/src/server/src/routes/openai/completions.rs b/rust/src/server/src/routes/openai/completions.rs index b6f2138f12..44c4bf9ae0 100644 --- a/rust/src/server/src/routes/openai/completions.rs +++ b/rust/src/server/src/routes/openai/completions.rs @@ -21,12 +21,14 @@ use thiserror_ext::AsReport as _; use tracing::{debug, error, info, trace}; use tracing_futures::Instrument as _; use vllm_engine_core_client::protocol::output::StopReason; +use vllm_text::tokenizer::Tokenizer; use vllm_text::{ DecodedPromptLogprobs, DecodedTextEvent, FinishReason, TextOutputStream, - TextOutputStreamExt as _, + TextOutputStreamExt as _, TextRequest, }; use self::convert::{ResponseOptions, prepare_completion_request}; +pub(crate) use self::types::CompletionRequest; use super::utils::logprobs::{ collected_logprobs_to_openai, decoded_logprobs_to_openai, decoded_prompt_logprobs_to_openai, prompt_logprobs_to_maps, text_len, @@ -34,15 +36,26 @@ use super::utils::logprobs::{ use super::utils::types::Usage; use crate::config::ApiServerOptions; use crate::error::{ApiError, bail_server_error, server_error, text_submit_error}; +use crate::lora::LoraModelResolution; use crate::routes::openai::completions::types::{ - CompletionChoice, CompletionRequest, CompletionResponse, CompletionSseChunk, - CompletionStreamChoice, CompletionStreamResponse, + CompletionChoice, CompletionResponse, CompletionSseChunk, CompletionStreamChoice, + CompletionStreamResponse, }; use crate::routes::openai::utils::types::LogProbs; use crate::routes::openai::utils::usage::ContinuousUsage; use crate::routes::openai::utils::validated_json::ValidatedJson; use crate::state::AppState; -use crate::utils::{resolve_request_context, unix_timestamp}; +use crate::utils::{ResolvedRequestContext, resolve_request_context, unix_timestamp}; + +pub(crate) fn lower_completion_request( + request: CompletionRequest, + lora_resolution: &LoraModelResolution, + ctx: ResolvedRequestContext, + tokenizer: &dyn Tokenizer, +) -> Result { + prepare_completion_request(request, lora_resolution, ctx, tokenizer) + .map(|prepared| prepared.text_request) +} /// Validate one completions request and proxy it into the shared `vllm-text` /// stack. diff --git a/rust/src/server/src/routes/openai/mod.rs b/rust/src/server/src/routes/openai/mod.rs index 086a2dedd3..2ef98826be 100644 --- a/rust/src/server/src/routes/openai/mod.rs +++ b/rust/src/server/src/routes/openai/mod.rs @@ -7,5 +7,7 @@ mod models; pub(crate) mod utils; pub use chat_completions::chat_completions; +pub(crate) use chat_completions::{ChatCompletionRequest, lower_chat_request}; pub use completions::completions; +pub(crate) use completions::{CompletionRequest, lower_completion_request}; pub use models::list_models; diff --git a/rust/src/server/src/routes/render.rs b/rust/src/server/src/routes/render.rs new file mode 100644 index 0000000000..e5a66cec95 --- /dev/null +++ b/rust/src/server/src/routes/render.rs @@ -0,0 +1,149 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +use std::sync::Arc; + +use axum::Json; +use axum::Router; +use axum::extract::DefaultBodyLimit; +use axum::extract::State; +use axum::http::{HeaderMap, StatusCode}; +use axum::routing::{get, post}; +use thiserror_ext::AsReport as _; +use vllm_text::TextRequest; + +use crate::DEFAULT_REQUEST_BODY_LIMIT_BYTES; +use crate::error::{ApiError, text_submit_error}; +use crate::lora::LoraModelResolution; +use crate::render::RenderState; +use crate::routes::inference::generate::{ + GenerateRequest, validate_request_compat as validate_generate_request, +}; +use crate::routes::openai::utils::types::{ListModelsResponse, ModelObject, StreamOptions}; +use crate::routes::openai::utils::validated_json::ValidatedJson; +use crate::routes::openai::{ + ChatCompletionRequest, CompletionRequest, lower_chat_request, lower_completion_request, +}; +use crate::utils::{resolve_request_context, unix_timestamp}; + +pub(crate) fn build_router(state: Arc) -> Router { + Router::new() + .route("/health", get(health)) + .route("/ping", get(health).post(health)) + .route("/v1/models", get(list_models)) + .route("/v1/chat/completions/render", post(render_chat)) + .route("/v1/completions/render", post(render_completion)) + .with_state(state) + .layer(DefaultBodyLimit::max(DEFAULT_REQUEST_BODY_LIMIT_BYTES)) +} + +async fn health() -> StatusCode { + StatusCode::OK +} + +async fn list_models(State(state): State>) -> Json { + let created = unix_timestamp() as i64; + Json(ListModelsResponse { + object: "list".to_string(), + data: state + .served_model_names + .iter() + .map(|model| ModelObject { + id: model.clone(), + object: "model".to_string(), + created, + owned_by: "vllm".to_string(), + root: Some(state.model.clone()), + parent: None, + max_model_len: Some(state.text.max_model_len()), + }) + .collect(), + }) +} + +fn model_resolution(state: &RenderState) -> LoraModelResolution { + LoraModelResolution { + model_names: state.served_model_names.clone(), + lora_request: None, + } +} + +fn lower_render_request( + state: &RenderState, + text_request: TextRequest, + model: String, + stream: bool, + stream_options: Option, +) -> Result { + let prepared = state + .text + .prepare(text_request) + .map_err(|error| text_submit_error("failed to prepare render request", error))?; + let token_ids = prepared.generate_request.prompt_token_ids; + let text_request = prepared.text_request; + + let request = GenerateRequest { + request_id: Some(text_request.request_id), + model: Some(model), + token_ids, + sampling_params: text_request.sampling_params, + stream, + stream_options, + cache_salt: text_request.cache_salt, + priority: text_request.priority, + kv_transfer_params: None, + ec_transfer_params: None, + other: Default::default(), + }; + validate_generate_request(&request, &state.served_model_names)?; + Ok(request) +} + +async fn render_chat( + State(state): State>, + headers: HeaderMap, + ValidatedJson(body): ValidatedJson, +) -> Result, ApiError> { + let model = body.model.clone(); + let stream = body.stream; + let stream_options = body.stream_options.clone(); + let request_context = resolve_request_context(&headers, body.request_id.as_deref()); + let chat_request = lower_chat_request(body, &model_resolution(&state), request_context)?; + let (text_request, _) = state + .chat + .prepare(chat_request) + .await + .map_err(|error| ApiError::invalid_request(error.to_report_string(), None))?; + Ok(Json(lower_render_request( + &state, + text_request, + model, + stream, + stream_options, + )?)) +} + +async fn render_completion( + State(state): State>, + headers: HeaderMap, + ValidatedJson(body): ValidatedJson, +) -> Result>, ApiError> { + let model = body.model.clone(); + let stream = body.stream; + let stream_options = body.stream_options.clone(); + let request_context = resolve_request_context(&headers, body.request_id.as_deref()); + let tokenizer = state.text.tokenizer(); + let text_request = lower_completion_request( + body, + &model_resolution(&state), + request_context, + tokenizer.as_ref(), + )?; + Ok(Json(vec![lower_render_request( + &state, + text_request, + model, + stream, + stream_options, + )?])) +} diff --git a/rust/src/server/src/routes/tests.rs b/rust/src/server/src/routes/tests.rs index 7480656b02..646d904ea9 100644 --- a/rust/src/server/src/routes/tests.rs +++ b/rust/src/server/src/routes/tests.rs @@ -23,8 +23,8 @@ use serial_test::serial; use tower::{Service as _, ServiceExt as _}; use vllm_chat::{ ChatBackend, ChatContent, ChatContentPart, ChatLlm, ChatMessage, ChatRenderer, ChatRequest, - ChatTextBackend, DefaultChatOutputProcessor, DynChatOutputProcessor, DynChatRenderer, - NewChatOutputProcessorOptions, + ChatRequestProcessor, ChatTextBackend, DefaultChatOutputProcessor, DynChatOutputProcessor, + DynChatRenderer, NewChatOutputProcessorOptions, ParserSelection, }; use vllm_engine_core_client::mock_engine::default_ready_response; use vllm_engine_core_client::protocol::decode_value; @@ -46,13 +46,17 @@ use vllm_engine_core_client::{ use vllm_llm::Llm; use vllm_metrics::METRICS; use vllm_text::tokenizer::DynTokenizer; -use vllm_text::{Prompt, TextBackend}; +use vllm_text::{Prompt, TextBackend, TextRequestProcessor}; use vllm_tokenizer::test_utils::TestTokenizer; use zeromq::prelude::{SocketRecv, SocketSend}; use zeromq::{DealerSocket, PushSocket, ZmqMessage}; -use super::{build_router, build_router_with_dev_mode, build_router_with_dev_mode_and_lora}; +use super::{ + build_router, build_router_with_dev_mode, build_router_with_dev_mode_and_lora, + render::build_router as build_render_router, +}; use crate::config::{ApiServerOptions, CorsConfig}; +use crate::render::RenderState; use crate::state::AppState; fn request_output( @@ -682,6 +686,24 @@ async fn test_app() -> axum::Router { test_app_with_dev_mode(false).await } +fn test_render_app() -> axum::Router { + test_render_app_with_parser_selections(ParserSelection::Auto, ParserSelection::Auto) +} + +fn test_render_app_with_parser_selections( + tool_call_parser: ParserSelection, + reasoning_parser: ParserSelection, +) -> axum::Router { + let backend = Arc::new(FakeChatBackend::new()); + build_render_router(Arc::new(RenderState { + model: "backend-model".to_string(), + served_model_names: vec!["render-model".to_string()], + text: TextRequestProcessor::new(backend.clone(), 128), + chat: ChatRequestProcessor::render_only(backend) + .with_parser_selections(tool_call_parser, reasoning_parser), + })) +} + async fn test_app_with_dev_mode(dev_mode_enabled: bool) -> axum::Router { let (chat, _engine_task) = test_models_with_engine_outputs_and_backend( b"engine-openai", @@ -1043,6 +1065,191 @@ fn metric_delta( - metric_value(rendered_before, metric, labels).unwrap_or(0.0) } +#[tokio::test] +async fn render_chat_returns_generate_request_with_header_request_id() { + let mut app = test_render_app(); + let response = app + .call( + Request::builder() + .method("POST") + .uri("/v1/chat/completions/render") + .header("content-type", "application/json") + .header("X-Request-Id", "header-req") + .body(Body::from( + json!({ + "request_id": "body-req", + "model": "render-model", + "messages": [{"role": "user", "content": "hello"}], + "max_completion_tokens": 8, + "stream": true, + "stream_options": {"include_usage": true}, + "cache_salt": "render-cache-salt", + "priority": -3 + }) + .to_string(), + )) + .expect("build request"), + ) + .await + .expect("call app"); + + assert_eq!(response.status(), StatusCode::OK); + let body = to_bytes(response.into_body(), usize::MAX).await.expect("read body"); + let json: serde_json::Value = serde_json::from_slice(&body).expect("decode json"); + + assert_eq!(json["request_id"], "chatcmpl-header-req"); + assert_eq!(json["model"], "render-model"); + assert_eq!(json["stream"], true); + assert_eq!(json["stream_options"]["include_usage"], true); + assert_eq!(json["cache_salt"], "render-cache-salt"); + assert_eq!(json["priority"], -3); + assert_eq!(json["sampling_params"]["max_tokens"], 8); + assert!(!json["token_ids"].as_array().unwrap().is_empty()); + assert!(json.get("prompt_token_ids").is_none()); + assert!(json.get("mm_features").is_none()); +} + +#[tokio::test] +async fn render_chat_uses_processor_owned_parser_selections() { + let mut app = test_render_app_with_parser_selections( + ParserSelection::Auto, + ParserSelection::Explicit("definitely_missing_reasoning_parser".to_string()), + ); + let response = app + .call( + Request::builder() + .method("POST") + .uri("/v1/chat/completions/render") + .header("content-type", "application/json") + .body(Body::from( + json!({ + "model": "render-model", + "messages": [{"role": "user", "content": "hello"}], + "max_completion_tokens": 8 + }) + .to_string(), + )) + .expect("build request"), + ) + .await + .expect("call app"); + + assert_eq!(response.status(), StatusCode::BAD_REQUEST); + let body = to_bytes(response.into_body(), usize::MAX).await.expect("read body"); + let body = String::from_utf8(body.to_vec()).expect("decode body"); + assert!( + body.contains("reasoning parser `definitely_missing_reasoning_parser` is not registered") + ); +} + +#[tokio::test] +async fn render_completion_returns_generate_request_with_body_request_id() { + let mut app = test_render_app(); + let response = app + .call( + Request::builder() + .method("POST") + .uri("/v1/completions/render") + .header("content-type", "application/json") + .body(Body::from( + json!({ + "request_id": "body-req", + "model": "render-model", + "prompt": "hello", + "max_tokens": 8 + }) + .to_string(), + )) + .expect("build request"), + ) + .await + .expect("call app"); + + assert_eq!(response.status(), StatusCode::OK); + let body = to_bytes(response.into_body(), usize::MAX).await.expect("read body"); + let json: serde_json::Value = serde_json::from_slice(&body).expect("decode json"); + + assert_eq!(json[0]["request_id"], "cmpl-body-req"); + assert_eq!(json[0]["model"], "render-model"); + assert_eq!(json[0]["stream"], false); + assert_eq!(json[0]["sampling_params"]["max_tokens"], 8); + assert!(!json[0]["token_ids"].as_array().unwrap().is_empty()); + assert!(json[0].get("stream_options").is_none()); + assert!(json[0].get("prompt_token_ids").is_none()); + assert!(json[0].get("mm_features").is_none()); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[serial] +async fn render_chat_response_can_be_submitted_to_generate_unchanged() { + let mut render_app = test_render_app(); + let render_response = render_app + .call( + Request::builder() + .method("POST") + .uri("/v1/chat/completions/render") + .header("content-type", "application/json") + .body(Body::from( + json!({ + "request_id": "round-trip", + "model": "render-model", + "messages": [{"role": "user", "content": "hello"}], + "max_completion_tokens": 8, + "bad_words": ["blocked"], + "vllm_xargs": {"custom": 1} + }) + .to_string(), + )) + .expect("build render request"), + ) + .await + .expect("call render app"); + + assert_eq!(render_response.status(), StatusCode::OK); + let render_body = to_bytes(render_response.into_body(), usize::MAX) + .await + .expect("read render response"); + let render_json: serde_json::Value = + serde_json::from_slice(&render_body).expect("decode render response"); + let token_ids: Vec = serde_json::from_value(render_json["token_ids"].clone()) + .expect("decode rendered token IDs"); + + assert_eq!( + render_json["sampling_params"]["bad_words"], + json!(["blocked"]) + ); + assert_eq!(render_json["sampling_params"]["vllm_xargs"]["custom"], 1); + + let (chat, engine_task) = test_models_with_engine_outputs_and_backend_inner( + b"engine-render-generate-round-trip", + default_stream_output_specs(), + Some(token_ids), + Arc::new(FakeChatBackend::new()), + ) + .await; + let mut generate_app = build_router(Arc::new(AppState::new( + vec!["render-model".to_string()], + chat, + ))); + let generate_response = generate_app + .call( + Request::builder() + .method("POST") + .uri("/inference/v1/generate") + .header("content-type", "application/json") + .body(Body::from(render_body)) + .expect("build generate request"), + ) + .await + .expect("call generate app"); + + assert_eq!(generate_response.status(), StatusCode::OK); + let _ = to_bytes(generate_response.into_body(), usize::MAX) + .await + .expect("read generate response"); + engine_task.finish().await; +} + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] #[serial] async fn list_models_returns_configured_model() {