[Rust Frontend] Add standalone Rust renderer (#50289)

This commit is contained in:
Sage
2026-08-06 11:23:26 +00:00
committed by GitHub
parent ef2615c2e0
commit 865781e627
15 changed files with 720 additions and 40 deletions
+39 -1
View File
@@ -15,6 +15,7 @@ The component is organized as a Cargo workspace with several crates, layered bot
│ vllm-cmd / vllm-rs │ CLI entrypoint:
│ │ Python vLLM frontend subprocess
│ │ Rust managed-engine serve mode
│ │ Engine-free render mode
├─────────────────────────────────┤
│ vllm-server │ OpenAI-compatible HTTP API (axum)
├─────────────────────────────────┤
@@ -74,9 +75,46 @@ To build the `vllm-rs` in isolation:
./build_rust.sh
```
### Engine-free renderer
`vllm-rs render` serves text-only request preprocessing without starting or
connecting to a Python inference engine:
```bash
cargo run --manifest-path rust/Cargo.toml -p vllm-cmd --release -- \
render Qwen/Qwen3-32B \
--host 127.0.0.1 --max-model-len 32768
```
It exposes `/v1/chat/completions/render` and `/v1/completions/render`. Only
tokenizer and model configuration files are loaded; model weights, PyTorch,
and vLLM kernels are not required.
The render endpoints return the public token-in `GenerateRequest` consumed by
the Rust `/inference/v1/generate` endpoint. A chat render response, or one item
from a completion render response, can be submitted to that endpoint without
changing its fields.
The render and inference paths use the same `vllm-chat` and `vllm-text`
request-preparation logic; render mode stops before engine submission.
Tool-call and reasoning parsers use model-based auto-detection by default. Use
`--tool-call-parser` and `--reasoning-parser` to override either selection;
unified parsers require the same selection for both options.
```bash
curl http://127.0.0.1:8000/v1/chat/completions/render \
-H "Content-Type: application/json" \
-d '{
"model": "Qwen/Qwen3-32B",
"messages": [{"role": "user", "content": "Hello"}],
"max_completion_tokens": 16
}'
```
### Example Request
After either startup path, you can use any OpenAI-compatible client:
After either full-frontend startup path, you can use any OpenAI-compatible
client against the inference endpoints:
```bash
curl http://127.0.0.1:8000/v1/chat/completions \
+31 -22
View File
@@ -97,6 +97,10 @@ pub struct ChatRequestProcessor {
/// Effective model dtype reported by the engine.
/// Absent for text-only frontends without an engine handshake.
model_dtype: Option<ModelDtype>,
/// Tool-call parser selection used when preparing generation requests.
tool_call_parser: ParserSelection,
/// Reasoning parser selection used when preparing generation requests.
reasoning_parser: ParserSelection,
}
impl ChatRequestProcessor {
@@ -106,6 +110,8 @@ impl ChatRequestProcessor {
Self {
backend,
model_dtype: Some(model_dtype),
tool_call_parser: ParserSelection::Auto,
reasoning_parser: ParserSelection::Auto,
}
}
@@ -114,9 +120,22 @@ impl ChatRequestProcessor {
Self {
backend,
model_dtype: None,
tool_call_parser: ParserSelection::Auto,
reasoning_parser: ParserSelection::Auto,
}
}
/// Configure the parser selections used to prepare generation requests.
pub fn with_parser_selections(
mut self,
tool_call_parser: ParserSelection,
reasoning_parser: ParserSelection,
) -> Self {
self.tool_call_parser = tool_call_parser;
self.reasoning_parser = reasoning_parser;
self
}
async fn finalize_rendered_prompt(
&self,
request: &ChatRequest,
@@ -196,10 +215,15 @@ impl ChatRequestProcessor {
pub async fn prepare(
&self,
mut request: ChatRequest,
options: NewChatOutputProcessorOptions<'_>,
) -> Result<(TextRequest, DynChatOutputProcessor)> {
request.validate()?;
let output_processor = self.backend.new_chat_output_processor(&mut request, options)?;
let output_processor = self.backend.new_chat_output_processor(
&mut request,
NewChatOutputProcessorOptions {
tool_call_parser: &self.tool_call_parser,
reasoning_parser: &self.reasoning_parser,
},
)?;
let text_request = self.prepare_text_request(request).await?;
Ok((text_request, output_processor))
}
@@ -213,10 +237,6 @@ impl ChatRequestProcessor {
pub struct ChatLlm {
text: TextLlm,
processor: ChatRequestProcessor,
/// Tool-call parser selection.
tool_call_parser: ParserSelection,
/// Reasoning parser selection.
reasoning_parser: ParserSelection,
}
impl ChatLlm {
@@ -228,8 +248,6 @@ impl ChatLlm {
Self {
text,
processor: ChatRequestProcessor::new(backend, model_dtype),
tool_call_parser: ParserSelection::Auto,
reasoning_parser: ParserSelection::Auto,
}
}
@@ -242,13 +260,13 @@ impl ChatLlm {
/// Set tool-call parser selection.
pub fn with_tool_call_parser(mut self, selection: ParserSelection) -> Self {
self.tool_call_parser = selection;
self.processor.tool_call_parser = selection;
self
}
/// Set reasoning parser selection.
pub fn with_reasoning_parser(mut self, selection: ParserSelection) -> Self {
self.reasoning_parser = selection;
self.processor.reasoning_parser = selection;
self
}
@@ -306,7 +324,7 @@ impl ChatLlm {
/// Effective tool-call parser name for this model, if parsing is enabled.
pub fn tool_call_parser_name(&self) -> Option<&str> {
match &self.tool_call_parser {
match &self.processor.tool_call_parser {
ParserSelection::Auto => {
ToolParserFactory::global().resolve_name_for_model(self.model_id())
}
@@ -317,7 +335,7 @@ impl ChatLlm {
/// Effective reasoning parser name for this model, if parsing is enabled.
pub fn reasoning_parser_name(&self) -> Option<&str> {
match &self.reasoning_parser {
match &self.processor.reasoning_parser {
ParserSelection::Auto => {
ReasoningParserFactory::global().resolve_name_for_model(self.model_id())
}
@@ -328,16 +346,7 @@ impl ChatLlm {
/// Render, tokenize, and submit one chat request.
pub async fn chat(&self, request: ChatRequest) -> Result<ChatEventStream> {
let (text_request, output_processor) = self
.processor
.prepare(
request,
NewChatOutputProcessorOptions {
tool_call_parser: &self.tool_call_parser,
reasoning_parser: &self.reasoning_parser,
},
)
.await?;
let (text_request, output_processor) = self.processor.prepare(request).await?;
let request_id = text_request.request_id.clone();
let decoded_stream = self.text.generate(text_request).await?.map_err(Error::from).boxed();
+65 -1
View File
@@ -29,7 +29,8 @@ use vllm_managed_engine::ManagedEngineConfig;
use vllm_managed_engine::cli::{ManagedEngineArgs, repartition_managed_engine_args};
use vllm_server::{
ApiServerOptions, ChatTemplateContentFormatOption, Config, CoordinatorMode, CorsConfig,
DEFAULT_KEEP_ALIVE_TIMEOUT, HttpListenerMode, ParserSelection, RendererSelection, TlsConfig,
DEFAULT_KEEP_ALIVE_TIMEOUT, HttpListenerMode, ParserSelection, RenderConfig, RendererSelection,
TlsConfig,
};
use crate::cli::unsupported::UnsupportedArgs;
@@ -90,6 +91,8 @@ pub enum Command {
/// Run vLLM benchmarks.
#[command(subcommand)]
Bench(BenchCommand),
/// Run engine-free request rendering and preprocessing.
Render(RenderArgs),
}
/// Supported benchmark commands.
@@ -99,6 +102,67 @@ pub enum BenchCommand {
Serve(vllm_bench::BenchServeArgs),
}
/// Arguments for the engine-free text renderer.
#[derive(Debug, Args, PartialEq, Eq)]
pub struct RenderArgs {
/// Model identifier or local model directory containing tokenizer files.
model: String,
/// HTTP bind host.
#[arg(long, default_value = "127.0.0.1")]
host: String,
/// HTTP bind port.
#[arg(long, default_value_t = 8000)]
port: u16,
/// Public model names accepted by the API.
#[arg(long, num_args = 0..)]
served_model_name: Vec<String>,
/// Select the tool-call parser. Use `auto` to infer from the model or
/// `none` to disable parsing.
#[arg(long, default_value_t)]
tool_call_parser: ParserSelection,
/// Select the reasoning parser. Use `auto` to infer from the model or
/// `none` to disable parsing.
#[arg(long, default_value_t)]
reasoning_parser: ParserSelection,
/// Select the native chat renderer implementation.
#[arg(long = "tokenizer-mode", default_value_t)]
renderer: RendererSelection,
/// Override the model chat template with a file path or inline template.
#[arg(long)]
chat_template: Option<String>,
/// Default JSON keyword arguments merged into every chat-template render.
#[arg(long, value_parser = parse_json::<HashMap<String, Value>>, value_name = "JSON")]
default_chat_template_kwargs: Option<HashMap<String, Value>>,
/// How message content is exposed to the chat template.
#[arg(long, default_value_t)]
chat_template_content_format: ChatTemplateContentFormatOption,
/// Maximum model context length used for request validation.
#[arg(long)]
max_model_len: u32,
/// Maximum accepted logprobs count; -1 disables the cap.
#[arg(long, value_parser = clap::value_parser!(i32).range(-1..), allow_negative_numbers = true)]
max_logprobs: Option<i32>,
}
impl RenderArgs {
pub(super) fn into_config(self) -> RenderConfig {
RenderConfig {
model: self.model,
served_model_name: self.served_model_name,
host: self.host,
port: self.port,
tool_call_parser: self.tool_call_parser,
reasoning_parser: self.reasoning_parser,
renderer: self.renderer,
chat_template: self.chat_template,
default_chat_template_kwargs: self.default_chat_template_kwargs.unwrap_or_default(),
chat_template_content_format: self.chat_template_content_format,
max_model_len: self.max_model_len,
max_logprobs: self.max_logprobs,
}
}
}
/// A JSON-encoded list of strings, matching Python's `json.loads` CLI type for
/// the CORS list arguments (e.g. `--allowed-origins '["*"]'`). Parsing the whole
/// value as one item keeps clap from treating the field as a repeated flag.
+35
View File
@@ -27,6 +27,41 @@ fn bench_serve_args_parse_without_managed_engine_repartition() {
assert!(args.request_rate.is_infinite());
}
#[test]
fn render_args_build_supported_config() {
let cli = Cli::try_parse_from([
"vllm-rs",
"render",
"Qwen/Qwen2.5-0.5B-Instruct",
"--port",
"8080",
"--max-model-len",
"32768",
"--served-model-name",
"qwen",
"--tokenizer-mode",
"deepseek_v32",
"--max-logprobs",
"-1",
])
.unwrap();
let Command::Render(args) = cli.command else {
panic!("expected render args");
};
let config = args.into_config();
assert_eq!(config.model, "Qwen/Qwen2.5-0.5B-Instruct");
assert_eq!(config.host, "127.0.0.1");
assert_eq!(config.port, 8080);
assert_eq!(config.max_model_len, 32768);
assert_eq!(config.served_model_name, ["qwen"]);
assert_eq!(config.tool_call_parser, ParserSelection::Auto);
assert_eq!(config.reasoning_parser, ParserSelection::Auto);
assert_eq!(config.renderer, RendererSelection::DeepSeekV32);
assert_eq!(config.max_logprobs, Some(-1));
}
#[test]
fn serve_args_forward_python_flags_with_separator() {
let cli = Cli::try_parse_from([
+3
View File
@@ -211,5 +211,8 @@ async fn async_main(cli: Cli) -> Result<()> {
)),
}
}
Command::Render(args) => {
vllm_server::serve_render(args.into_config(), shutdown_signal()).await
}
}
}
+2
View File
@@ -9,6 +9,7 @@ mod grpc;
mod listener;
mod lora;
mod middleware;
mod render;
mod routes;
mod runtime;
mod server_info;
@@ -35,6 +36,7 @@ use hyper::server::conn::http1;
use hyper_util::rt::{TokioIo, TokioTimer};
use hyper_util::server::graceful::GracefulShutdown;
use hyper_util::service::TowerToHyperService;
pub use render::{RenderConfig, serve_render};
use tokio::net::TcpListener;
use tokio::time::{Instant, sleep_until};
use tokio_util::sync::CancellationToken;
+146
View File
@@ -0,0 +1,146 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright contributors to the vLLM project
use std::collections::HashMap;
use std::sync::Arc;
use anyhow::{Context as _, Result, bail};
use serde_json::Value;
use tokio_util::sync::CancellationToken;
use tracing::info;
use vllm_chat::{
ChatRequestProcessor, ChatTemplateContentFormatOption, LoadModelBackendsOptions,
ParserSelection, RendererSelection, load_model_backends,
};
use vllm_text::TextRequestProcessor;
use crate::{HttpListenerMode, listener::Listener};
/// Configuration for the engine-free text rendering server.
#[derive(Debug)]
pub struct RenderConfig {
pub model: String,
pub served_model_name: Vec<String>,
pub host: String,
pub port: u16,
pub tool_call_parser: ParserSelection,
pub reasoning_parser: ParserSelection,
pub renderer: RendererSelection,
pub chat_template: Option<String>,
pub default_chat_template_kwargs: HashMap<String, Value>,
pub chat_template_content_format: ChatTemplateContentFormatOption,
pub max_model_len: u32,
pub max_logprobs: Option<i32>,
}
impl RenderConfig {
/// Validate configuration before initializing renderer/tokenizer backends
/// or binding a listener.
pub fn validate(&self) -> Result<()> {
vllm_chat::validate_parser_overrides(&self.tool_call_parser, &self.reasoning_parser)?;
if self.max_logprobs.is_some_and(|value| value < -1) {
bail!("max_logprobs must be non-negative or -1");
}
Ok(())
}
}
pub(crate) struct RenderState {
pub(crate) model: String,
pub(crate) served_model_names: Vec<String>,
pub(crate) text: TextRequestProcessor,
pub(crate) chat: ChatRequestProcessor,
}
async fn build_state(config: &RenderConfig) -> Result<Arc<RenderState>> {
let loaded = load_model_backends(
&config.model,
LoadModelBackendsOptions {
renderer: config.renderer,
language_model_only: true,
chat_template: config.chat_template.clone(),
chat_template_content_format: config.chat_template_content_format,
default_chat_template_kwargs: config.default_chat_template_kwargs.clone(),
limit_mm_per_prompt: Default::default(),
},
)
.await
.context("failed to load renderer/tokenizer backends")?;
let served_model_names =
crate::effective_served_model_names(&config.model, &config.served_model_name);
let text = TextRequestProcessor::new(loaded.text_backend, config.max_model_len)
.with_max_logprobs(config.max_logprobs);
let chat = ChatRequestProcessor::render_only(loaded.chat_backend).with_parser_selections(
config.tool_call_parser.clone(),
config.reasoning_parser.clone(),
);
Ok(Arc::new(RenderState {
model: config.model.clone(),
served_model_names,
text,
chat,
}))
}
/// Run the text-only preprocessing server without starting or connecting
/// to an inference engine.
pub async fn serve_render(config: RenderConfig, shutdown: CancellationToken) -> Result<()> {
config.validate().context("invalid render server configuration")?;
let state = tokio::select! {
result = build_state(&config) => result?,
_ = shutdown.cancelled() => return Ok(()),
};
let listener_mode = HttpListenerMode::BindTcp {
host: config.host.clone(),
port: config.port,
};
let listener = Listener::bind(&listener_mode)
.await
.with_context(|| format!("failed to bind {}:{}", config.host, config.port))?;
let address = listener.local_addr_display()?;
info!(
%address,
model = %config.model,
"starting engine-free Rust render server"
);
axum::serve(listener, crate::routes::render::build_router(state))
.with_graceful_shutdown(shutdown.cancelled_owned())
.await
.context("render server failed")
}
#[cfg(test)]
mod tests {
use super::*;
#[tokio::test]
async fn serve_render_rejects_unknown_parser_before_startup() {
let shutdown = CancellationToken::new();
shutdown.cancel();
let error = serve_render(
RenderConfig {
model: "test-model".to_string(),
served_model_name: Vec::new(),
host: "127.0.0.1".to_string(),
port: 8000,
tool_call_parser: ParserSelection::Auto,
reasoning_parser: ParserSelection::Explicit("typo".to_string()),
renderer: RendererSelection::Auto,
chat_template: None,
default_chat_template_kwargs: HashMap::new(),
chat_template_content_format: ChatTemplateContentFormatOption::Auto,
max_model_len: 128,
max_logprobs: None,
},
shutdown,
)
.await
.unwrap_err();
let report = format!("{error:#}");
assert!(
report.contains("reasoning parser `typo` is not registered"),
"{report}"
);
}
}
+1
View File
@@ -12,6 +12,7 @@ mod metrics;
pub(crate) mod openai;
mod pause;
mod profile;
pub(super) mod render;
mod server_info;
mod sleep;
mod tokenize;
@@ -26,10 +26,12 @@ use vllm_llm::{
};
use self::convert::{ResponseOptions, prepare_generate_request};
pub(crate) use self::types::GenerateRequest;
use self::types::{
GenerateLogprob, GenerateRequest, GenerateResponse, GenerateResponseChoice,
GenerateResponseStreamChoice, GenerateStreamResponse,
GenerateLogprob, GenerateResponse, GenerateResponseChoice, GenerateResponseStreamChoice,
GenerateStreamResponse,
};
pub(crate) use self::validate::validate_request_compat;
use crate::config::ApiServerOptions;
use crate::error::{ApiError, bail_server_error, server_error, text_submit_error};
use crate::routes::openai::utils::logprobs::clamp_logprob;
@@ -6,7 +6,7 @@ use crate::error::{ApiError, bail_invalid_request};
/// Enforce the minimal compatibility contract for the Rust token generate
/// route.
pub(super) fn validate_request_compat(
pub(crate) fn validate_request_compat(
request: &GenerateRequest,
served_model_names: &[String],
) -> Result<(), ApiError> {
@@ -22,17 +22,18 @@ use tracing::{debug, error, info, trace};
use tracing_futures::Instrument as _;
use vllm_chat::{
AssistantBlockKind, AssistantMessageExt as _, ChatEvent, ChatEventStream, ChatEventStreamTrait,
CollectedAssistantMessage, FinishReason,
ChatRequest, CollectedAssistantMessage, FinishReason,
};
use vllm_engine_core_client::protocol::output::StopReason;
use self::convert::{ResponseOptions, prepare_chat_request};
pub(crate) use self::types::ChatCompletionRequest;
use crate::config::ApiServerOptions;
use crate::error::{ApiError, bail_server_error, chat_submit_error, server_error};
use crate::lora::LoraModelResolution;
use crate::routes::openai::chat_completions::types::{
AssistantRole, ChatCompletionChoice, ChatCompletionMessage, ChatCompletionRequest,
ChatCompletionResponse, ChatCompletionStreamChoice, ChatCompletionStreamResponse,
ChatMessageDelta,
AssistantRole, ChatCompletionChoice, ChatCompletionMessage, ChatCompletionResponse,
ChatCompletionStreamChoice, ChatCompletionStreamResponse, ChatMessageDelta,
};
use crate::routes::openai::utils::logprobs::{
decoded_logprobs_to_openai_chat, prompt_logprobs_to_maps,
@@ -43,7 +44,15 @@ use crate::routes::openai::utils::types::{
use crate::routes::openai::utils::usage::ContinuousUsage;
use crate::routes::openai::utils::validated_json::ValidatedJson;
use crate::state::AppState;
use crate::utils::{resolve_request_context, unix_timestamp};
use crate::utils::{ResolvedRequestContext, resolve_request_context, unix_timestamp};
pub(crate) fn lower_chat_request(
request: ChatCompletionRequest,
lora_resolution: &LoraModelResolution,
ctx: ResolvedRequestContext,
) -> Result<ChatRequest, ApiError> {
prepare_chat_request(request, lora_resolution, ctx).map(|prepared| prepared.chat_request)
}
/// Validate one chat completion request and proxy it into the shared
/// `vllm-chat` stack.
@@ -21,12 +21,14 @@ use thiserror_ext::AsReport as _;
use tracing::{debug, error, info, trace};
use tracing_futures::Instrument as _;
use vllm_engine_core_client::protocol::output::StopReason;
use vllm_text::tokenizer::Tokenizer;
use vllm_text::{
DecodedPromptLogprobs, DecodedTextEvent, FinishReason, TextOutputStream,
TextOutputStreamExt as _,
TextOutputStreamExt as _, TextRequest,
};
use self::convert::{ResponseOptions, prepare_completion_request};
pub(crate) use self::types::CompletionRequest;
use super::utils::logprobs::{
collected_logprobs_to_openai, decoded_logprobs_to_openai, decoded_prompt_logprobs_to_openai,
prompt_logprobs_to_maps, text_len,
@@ -34,15 +36,26 @@ use super::utils::logprobs::{
use super::utils::types::Usage;
use crate::config::ApiServerOptions;
use crate::error::{ApiError, bail_server_error, server_error, text_submit_error};
use crate::lora::LoraModelResolution;
use crate::routes::openai::completions::types::{
CompletionChoice, CompletionRequest, CompletionResponse, CompletionSseChunk,
CompletionStreamChoice, CompletionStreamResponse,
CompletionChoice, CompletionResponse, CompletionSseChunk, CompletionStreamChoice,
CompletionStreamResponse,
};
use crate::routes::openai::utils::types::LogProbs;
use crate::routes::openai::utils::usage::ContinuousUsage;
use crate::routes::openai::utils::validated_json::ValidatedJson;
use crate::state::AppState;
use crate::utils::{resolve_request_context, unix_timestamp};
use crate::utils::{ResolvedRequestContext, resolve_request_context, unix_timestamp};
pub(crate) fn lower_completion_request(
request: CompletionRequest,
lora_resolution: &LoraModelResolution,
ctx: ResolvedRequestContext,
tokenizer: &dyn Tokenizer,
) -> Result<TextRequest, ApiError> {
prepare_completion_request(request, lora_resolution, ctx, tokenizer)
.map(|prepared| prepared.text_request)
}
/// Validate one completions request and proxy it into the shared `vllm-text`
/// stack.
+2
View File
@@ -7,5 +7,7 @@ mod models;
pub(crate) mod utils;
pub use chat_completions::chat_completions;
pub(crate) use chat_completions::{ChatCompletionRequest, lower_chat_request};
pub use completions::completions;
pub(crate) use completions::{CompletionRequest, lower_completion_request};
pub use models::list_models;
+149
View File
@@ -0,0 +1,149 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright contributors to the vLLM project
use std::sync::Arc;
use axum::Json;
use axum::Router;
use axum::extract::DefaultBodyLimit;
use axum::extract::State;
use axum::http::{HeaderMap, StatusCode};
use axum::routing::{get, post};
use thiserror_ext::AsReport as _;
use vllm_text::TextRequest;
use crate::DEFAULT_REQUEST_BODY_LIMIT_BYTES;
use crate::error::{ApiError, text_submit_error};
use crate::lora::LoraModelResolution;
use crate::render::RenderState;
use crate::routes::inference::generate::{
GenerateRequest, validate_request_compat as validate_generate_request,
};
use crate::routes::openai::utils::types::{ListModelsResponse, ModelObject, StreamOptions};
use crate::routes::openai::utils::validated_json::ValidatedJson;
use crate::routes::openai::{
ChatCompletionRequest, CompletionRequest, lower_chat_request, lower_completion_request,
};
use crate::utils::{resolve_request_context, unix_timestamp};
pub(crate) fn build_router(state: Arc<RenderState>) -> Router {
Router::new()
.route("/health", get(health))
.route("/ping", get(health).post(health))
.route("/v1/models", get(list_models))
.route("/v1/chat/completions/render", post(render_chat))
.route("/v1/completions/render", post(render_completion))
.with_state(state)
.layer(DefaultBodyLimit::max(DEFAULT_REQUEST_BODY_LIMIT_BYTES))
}
async fn health() -> StatusCode {
StatusCode::OK
}
async fn list_models(State(state): State<Arc<RenderState>>) -> Json<ListModelsResponse> {
let created = unix_timestamp() as i64;
Json(ListModelsResponse {
object: "list".to_string(),
data: state
.served_model_names
.iter()
.map(|model| ModelObject {
id: model.clone(),
object: "model".to_string(),
created,
owned_by: "vllm".to_string(),
root: Some(state.model.clone()),
parent: None,
max_model_len: Some(state.text.max_model_len()),
})
.collect(),
})
}
fn model_resolution(state: &RenderState) -> LoraModelResolution {
LoraModelResolution {
model_names: state.served_model_names.clone(),
lora_request: None,
}
}
fn lower_render_request(
state: &RenderState,
text_request: TextRequest,
model: String,
stream: bool,
stream_options: Option<StreamOptions>,
) -> Result<GenerateRequest, ApiError> {
let prepared = state
.text
.prepare(text_request)
.map_err(|error| text_submit_error("failed to prepare render request", error))?;
let token_ids = prepared.generate_request.prompt_token_ids;
let text_request = prepared.text_request;
let request = GenerateRequest {
request_id: Some(text_request.request_id),
model: Some(model),
token_ids,
sampling_params: text_request.sampling_params,
stream,
stream_options,
cache_salt: text_request.cache_salt,
priority: text_request.priority,
kv_transfer_params: None,
ec_transfer_params: None,
other: Default::default(),
};
validate_generate_request(&request, &state.served_model_names)?;
Ok(request)
}
async fn render_chat(
State(state): State<Arc<RenderState>>,
headers: HeaderMap,
ValidatedJson(body): ValidatedJson<ChatCompletionRequest>,
) -> Result<Json<GenerateRequest>, ApiError> {
let model = body.model.clone();
let stream = body.stream;
let stream_options = body.stream_options.clone();
let request_context = resolve_request_context(&headers, body.request_id.as_deref());
let chat_request = lower_chat_request(body, &model_resolution(&state), request_context)?;
let (text_request, _) = state
.chat
.prepare(chat_request)
.await
.map_err(|error| ApiError::invalid_request(error.to_report_string(), None))?;
Ok(Json(lower_render_request(
&state,
text_request,
model,
stream,
stream_options,
)?))
}
async fn render_completion(
State(state): State<Arc<RenderState>>,
headers: HeaderMap,
ValidatedJson(body): ValidatedJson<CompletionRequest>,
) -> Result<Json<Vec<GenerateRequest>>, ApiError> {
let model = body.model.clone();
let stream = body.stream;
let stream_options = body.stream_options.clone();
let request_context = resolve_request_context(&headers, body.request_id.as_deref());
let tokenizer = state.text.tokenizer();
let text_request = lower_completion_request(
body,
&model_resolution(&state),
request_context,
tokenizer.as_ref(),
)?;
Ok(Json(vec![lower_render_request(
&state,
text_request,
model,
stream,
stream_options,
)?]))
}
+211 -4
View File
@@ -23,8 +23,8 @@ use serial_test::serial;
use tower::{Service as _, ServiceExt as _};
use vllm_chat::{
ChatBackend, ChatContent, ChatContentPart, ChatLlm, ChatMessage, ChatRenderer, ChatRequest,
ChatTextBackend, DefaultChatOutputProcessor, DynChatOutputProcessor, DynChatRenderer,
NewChatOutputProcessorOptions,
ChatRequestProcessor, ChatTextBackend, DefaultChatOutputProcessor, DynChatOutputProcessor,
DynChatRenderer, NewChatOutputProcessorOptions, ParserSelection,
};
use vllm_engine_core_client::mock_engine::default_ready_response;
use vllm_engine_core_client::protocol::decode_value;
@@ -46,13 +46,17 @@ use vllm_engine_core_client::{
use vllm_llm::Llm;
use vllm_metrics::METRICS;
use vllm_text::tokenizer::DynTokenizer;
use vllm_text::{Prompt, TextBackend};
use vllm_text::{Prompt, TextBackend, TextRequestProcessor};
use vllm_tokenizer::test_utils::TestTokenizer;
use zeromq::prelude::{SocketRecv, SocketSend};
use zeromq::{DealerSocket, PushSocket, ZmqMessage};
use super::{build_router, build_router_with_dev_mode, build_router_with_dev_mode_and_lora};
use super::{
build_router, build_router_with_dev_mode, build_router_with_dev_mode_and_lora,
render::build_router as build_render_router,
};
use crate::config::{ApiServerOptions, CorsConfig};
use crate::render::RenderState;
use crate::state::AppState;
fn request_output(
@@ -682,6 +686,24 @@ async fn test_app() -> axum::Router {
test_app_with_dev_mode(false).await
}
fn test_render_app() -> axum::Router {
test_render_app_with_parser_selections(ParserSelection::Auto, ParserSelection::Auto)
}
fn test_render_app_with_parser_selections(
tool_call_parser: ParserSelection,
reasoning_parser: ParserSelection,
) -> axum::Router {
let backend = Arc::new(FakeChatBackend::new());
build_render_router(Arc::new(RenderState {
model: "backend-model".to_string(),
served_model_names: vec!["render-model".to_string()],
text: TextRequestProcessor::new(backend.clone(), 128),
chat: ChatRequestProcessor::render_only(backend)
.with_parser_selections(tool_call_parser, reasoning_parser),
}))
}
async fn test_app_with_dev_mode(dev_mode_enabled: bool) -> axum::Router {
let (chat, _engine_task) = test_models_with_engine_outputs_and_backend(
b"engine-openai",
@@ -1043,6 +1065,191 @@ fn metric_delta(
- metric_value(rendered_before, metric, labels).unwrap_or(0.0)
}
#[tokio::test]
async fn render_chat_returns_generate_request_with_header_request_id() {
let mut app = test_render_app();
let response = app
.call(
Request::builder()
.method("POST")
.uri("/v1/chat/completions/render")
.header("content-type", "application/json")
.header("X-Request-Id", "header-req")
.body(Body::from(
json!({
"request_id": "body-req",
"model": "render-model",
"messages": [{"role": "user", "content": "hello"}],
"max_completion_tokens": 8,
"stream": true,
"stream_options": {"include_usage": true},
"cache_salt": "render-cache-salt",
"priority": -3
})
.to_string(),
))
.expect("build request"),
)
.await
.expect("call app");
assert_eq!(response.status(), StatusCode::OK);
let body = to_bytes(response.into_body(), usize::MAX).await.expect("read body");
let json: serde_json::Value = serde_json::from_slice(&body).expect("decode json");
assert_eq!(json["request_id"], "chatcmpl-header-req");
assert_eq!(json["model"], "render-model");
assert_eq!(json["stream"], true);
assert_eq!(json["stream_options"]["include_usage"], true);
assert_eq!(json["cache_salt"], "render-cache-salt");
assert_eq!(json["priority"], -3);
assert_eq!(json["sampling_params"]["max_tokens"], 8);
assert!(!json["token_ids"].as_array().unwrap().is_empty());
assert!(json.get("prompt_token_ids").is_none());
assert!(json.get("mm_features").is_none());
}
#[tokio::test]
async fn render_chat_uses_processor_owned_parser_selections() {
let mut app = test_render_app_with_parser_selections(
ParserSelection::Auto,
ParserSelection::Explicit("definitely_missing_reasoning_parser".to_string()),
);
let response = app
.call(
Request::builder()
.method("POST")
.uri("/v1/chat/completions/render")
.header("content-type", "application/json")
.body(Body::from(
json!({
"model": "render-model",
"messages": [{"role": "user", "content": "hello"}],
"max_completion_tokens": 8
})
.to_string(),
))
.expect("build request"),
)
.await
.expect("call app");
assert_eq!(response.status(), StatusCode::BAD_REQUEST);
let body = to_bytes(response.into_body(), usize::MAX).await.expect("read body");
let body = String::from_utf8(body.to_vec()).expect("decode body");
assert!(
body.contains("reasoning parser `definitely_missing_reasoning_parser` is not registered")
);
}
#[tokio::test]
async fn render_completion_returns_generate_request_with_body_request_id() {
let mut app = test_render_app();
let response = app
.call(
Request::builder()
.method("POST")
.uri("/v1/completions/render")
.header("content-type", "application/json")
.body(Body::from(
json!({
"request_id": "body-req",
"model": "render-model",
"prompt": "hello",
"max_tokens": 8
})
.to_string(),
))
.expect("build request"),
)
.await
.expect("call app");
assert_eq!(response.status(), StatusCode::OK);
let body = to_bytes(response.into_body(), usize::MAX).await.expect("read body");
let json: serde_json::Value = serde_json::from_slice(&body).expect("decode json");
assert_eq!(json[0]["request_id"], "cmpl-body-req");
assert_eq!(json[0]["model"], "render-model");
assert_eq!(json[0]["stream"], false);
assert_eq!(json[0]["sampling_params"]["max_tokens"], 8);
assert!(!json[0]["token_ids"].as_array().unwrap().is_empty());
assert!(json[0].get("stream_options").is_none());
assert!(json[0].get("prompt_token_ids").is_none());
assert!(json[0].get("mm_features").is_none());
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
#[serial]
async fn render_chat_response_can_be_submitted_to_generate_unchanged() {
let mut render_app = test_render_app();
let render_response = render_app
.call(
Request::builder()
.method("POST")
.uri("/v1/chat/completions/render")
.header("content-type", "application/json")
.body(Body::from(
json!({
"request_id": "round-trip",
"model": "render-model",
"messages": [{"role": "user", "content": "hello"}],
"max_completion_tokens": 8,
"bad_words": ["blocked"],
"vllm_xargs": {"custom": 1}
})
.to_string(),
))
.expect("build render request"),
)
.await
.expect("call render app");
assert_eq!(render_response.status(), StatusCode::OK);
let render_body = to_bytes(render_response.into_body(), usize::MAX)
.await
.expect("read render response");
let render_json: serde_json::Value =
serde_json::from_slice(&render_body).expect("decode render response");
let token_ids: Vec<u32> = serde_json::from_value(render_json["token_ids"].clone())
.expect("decode rendered token IDs");
assert_eq!(
render_json["sampling_params"]["bad_words"],
json!(["blocked"])
);
assert_eq!(render_json["sampling_params"]["vllm_xargs"]["custom"], 1);
let (chat, engine_task) = test_models_with_engine_outputs_and_backend_inner(
b"engine-render-generate-round-trip",
default_stream_output_specs(),
Some(token_ids),
Arc::new(FakeChatBackend::new()),
)
.await;
let mut generate_app = build_router(Arc::new(AppState::new(
vec!["render-model".to_string()],
chat,
)));
let generate_response = generate_app
.call(
Request::builder()
.method("POST")
.uri("/inference/v1/generate")
.header("content-type", "application/json")
.body(Body::from(render_body))
.expect("build generate request"),
)
.await
.expect("call generate app");
assert_eq!(generate_response.status(), StatusCode::OK);
let _ = to_bytes(generate_response.into_body(), usize::MAX)
.await
.expect("read generate response");
engine_task.finish().await;
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
#[serial]
async fn list_models_returns_configured_model() {