[Rust Frontend] Add standalone Rust renderer (#50289)

This commit is contained in:
Sage
2026-08-06 11:23:26 +00:00
committed by GitHub
parent ef2615c2e0
commit 865781e627
15 changed files with 720 additions and 40 deletions
+2
View File
@@ -9,6 +9,7 @@ mod grpc;
mod listener;
mod lora;
mod middleware;
mod render;
mod routes;
mod runtime;
mod server_info;
@@ -35,6 +36,7 @@ use hyper::server::conn::http1;
use hyper_util::rt::{TokioIo, TokioTimer};
use hyper_util::server::graceful::GracefulShutdown;
use hyper_util::service::TowerToHyperService;
pub use render::{RenderConfig, serve_render};
use tokio::net::TcpListener;
use tokio::time::{Instant, sleep_until};
use tokio_util::sync::CancellationToken;
+146
View File
@@ -0,0 +1,146 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright contributors to the vLLM project
use std::collections::HashMap;
use std::sync::Arc;
use anyhow::{Context as _, Result, bail};
use serde_json::Value;
use tokio_util::sync::CancellationToken;
use tracing::info;
use vllm_chat::{
ChatRequestProcessor, ChatTemplateContentFormatOption, LoadModelBackendsOptions,
ParserSelection, RendererSelection, load_model_backends,
};
use vllm_text::TextRequestProcessor;
use crate::{HttpListenerMode, listener::Listener};
/// Configuration for the engine-free text rendering server.
#[derive(Debug)]
pub struct RenderConfig {
pub model: String,
pub served_model_name: Vec<String>,
pub host: String,
pub port: u16,
pub tool_call_parser: ParserSelection,
pub reasoning_parser: ParserSelection,
pub renderer: RendererSelection,
pub chat_template: Option<String>,
pub default_chat_template_kwargs: HashMap<String, Value>,
pub chat_template_content_format: ChatTemplateContentFormatOption,
pub max_model_len: u32,
pub max_logprobs: Option<i32>,
}
impl RenderConfig {
/// Validate configuration before initializing renderer/tokenizer backends
/// or binding a listener.
pub fn validate(&self) -> Result<()> {
vllm_chat::validate_parser_overrides(&self.tool_call_parser, &self.reasoning_parser)?;
if self.max_logprobs.is_some_and(|value| value < -1) {
bail!("max_logprobs must be non-negative or -1");
}
Ok(())
}
}
pub(crate) struct RenderState {
pub(crate) model: String,
pub(crate) served_model_names: Vec<String>,
pub(crate) text: TextRequestProcessor,
pub(crate) chat: ChatRequestProcessor,
}
async fn build_state(config: &RenderConfig) -> Result<Arc<RenderState>> {
let loaded = load_model_backends(
&config.model,
LoadModelBackendsOptions {
renderer: config.renderer,
language_model_only: true,
chat_template: config.chat_template.clone(),
chat_template_content_format: config.chat_template_content_format,
default_chat_template_kwargs: config.default_chat_template_kwargs.clone(),
limit_mm_per_prompt: Default::default(),
},
)
.await
.context("failed to load renderer/tokenizer backends")?;
let served_model_names =
crate::effective_served_model_names(&config.model, &config.served_model_name);
let text = TextRequestProcessor::new(loaded.text_backend, config.max_model_len)
.with_max_logprobs(config.max_logprobs);
let chat = ChatRequestProcessor::render_only(loaded.chat_backend).with_parser_selections(
config.tool_call_parser.clone(),
config.reasoning_parser.clone(),
);
Ok(Arc::new(RenderState {
model: config.model.clone(),
served_model_names,
text,
chat,
}))
}
/// Run the text-only preprocessing server without starting or connecting
/// to an inference engine.
pub async fn serve_render(config: RenderConfig, shutdown: CancellationToken) -> Result<()> {
config.validate().context("invalid render server configuration")?;
let state = tokio::select! {
result = build_state(&config) => result?,
_ = shutdown.cancelled() => return Ok(()),
};
let listener_mode = HttpListenerMode::BindTcp {
host: config.host.clone(),
port: config.port,
};
let listener = Listener::bind(&listener_mode)
.await
.with_context(|| format!("failed to bind {}:{}", config.host, config.port))?;
let address = listener.local_addr_display()?;
info!(
%address,
model = %config.model,
"starting engine-free Rust render server"
);
axum::serve(listener, crate::routes::render::build_router(state))
.with_graceful_shutdown(shutdown.cancelled_owned())
.await
.context("render server failed")
}
#[cfg(test)]
mod tests {
use super::*;
#[tokio::test]
async fn serve_render_rejects_unknown_parser_before_startup() {
let shutdown = CancellationToken::new();
shutdown.cancel();
let error = serve_render(
RenderConfig {
model: "test-model".to_string(),
served_model_name: Vec::new(),
host: "127.0.0.1".to_string(),
port: 8000,
tool_call_parser: ParserSelection::Auto,
reasoning_parser: ParserSelection::Explicit("typo".to_string()),
renderer: RendererSelection::Auto,
chat_template: None,
default_chat_template_kwargs: HashMap::new(),
chat_template_content_format: ChatTemplateContentFormatOption::Auto,
max_model_len: 128,
max_logprobs: None,
},
shutdown,
)
.await
.unwrap_err();
let report = format!("{error:#}");
assert!(
report.contains("reasoning parser `typo` is not registered"),
"{report}"
);
}
}
+1
View File
@@ -12,6 +12,7 @@ mod metrics;
pub(crate) mod openai;
mod pause;
mod profile;
pub(super) mod render;
mod server_info;
mod sleep;
mod tokenize;
@@ -26,10 +26,12 @@ use vllm_llm::{
};
use self::convert::{ResponseOptions, prepare_generate_request};
pub(crate) use self::types::GenerateRequest;
use self::types::{
GenerateLogprob, GenerateRequest, GenerateResponse, GenerateResponseChoice,
GenerateResponseStreamChoice, GenerateStreamResponse,
GenerateLogprob, GenerateResponse, GenerateResponseChoice, GenerateResponseStreamChoice,
GenerateStreamResponse,
};
pub(crate) use self::validate::validate_request_compat;
use crate::config::ApiServerOptions;
use crate::error::{ApiError, bail_server_error, server_error, text_submit_error};
use crate::routes::openai::utils::logprobs::clamp_logprob;
@@ -6,7 +6,7 @@ use crate::error::{ApiError, bail_invalid_request};
/// Enforce the minimal compatibility contract for the Rust token generate
/// route.
pub(super) fn validate_request_compat(
pub(crate) fn validate_request_compat(
request: &GenerateRequest,
served_model_names: &[String],
) -> Result<(), ApiError> {
@@ -22,17 +22,18 @@ use tracing::{debug, error, info, trace};
use tracing_futures::Instrument as _;
use vllm_chat::{
AssistantBlockKind, AssistantMessageExt as _, ChatEvent, ChatEventStream, ChatEventStreamTrait,
CollectedAssistantMessage, FinishReason,
ChatRequest, CollectedAssistantMessage, FinishReason,
};
use vllm_engine_core_client::protocol::output::StopReason;
use self::convert::{ResponseOptions, prepare_chat_request};
pub(crate) use self::types::ChatCompletionRequest;
use crate::config::ApiServerOptions;
use crate::error::{ApiError, bail_server_error, chat_submit_error, server_error};
use crate::lora::LoraModelResolution;
use crate::routes::openai::chat_completions::types::{
AssistantRole, ChatCompletionChoice, ChatCompletionMessage, ChatCompletionRequest,
ChatCompletionResponse, ChatCompletionStreamChoice, ChatCompletionStreamResponse,
ChatMessageDelta,
AssistantRole, ChatCompletionChoice, ChatCompletionMessage, ChatCompletionResponse,
ChatCompletionStreamChoice, ChatCompletionStreamResponse, ChatMessageDelta,
};
use crate::routes::openai::utils::logprobs::{
decoded_logprobs_to_openai_chat, prompt_logprobs_to_maps,
@@ -43,7 +44,15 @@ use crate::routes::openai::utils::types::{
use crate::routes::openai::utils::usage::ContinuousUsage;
use crate::routes::openai::utils::validated_json::ValidatedJson;
use crate::state::AppState;
use crate::utils::{resolve_request_context, unix_timestamp};
use crate::utils::{ResolvedRequestContext, resolve_request_context, unix_timestamp};
pub(crate) fn lower_chat_request(
request: ChatCompletionRequest,
lora_resolution: &LoraModelResolution,
ctx: ResolvedRequestContext,
) -> Result<ChatRequest, ApiError> {
prepare_chat_request(request, lora_resolution, ctx).map(|prepared| prepared.chat_request)
}
/// Validate one chat completion request and proxy it into the shared
/// `vllm-chat` stack.
@@ -21,12 +21,14 @@ use thiserror_ext::AsReport as _;
use tracing::{debug, error, info, trace};
use tracing_futures::Instrument as _;
use vllm_engine_core_client::protocol::output::StopReason;
use vllm_text::tokenizer::Tokenizer;
use vllm_text::{
DecodedPromptLogprobs, DecodedTextEvent, FinishReason, TextOutputStream,
TextOutputStreamExt as _,
TextOutputStreamExt as _, TextRequest,
};
use self::convert::{ResponseOptions, prepare_completion_request};
pub(crate) use self::types::CompletionRequest;
use super::utils::logprobs::{
collected_logprobs_to_openai, decoded_logprobs_to_openai, decoded_prompt_logprobs_to_openai,
prompt_logprobs_to_maps, text_len,
@@ -34,15 +36,26 @@ use super::utils::logprobs::{
use super::utils::types::Usage;
use crate::config::ApiServerOptions;
use crate::error::{ApiError, bail_server_error, server_error, text_submit_error};
use crate::lora::LoraModelResolution;
use crate::routes::openai::completions::types::{
CompletionChoice, CompletionRequest, CompletionResponse, CompletionSseChunk,
CompletionStreamChoice, CompletionStreamResponse,
CompletionChoice, CompletionResponse, CompletionSseChunk, CompletionStreamChoice,
CompletionStreamResponse,
};
use crate::routes::openai::utils::types::LogProbs;
use crate::routes::openai::utils::usage::ContinuousUsage;
use crate::routes::openai::utils::validated_json::ValidatedJson;
use crate::state::AppState;
use crate::utils::{resolve_request_context, unix_timestamp};
use crate::utils::{ResolvedRequestContext, resolve_request_context, unix_timestamp};
pub(crate) fn lower_completion_request(
request: CompletionRequest,
lora_resolution: &LoraModelResolution,
ctx: ResolvedRequestContext,
tokenizer: &dyn Tokenizer,
) -> Result<TextRequest, ApiError> {
prepare_completion_request(request, lora_resolution, ctx, tokenizer)
.map(|prepared| prepared.text_request)
}
/// Validate one completions request and proxy it into the shared `vllm-text`
/// stack.
+2
View File
@@ -7,5 +7,7 @@ mod models;
pub(crate) mod utils;
pub use chat_completions::chat_completions;
pub(crate) use chat_completions::{ChatCompletionRequest, lower_chat_request};
pub use completions::completions;
pub(crate) use completions::{CompletionRequest, lower_completion_request};
pub use models::list_models;
+149
View File
@@ -0,0 +1,149 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright contributors to the vLLM project
use std::sync::Arc;
use axum::Json;
use axum::Router;
use axum::extract::DefaultBodyLimit;
use axum::extract::State;
use axum::http::{HeaderMap, StatusCode};
use axum::routing::{get, post};
use thiserror_ext::AsReport as _;
use vllm_text::TextRequest;
use crate::DEFAULT_REQUEST_BODY_LIMIT_BYTES;
use crate::error::{ApiError, text_submit_error};
use crate::lora::LoraModelResolution;
use crate::render::RenderState;
use crate::routes::inference::generate::{
GenerateRequest, validate_request_compat as validate_generate_request,
};
use crate::routes::openai::utils::types::{ListModelsResponse, ModelObject, StreamOptions};
use crate::routes::openai::utils::validated_json::ValidatedJson;
use crate::routes::openai::{
ChatCompletionRequest, CompletionRequest, lower_chat_request, lower_completion_request,
};
use crate::utils::{resolve_request_context, unix_timestamp};
pub(crate) fn build_router(state: Arc<RenderState>) -> Router {
Router::new()
.route("/health", get(health))
.route("/ping", get(health).post(health))
.route("/v1/models", get(list_models))
.route("/v1/chat/completions/render", post(render_chat))
.route("/v1/completions/render", post(render_completion))
.with_state(state)
.layer(DefaultBodyLimit::max(DEFAULT_REQUEST_BODY_LIMIT_BYTES))
}
async fn health() -> StatusCode {
StatusCode::OK
}
async fn list_models(State(state): State<Arc<RenderState>>) -> Json<ListModelsResponse> {
let created = unix_timestamp() as i64;
Json(ListModelsResponse {
object: "list".to_string(),
data: state
.served_model_names
.iter()
.map(|model| ModelObject {
id: model.clone(),
object: "model".to_string(),
created,
owned_by: "vllm".to_string(),
root: Some(state.model.clone()),
parent: None,
max_model_len: Some(state.text.max_model_len()),
})
.collect(),
})
}
fn model_resolution(state: &RenderState) -> LoraModelResolution {
LoraModelResolution {
model_names: state.served_model_names.clone(),
lora_request: None,
}
}
fn lower_render_request(
state: &RenderState,
text_request: TextRequest,
model: String,
stream: bool,
stream_options: Option<StreamOptions>,
) -> Result<GenerateRequest, ApiError> {
let prepared = state
.text
.prepare(text_request)
.map_err(|error| text_submit_error("failed to prepare render request", error))?;
let token_ids = prepared.generate_request.prompt_token_ids;
let text_request = prepared.text_request;
let request = GenerateRequest {
request_id: Some(text_request.request_id),
model: Some(model),
token_ids,
sampling_params: text_request.sampling_params,
stream,
stream_options,
cache_salt: text_request.cache_salt,
priority: text_request.priority,
kv_transfer_params: None,
ec_transfer_params: None,
other: Default::default(),
};
validate_generate_request(&request, &state.served_model_names)?;
Ok(request)
}
async fn render_chat(
State(state): State<Arc<RenderState>>,
headers: HeaderMap,
ValidatedJson(body): ValidatedJson<ChatCompletionRequest>,
) -> Result<Json<GenerateRequest>, ApiError> {
let model = body.model.clone();
let stream = body.stream;
let stream_options = body.stream_options.clone();
let request_context = resolve_request_context(&headers, body.request_id.as_deref());
let chat_request = lower_chat_request(body, &model_resolution(&state), request_context)?;
let (text_request, _) = state
.chat
.prepare(chat_request)
.await
.map_err(|error| ApiError::invalid_request(error.to_report_string(), None))?;
Ok(Json(lower_render_request(
&state,
text_request,
model,
stream,
stream_options,
)?))
}
async fn render_completion(
State(state): State<Arc<RenderState>>,
headers: HeaderMap,
ValidatedJson(body): ValidatedJson<CompletionRequest>,
) -> Result<Json<Vec<GenerateRequest>>, ApiError> {
let model = body.model.clone();
let stream = body.stream;
let stream_options = body.stream_options.clone();
let request_context = resolve_request_context(&headers, body.request_id.as_deref());
let tokenizer = state.text.tokenizer();
let text_request = lower_completion_request(
body,
&model_resolution(&state),
request_context,
tokenizer.as_ref(),
)?;
Ok(Json(vec![lower_render_request(
&state,
text_request,
model,
stream,
stream_options,
)?]))
}
+211 -4
View File
@@ -23,8 +23,8 @@ use serial_test::serial;
use tower::{Service as _, ServiceExt as _};
use vllm_chat::{
ChatBackend, ChatContent, ChatContentPart, ChatLlm, ChatMessage, ChatRenderer, ChatRequest,
ChatTextBackend, DefaultChatOutputProcessor, DynChatOutputProcessor, DynChatRenderer,
NewChatOutputProcessorOptions,
ChatRequestProcessor, ChatTextBackend, DefaultChatOutputProcessor, DynChatOutputProcessor,
DynChatRenderer, NewChatOutputProcessorOptions, ParserSelection,
};
use vllm_engine_core_client::mock_engine::default_ready_response;
use vllm_engine_core_client::protocol::decode_value;
@@ -46,13 +46,17 @@ use vllm_engine_core_client::{
use vllm_llm::Llm;
use vllm_metrics::METRICS;
use vllm_text::tokenizer::DynTokenizer;
use vllm_text::{Prompt, TextBackend};
use vllm_text::{Prompt, TextBackend, TextRequestProcessor};
use vllm_tokenizer::test_utils::TestTokenizer;
use zeromq::prelude::{SocketRecv, SocketSend};
use zeromq::{DealerSocket, PushSocket, ZmqMessage};
use super::{build_router, build_router_with_dev_mode, build_router_with_dev_mode_and_lora};
use super::{
build_router, build_router_with_dev_mode, build_router_with_dev_mode_and_lora,
render::build_router as build_render_router,
};
use crate::config::{ApiServerOptions, CorsConfig};
use crate::render::RenderState;
use crate::state::AppState;
fn request_output(
@@ -682,6 +686,24 @@ async fn test_app() -> axum::Router {
test_app_with_dev_mode(false).await
}
fn test_render_app() -> axum::Router {
test_render_app_with_parser_selections(ParserSelection::Auto, ParserSelection::Auto)
}
fn test_render_app_with_parser_selections(
tool_call_parser: ParserSelection,
reasoning_parser: ParserSelection,
) -> axum::Router {
let backend = Arc::new(FakeChatBackend::new());
build_render_router(Arc::new(RenderState {
model: "backend-model".to_string(),
served_model_names: vec!["render-model".to_string()],
text: TextRequestProcessor::new(backend.clone(), 128),
chat: ChatRequestProcessor::render_only(backend)
.with_parser_selections(tool_call_parser, reasoning_parser),
}))
}
async fn test_app_with_dev_mode(dev_mode_enabled: bool) -> axum::Router {
let (chat, _engine_task) = test_models_with_engine_outputs_and_backend(
b"engine-openai",
@@ -1043,6 +1065,191 @@ fn metric_delta(
- metric_value(rendered_before, metric, labels).unwrap_or(0.0)
}
#[tokio::test]
async fn render_chat_returns_generate_request_with_header_request_id() {
let mut app = test_render_app();
let response = app
.call(
Request::builder()
.method("POST")
.uri("/v1/chat/completions/render")
.header("content-type", "application/json")
.header("X-Request-Id", "header-req")
.body(Body::from(
json!({
"request_id": "body-req",
"model": "render-model",
"messages": [{"role": "user", "content": "hello"}],
"max_completion_tokens": 8,
"stream": true,
"stream_options": {"include_usage": true},
"cache_salt": "render-cache-salt",
"priority": -3
})
.to_string(),
))
.expect("build request"),
)
.await
.expect("call app");
assert_eq!(response.status(), StatusCode::OK);
let body = to_bytes(response.into_body(), usize::MAX).await.expect("read body");
let json: serde_json::Value = serde_json::from_slice(&body).expect("decode json");
assert_eq!(json["request_id"], "chatcmpl-header-req");
assert_eq!(json["model"], "render-model");
assert_eq!(json["stream"], true);
assert_eq!(json["stream_options"]["include_usage"], true);
assert_eq!(json["cache_salt"], "render-cache-salt");
assert_eq!(json["priority"], -3);
assert_eq!(json["sampling_params"]["max_tokens"], 8);
assert!(!json["token_ids"].as_array().unwrap().is_empty());
assert!(json.get("prompt_token_ids").is_none());
assert!(json.get("mm_features").is_none());
}
#[tokio::test]
async fn render_chat_uses_processor_owned_parser_selections() {
let mut app = test_render_app_with_parser_selections(
ParserSelection::Auto,
ParserSelection::Explicit("definitely_missing_reasoning_parser".to_string()),
);
let response = app
.call(
Request::builder()
.method("POST")
.uri("/v1/chat/completions/render")
.header("content-type", "application/json")
.body(Body::from(
json!({
"model": "render-model",
"messages": [{"role": "user", "content": "hello"}],
"max_completion_tokens": 8
})
.to_string(),
))
.expect("build request"),
)
.await
.expect("call app");
assert_eq!(response.status(), StatusCode::BAD_REQUEST);
let body = to_bytes(response.into_body(), usize::MAX).await.expect("read body");
let body = String::from_utf8(body.to_vec()).expect("decode body");
assert!(
body.contains("reasoning parser `definitely_missing_reasoning_parser` is not registered")
);
}
#[tokio::test]
async fn render_completion_returns_generate_request_with_body_request_id() {
let mut app = test_render_app();
let response = app
.call(
Request::builder()
.method("POST")
.uri("/v1/completions/render")
.header("content-type", "application/json")
.body(Body::from(
json!({
"request_id": "body-req",
"model": "render-model",
"prompt": "hello",
"max_tokens": 8
})
.to_string(),
))
.expect("build request"),
)
.await
.expect("call app");
assert_eq!(response.status(), StatusCode::OK);
let body = to_bytes(response.into_body(), usize::MAX).await.expect("read body");
let json: serde_json::Value = serde_json::from_slice(&body).expect("decode json");
assert_eq!(json[0]["request_id"], "cmpl-body-req");
assert_eq!(json[0]["model"], "render-model");
assert_eq!(json[0]["stream"], false);
assert_eq!(json[0]["sampling_params"]["max_tokens"], 8);
assert!(!json[0]["token_ids"].as_array().unwrap().is_empty());
assert!(json[0].get("stream_options").is_none());
assert!(json[0].get("prompt_token_ids").is_none());
assert!(json[0].get("mm_features").is_none());
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
#[serial]
async fn render_chat_response_can_be_submitted_to_generate_unchanged() {
let mut render_app = test_render_app();
let render_response = render_app
.call(
Request::builder()
.method("POST")
.uri("/v1/chat/completions/render")
.header("content-type", "application/json")
.body(Body::from(
json!({
"request_id": "round-trip",
"model": "render-model",
"messages": [{"role": "user", "content": "hello"}],
"max_completion_tokens": 8,
"bad_words": ["blocked"],
"vllm_xargs": {"custom": 1}
})
.to_string(),
))
.expect("build render request"),
)
.await
.expect("call render app");
assert_eq!(render_response.status(), StatusCode::OK);
let render_body = to_bytes(render_response.into_body(), usize::MAX)
.await
.expect("read render response");
let render_json: serde_json::Value =
serde_json::from_slice(&render_body).expect("decode render response");
let token_ids: Vec<u32> = serde_json::from_value(render_json["token_ids"].clone())
.expect("decode rendered token IDs");
assert_eq!(
render_json["sampling_params"]["bad_words"],
json!(["blocked"])
);
assert_eq!(render_json["sampling_params"]["vllm_xargs"]["custom"], 1);
let (chat, engine_task) = test_models_with_engine_outputs_and_backend_inner(
b"engine-render-generate-round-trip",
default_stream_output_specs(),
Some(token_ids),
Arc::new(FakeChatBackend::new()),
)
.await;
let mut generate_app = build_router(Arc::new(AppState::new(
vec!["render-model".to_string()],
chat,
)));
let generate_response = generate_app
.call(
Request::builder()
.method("POST")
.uri("/inference/v1/generate")
.header("content-type", "application/json")
.body(Body::from(render_body))
.expect("build generate request"),
)
.await
.expect("call generate app");
assert_eq!(generate_response.status(), StatusCode::OK);
let _ = to_bytes(generate_response.into_body(), usize::MAX)
.await
.expect("read generate response");
engine_task.finish().await;
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
#[serial]
async fn list_models_returns_configured_model() {