[Rust Frontend] Add standalone Rust renderer (#50289)
This commit is contained in:
@@ -9,6 +9,7 @@ mod grpc;
|
||||
mod listener;
|
||||
mod lora;
|
||||
mod middleware;
|
||||
mod render;
|
||||
mod routes;
|
||||
mod runtime;
|
||||
mod server_info;
|
||||
@@ -35,6 +36,7 @@ use hyper::server::conn::http1;
|
||||
use hyper_util::rt::{TokioIo, TokioTimer};
|
||||
use hyper_util::server::graceful::GracefulShutdown;
|
||||
use hyper_util::service::TowerToHyperService;
|
||||
pub use render::{RenderConfig, serve_render};
|
||||
use tokio::net::TcpListener;
|
||||
use tokio::time::{Instant, sleep_until};
|
||||
use tokio_util::sync::CancellationToken;
|
||||
|
||||
@@ -0,0 +1,146 @@
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
// SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::sync::Arc;
|
||||
|
||||
use anyhow::{Context as _, Result, bail};
|
||||
use serde_json::Value;
|
||||
use tokio_util::sync::CancellationToken;
|
||||
use tracing::info;
|
||||
use vllm_chat::{
|
||||
ChatRequestProcessor, ChatTemplateContentFormatOption, LoadModelBackendsOptions,
|
||||
ParserSelection, RendererSelection, load_model_backends,
|
||||
};
|
||||
use vllm_text::TextRequestProcessor;
|
||||
|
||||
use crate::{HttpListenerMode, listener::Listener};
|
||||
|
||||
/// Configuration for the engine-free text rendering server.
|
||||
#[derive(Debug)]
|
||||
pub struct RenderConfig {
|
||||
pub model: String,
|
||||
pub served_model_name: Vec<String>,
|
||||
pub host: String,
|
||||
pub port: u16,
|
||||
pub tool_call_parser: ParserSelection,
|
||||
pub reasoning_parser: ParserSelection,
|
||||
pub renderer: RendererSelection,
|
||||
pub chat_template: Option<String>,
|
||||
pub default_chat_template_kwargs: HashMap<String, Value>,
|
||||
pub chat_template_content_format: ChatTemplateContentFormatOption,
|
||||
pub max_model_len: u32,
|
||||
pub max_logprobs: Option<i32>,
|
||||
}
|
||||
|
||||
impl RenderConfig {
|
||||
/// Validate configuration before initializing renderer/tokenizer backends
|
||||
/// or binding a listener.
|
||||
pub fn validate(&self) -> Result<()> {
|
||||
vllm_chat::validate_parser_overrides(&self.tool_call_parser, &self.reasoning_parser)?;
|
||||
if self.max_logprobs.is_some_and(|value| value < -1) {
|
||||
bail!("max_logprobs must be non-negative or -1");
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) struct RenderState {
|
||||
pub(crate) model: String,
|
||||
pub(crate) served_model_names: Vec<String>,
|
||||
pub(crate) text: TextRequestProcessor,
|
||||
pub(crate) chat: ChatRequestProcessor,
|
||||
}
|
||||
|
||||
async fn build_state(config: &RenderConfig) -> Result<Arc<RenderState>> {
|
||||
let loaded = load_model_backends(
|
||||
&config.model,
|
||||
LoadModelBackendsOptions {
|
||||
renderer: config.renderer,
|
||||
language_model_only: true,
|
||||
chat_template: config.chat_template.clone(),
|
||||
chat_template_content_format: config.chat_template_content_format,
|
||||
default_chat_template_kwargs: config.default_chat_template_kwargs.clone(),
|
||||
limit_mm_per_prompt: Default::default(),
|
||||
},
|
||||
)
|
||||
.await
|
||||
.context("failed to load renderer/tokenizer backends")?;
|
||||
let served_model_names =
|
||||
crate::effective_served_model_names(&config.model, &config.served_model_name);
|
||||
let text = TextRequestProcessor::new(loaded.text_backend, config.max_model_len)
|
||||
.with_max_logprobs(config.max_logprobs);
|
||||
let chat = ChatRequestProcessor::render_only(loaded.chat_backend).with_parser_selections(
|
||||
config.tool_call_parser.clone(),
|
||||
config.reasoning_parser.clone(),
|
||||
);
|
||||
Ok(Arc::new(RenderState {
|
||||
model: config.model.clone(),
|
||||
served_model_names,
|
||||
text,
|
||||
chat,
|
||||
}))
|
||||
}
|
||||
|
||||
/// Run the text-only preprocessing server without starting or connecting
|
||||
/// to an inference engine.
|
||||
pub async fn serve_render(config: RenderConfig, shutdown: CancellationToken) -> Result<()> {
|
||||
config.validate().context("invalid render server configuration")?;
|
||||
let state = tokio::select! {
|
||||
result = build_state(&config) => result?,
|
||||
_ = shutdown.cancelled() => return Ok(()),
|
||||
};
|
||||
let listener_mode = HttpListenerMode::BindTcp {
|
||||
host: config.host.clone(),
|
||||
port: config.port,
|
||||
};
|
||||
let listener = Listener::bind(&listener_mode)
|
||||
.await
|
||||
.with_context(|| format!("failed to bind {}:{}", config.host, config.port))?;
|
||||
let address = listener.local_addr_display()?;
|
||||
info!(
|
||||
%address,
|
||||
model = %config.model,
|
||||
"starting engine-free Rust render server"
|
||||
);
|
||||
axum::serve(listener, crate::routes::render::build_router(state))
|
||||
.with_graceful_shutdown(shutdown.cancelled_owned())
|
||||
.await
|
||||
.context("render server failed")
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[tokio::test]
|
||||
async fn serve_render_rejects_unknown_parser_before_startup() {
|
||||
let shutdown = CancellationToken::new();
|
||||
shutdown.cancel();
|
||||
let error = serve_render(
|
||||
RenderConfig {
|
||||
model: "test-model".to_string(),
|
||||
served_model_name: Vec::new(),
|
||||
host: "127.0.0.1".to_string(),
|
||||
port: 8000,
|
||||
tool_call_parser: ParserSelection::Auto,
|
||||
reasoning_parser: ParserSelection::Explicit("typo".to_string()),
|
||||
renderer: RendererSelection::Auto,
|
||||
chat_template: None,
|
||||
default_chat_template_kwargs: HashMap::new(),
|
||||
chat_template_content_format: ChatTemplateContentFormatOption::Auto,
|
||||
max_model_len: 128,
|
||||
max_logprobs: None,
|
||||
},
|
||||
shutdown,
|
||||
)
|
||||
.await
|
||||
.unwrap_err();
|
||||
let report = format!("{error:#}");
|
||||
|
||||
assert!(
|
||||
report.contains("reasoning parser `typo` is not registered"),
|
||||
"{report}"
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -12,6 +12,7 @@ mod metrics;
|
||||
pub(crate) mod openai;
|
||||
mod pause;
|
||||
mod profile;
|
||||
pub(super) mod render;
|
||||
mod server_info;
|
||||
mod sleep;
|
||||
mod tokenize;
|
||||
|
||||
@@ -26,10 +26,12 @@ use vllm_llm::{
|
||||
};
|
||||
|
||||
use self::convert::{ResponseOptions, prepare_generate_request};
|
||||
pub(crate) use self::types::GenerateRequest;
|
||||
use self::types::{
|
||||
GenerateLogprob, GenerateRequest, GenerateResponse, GenerateResponseChoice,
|
||||
GenerateResponseStreamChoice, GenerateStreamResponse,
|
||||
GenerateLogprob, GenerateResponse, GenerateResponseChoice, GenerateResponseStreamChoice,
|
||||
GenerateStreamResponse,
|
||||
};
|
||||
pub(crate) use self::validate::validate_request_compat;
|
||||
use crate::config::ApiServerOptions;
|
||||
use crate::error::{ApiError, bail_server_error, server_error, text_submit_error};
|
||||
use crate::routes::openai::utils::logprobs::clamp_logprob;
|
||||
|
||||
@@ -6,7 +6,7 @@ use crate::error::{ApiError, bail_invalid_request};
|
||||
|
||||
/// Enforce the minimal compatibility contract for the Rust token generate
|
||||
/// route.
|
||||
pub(super) fn validate_request_compat(
|
||||
pub(crate) fn validate_request_compat(
|
||||
request: &GenerateRequest,
|
||||
served_model_names: &[String],
|
||||
) -> Result<(), ApiError> {
|
||||
|
||||
@@ -22,17 +22,18 @@ use tracing::{debug, error, info, trace};
|
||||
use tracing_futures::Instrument as _;
|
||||
use vllm_chat::{
|
||||
AssistantBlockKind, AssistantMessageExt as _, ChatEvent, ChatEventStream, ChatEventStreamTrait,
|
||||
CollectedAssistantMessage, FinishReason,
|
||||
ChatRequest, CollectedAssistantMessage, FinishReason,
|
||||
};
|
||||
use vllm_engine_core_client::protocol::output::StopReason;
|
||||
|
||||
use self::convert::{ResponseOptions, prepare_chat_request};
|
||||
pub(crate) use self::types::ChatCompletionRequest;
|
||||
use crate::config::ApiServerOptions;
|
||||
use crate::error::{ApiError, bail_server_error, chat_submit_error, server_error};
|
||||
use crate::lora::LoraModelResolution;
|
||||
use crate::routes::openai::chat_completions::types::{
|
||||
AssistantRole, ChatCompletionChoice, ChatCompletionMessage, ChatCompletionRequest,
|
||||
ChatCompletionResponse, ChatCompletionStreamChoice, ChatCompletionStreamResponse,
|
||||
ChatMessageDelta,
|
||||
AssistantRole, ChatCompletionChoice, ChatCompletionMessage, ChatCompletionResponse,
|
||||
ChatCompletionStreamChoice, ChatCompletionStreamResponse, ChatMessageDelta,
|
||||
};
|
||||
use crate::routes::openai::utils::logprobs::{
|
||||
decoded_logprobs_to_openai_chat, prompt_logprobs_to_maps,
|
||||
@@ -43,7 +44,15 @@ use crate::routes::openai::utils::types::{
|
||||
use crate::routes::openai::utils::usage::ContinuousUsage;
|
||||
use crate::routes::openai::utils::validated_json::ValidatedJson;
|
||||
use crate::state::AppState;
|
||||
use crate::utils::{resolve_request_context, unix_timestamp};
|
||||
use crate::utils::{ResolvedRequestContext, resolve_request_context, unix_timestamp};
|
||||
|
||||
pub(crate) fn lower_chat_request(
|
||||
request: ChatCompletionRequest,
|
||||
lora_resolution: &LoraModelResolution,
|
||||
ctx: ResolvedRequestContext,
|
||||
) -> Result<ChatRequest, ApiError> {
|
||||
prepare_chat_request(request, lora_resolution, ctx).map(|prepared| prepared.chat_request)
|
||||
}
|
||||
|
||||
/// Validate one chat completion request and proxy it into the shared
|
||||
/// `vllm-chat` stack.
|
||||
|
||||
@@ -21,12 +21,14 @@ use thiserror_ext::AsReport as _;
|
||||
use tracing::{debug, error, info, trace};
|
||||
use tracing_futures::Instrument as _;
|
||||
use vllm_engine_core_client::protocol::output::StopReason;
|
||||
use vllm_text::tokenizer::Tokenizer;
|
||||
use vllm_text::{
|
||||
DecodedPromptLogprobs, DecodedTextEvent, FinishReason, TextOutputStream,
|
||||
TextOutputStreamExt as _,
|
||||
TextOutputStreamExt as _, TextRequest,
|
||||
};
|
||||
|
||||
use self::convert::{ResponseOptions, prepare_completion_request};
|
||||
pub(crate) use self::types::CompletionRequest;
|
||||
use super::utils::logprobs::{
|
||||
collected_logprobs_to_openai, decoded_logprobs_to_openai, decoded_prompt_logprobs_to_openai,
|
||||
prompt_logprobs_to_maps, text_len,
|
||||
@@ -34,15 +36,26 @@ use super::utils::logprobs::{
|
||||
use super::utils::types::Usage;
|
||||
use crate::config::ApiServerOptions;
|
||||
use crate::error::{ApiError, bail_server_error, server_error, text_submit_error};
|
||||
use crate::lora::LoraModelResolution;
|
||||
use crate::routes::openai::completions::types::{
|
||||
CompletionChoice, CompletionRequest, CompletionResponse, CompletionSseChunk,
|
||||
CompletionStreamChoice, CompletionStreamResponse,
|
||||
CompletionChoice, CompletionResponse, CompletionSseChunk, CompletionStreamChoice,
|
||||
CompletionStreamResponse,
|
||||
};
|
||||
use crate::routes::openai::utils::types::LogProbs;
|
||||
use crate::routes::openai::utils::usage::ContinuousUsage;
|
||||
use crate::routes::openai::utils::validated_json::ValidatedJson;
|
||||
use crate::state::AppState;
|
||||
use crate::utils::{resolve_request_context, unix_timestamp};
|
||||
use crate::utils::{ResolvedRequestContext, resolve_request_context, unix_timestamp};
|
||||
|
||||
pub(crate) fn lower_completion_request(
|
||||
request: CompletionRequest,
|
||||
lora_resolution: &LoraModelResolution,
|
||||
ctx: ResolvedRequestContext,
|
||||
tokenizer: &dyn Tokenizer,
|
||||
) -> Result<TextRequest, ApiError> {
|
||||
prepare_completion_request(request, lora_resolution, ctx, tokenizer)
|
||||
.map(|prepared| prepared.text_request)
|
||||
}
|
||||
|
||||
/// Validate one completions request and proxy it into the shared `vllm-text`
|
||||
/// stack.
|
||||
|
||||
@@ -7,5 +7,7 @@ mod models;
|
||||
pub(crate) mod utils;
|
||||
|
||||
pub use chat_completions::chat_completions;
|
||||
pub(crate) use chat_completions::{ChatCompletionRequest, lower_chat_request};
|
||||
pub use completions::completions;
|
||||
pub(crate) use completions::{CompletionRequest, lower_completion_request};
|
||||
pub use models::list_models;
|
||||
|
||||
@@ -0,0 +1,149 @@
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
// SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
use std::sync::Arc;
|
||||
|
||||
use axum::Json;
|
||||
use axum::Router;
|
||||
use axum::extract::DefaultBodyLimit;
|
||||
use axum::extract::State;
|
||||
use axum::http::{HeaderMap, StatusCode};
|
||||
use axum::routing::{get, post};
|
||||
use thiserror_ext::AsReport as _;
|
||||
use vllm_text::TextRequest;
|
||||
|
||||
use crate::DEFAULT_REQUEST_BODY_LIMIT_BYTES;
|
||||
use crate::error::{ApiError, text_submit_error};
|
||||
use crate::lora::LoraModelResolution;
|
||||
use crate::render::RenderState;
|
||||
use crate::routes::inference::generate::{
|
||||
GenerateRequest, validate_request_compat as validate_generate_request,
|
||||
};
|
||||
use crate::routes::openai::utils::types::{ListModelsResponse, ModelObject, StreamOptions};
|
||||
use crate::routes::openai::utils::validated_json::ValidatedJson;
|
||||
use crate::routes::openai::{
|
||||
ChatCompletionRequest, CompletionRequest, lower_chat_request, lower_completion_request,
|
||||
};
|
||||
use crate::utils::{resolve_request_context, unix_timestamp};
|
||||
|
||||
pub(crate) fn build_router(state: Arc<RenderState>) -> Router {
|
||||
Router::new()
|
||||
.route("/health", get(health))
|
||||
.route("/ping", get(health).post(health))
|
||||
.route("/v1/models", get(list_models))
|
||||
.route("/v1/chat/completions/render", post(render_chat))
|
||||
.route("/v1/completions/render", post(render_completion))
|
||||
.with_state(state)
|
||||
.layer(DefaultBodyLimit::max(DEFAULT_REQUEST_BODY_LIMIT_BYTES))
|
||||
}
|
||||
|
||||
async fn health() -> StatusCode {
|
||||
StatusCode::OK
|
||||
}
|
||||
|
||||
async fn list_models(State(state): State<Arc<RenderState>>) -> Json<ListModelsResponse> {
|
||||
let created = unix_timestamp() as i64;
|
||||
Json(ListModelsResponse {
|
||||
object: "list".to_string(),
|
||||
data: state
|
||||
.served_model_names
|
||||
.iter()
|
||||
.map(|model| ModelObject {
|
||||
id: model.clone(),
|
||||
object: "model".to_string(),
|
||||
created,
|
||||
owned_by: "vllm".to_string(),
|
||||
root: Some(state.model.clone()),
|
||||
parent: None,
|
||||
max_model_len: Some(state.text.max_model_len()),
|
||||
})
|
||||
.collect(),
|
||||
})
|
||||
}
|
||||
|
||||
fn model_resolution(state: &RenderState) -> LoraModelResolution {
|
||||
LoraModelResolution {
|
||||
model_names: state.served_model_names.clone(),
|
||||
lora_request: None,
|
||||
}
|
||||
}
|
||||
|
||||
fn lower_render_request(
|
||||
state: &RenderState,
|
||||
text_request: TextRequest,
|
||||
model: String,
|
||||
stream: bool,
|
||||
stream_options: Option<StreamOptions>,
|
||||
) -> Result<GenerateRequest, ApiError> {
|
||||
let prepared = state
|
||||
.text
|
||||
.prepare(text_request)
|
||||
.map_err(|error| text_submit_error("failed to prepare render request", error))?;
|
||||
let token_ids = prepared.generate_request.prompt_token_ids;
|
||||
let text_request = prepared.text_request;
|
||||
|
||||
let request = GenerateRequest {
|
||||
request_id: Some(text_request.request_id),
|
||||
model: Some(model),
|
||||
token_ids,
|
||||
sampling_params: text_request.sampling_params,
|
||||
stream,
|
||||
stream_options,
|
||||
cache_salt: text_request.cache_salt,
|
||||
priority: text_request.priority,
|
||||
kv_transfer_params: None,
|
||||
ec_transfer_params: None,
|
||||
other: Default::default(),
|
||||
};
|
||||
validate_generate_request(&request, &state.served_model_names)?;
|
||||
Ok(request)
|
||||
}
|
||||
|
||||
async fn render_chat(
|
||||
State(state): State<Arc<RenderState>>,
|
||||
headers: HeaderMap,
|
||||
ValidatedJson(body): ValidatedJson<ChatCompletionRequest>,
|
||||
) -> Result<Json<GenerateRequest>, ApiError> {
|
||||
let model = body.model.clone();
|
||||
let stream = body.stream;
|
||||
let stream_options = body.stream_options.clone();
|
||||
let request_context = resolve_request_context(&headers, body.request_id.as_deref());
|
||||
let chat_request = lower_chat_request(body, &model_resolution(&state), request_context)?;
|
||||
let (text_request, _) = state
|
||||
.chat
|
||||
.prepare(chat_request)
|
||||
.await
|
||||
.map_err(|error| ApiError::invalid_request(error.to_report_string(), None))?;
|
||||
Ok(Json(lower_render_request(
|
||||
&state,
|
||||
text_request,
|
||||
model,
|
||||
stream,
|
||||
stream_options,
|
||||
)?))
|
||||
}
|
||||
|
||||
async fn render_completion(
|
||||
State(state): State<Arc<RenderState>>,
|
||||
headers: HeaderMap,
|
||||
ValidatedJson(body): ValidatedJson<CompletionRequest>,
|
||||
) -> Result<Json<Vec<GenerateRequest>>, ApiError> {
|
||||
let model = body.model.clone();
|
||||
let stream = body.stream;
|
||||
let stream_options = body.stream_options.clone();
|
||||
let request_context = resolve_request_context(&headers, body.request_id.as_deref());
|
||||
let tokenizer = state.text.tokenizer();
|
||||
let text_request = lower_completion_request(
|
||||
body,
|
||||
&model_resolution(&state),
|
||||
request_context,
|
||||
tokenizer.as_ref(),
|
||||
)?;
|
||||
Ok(Json(vec![lower_render_request(
|
||||
&state,
|
||||
text_request,
|
||||
model,
|
||||
stream,
|
||||
stream_options,
|
||||
)?]))
|
||||
}
|
||||
@@ -23,8 +23,8 @@ use serial_test::serial;
|
||||
use tower::{Service as _, ServiceExt as _};
|
||||
use vllm_chat::{
|
||||
ChatBackend, ChatContent, ChatContentPart, ChatLlm, ChatMessage, ChatRenderer, ChatRequest,
|
||||
ChatTextBackend, DefaultChatOutputProcessor, DynChatOutputProcessor, DynChatRenderer,
|
||||
NewChatOutputProcessorOptions,
|
||||
ChatRequestProcessor, ChatTextBackend, DefaultChatOutputProcessor, DynChatOutputProcessor,
|
||||
DynChatRenderer, NewChatOutputProcessorOptions, ParserSelection,
|
||||
};
|
||||
use vllm_engine_core_client::mock_engine::default_ready_response;
|
||||
use vllm_engine_core_client::protocol::decode_value;
|
||||
@@ -46,13 +46,17 @@ use vllm_engine_core_client::{
|
||||
use vllm_llm::Llm;
|
||||
use vllm_metrics::METRICS;
|
||||
use vllm_text::tokenizer::DynTokenizer;
|
||||
use vllm_text::{Prompt, TextBackend};
|
||||
use vllm_text::{Prompt, TextBackend, TextRequestProcessor};
|
||||
use vllm_tokenizer::test_utils::TestTokenizer;
|
||||
use zeromq::prelude::{SocketRecv, SocketSend};
|
||||
use zeromq::{DealerSocket, PushSocket, ZmqMessage};
|
||||
|
||||
use super::{build_router, build_router_with_dev_mode, build_router_with_dev_mode_and_lora};
|
||||
use super::{
|
||||
build_router, build_router_with_dev_mode, build_router_with_dev_mode_and_lora,
|
||||
render::build_router as build_render_router,
|
||||
};
|
||||
use crate::config::{ApiServerOptions, CorsConfig};
|
||||
use crate::render::RenderState;
|
||||
use crate::state::AppState;
|
||||
|
||||
fn request_output(
|
||||
@@ -682,6 +686,24 @@ async fn test_app() -> axum::Router {
|
||||
test_app_with_dev_mode(false).await
|
||||
}
|
||||
|
||||
fn test_render_app() -> axum::Router {
|
||||
test_render_app_with_parser_selections(ParserSelection::Auto, ParserSelection::Auto)
|
||||
}
|
||||
|
||||
fn test_render_app_with_parser_selections(
|
||||
tool_call_parser: ParserSelection,
|
||||
reasoning_parser: ParserSelection,
|
||||
) -> axum::Router {
|
||||
let backend = Arc::new(FakeChatBackend::new());
|
||||
build_render_router(Arc::new(RenderState {
|
||||
model: "backend-model".to_string(),
|
||||
served_model_names: vec!["render-model".to_string()],
|
||||
text: TextRequestProcessor::new(backend.clone(), 128),
|
||||
chat: ChatRequestProcessor::render_only(backend)
|
||||
.with_parser_selections(tool_call_parser, reasoning_parser),
|
||||
}))
|
||||
}
|
||||
|
||||
async fn test_app_with_dev_mode(dev_mode_enabled: bool) -> axum::Router {
|
||||
let (chat, _engine_task) = test_models_with_engine_outputs_and_backend(
|
||||
b"engine-openai",
|
||||
@@ -1043,6 +1065,191 @@ fn metric_delta(
|
||||
- metric_value(rendered_before, metric, labels).unwrap_or(0.0)
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn render_chat_returns_generate_request_with_header_request_id() {
|
||||
let mut app = test_render_app();
|
||||
let response = app
|
||||
.call(
|
||||
Request::builder()
|
||||
.method("POST")
|
||||
.uri("/v1/chat/completions/render")
|
||||
.header("content-type", "application/json")
|
||||
.header("X-Request-Id", "header-req")
|
||||
.body(Body::from(
|
||||
json!({
|
||||
"request_id": "body-req",
|
||||
"model": "render-model",
|
||||
"messages": [{"role": "user", "content": "hello"}],
|
||||
"max_completion_tokens": 8,
|
||||
"stream": true,
|
||||
"stream_options": {"include_usage": true},
|
||||
"cache_salt": "render-cache-salt",
|
||||
"priority": -3
|
||||
})
|
||||
.to_string(),
|
||||
))
|
||||
.expect("build request"),
|
||||
)
|
||||
.await
|
||||
.expect("call app");
|
||||
|
||||
assert_eq!(response.status(), StatusCode::OK);
|
||||
let body = to_bytes(response.into_body(), usize::MAX).await.expect("read body");
|
||||
let json: serde_json::Value = serde_json::from_slice(&body).expect("decode json");
|
||||
|
||||
assert_eq!(json["request_id"], "chatcmpl-header-req");
|
||||
assert_eq!(json["model"], "render-model");
|
||||
assert_eq!(json["stream"], true);
|
||||
assert_eq!(json["stream_options"]["include_usage"], true);
|
||||
assert_eq!(json["cache_salt"], "render-cache-salt");
|
||||
assert_eq!(json["priority"], -3);
|
||||
assert_eq!(json["sampling_params"]["max_tokens"], 8);
|
||||
assert!(!json["token_ids"].as_array().unwrap().is_empty());
|
||||
assert!(json.get("prompt_token_ids").is_none());
|
||||
assert!(json.get("mm_features").is_none());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn render_chat_uses_processor_owned_parser_selections() {
|
||||
let mut app = test_render_app_with_parser_selections(
|
||||
ParserSelection::Auto,
|
||||
ParserSelection::Explicit("definitely_missing_reasoning_parser".to_string()),
|
||||
);
|
||||
let response = app
|
||||
.call(
|
||||
Request::builder()
|
||||
.method("POST")
|
||||
.uri("/v1/chat/completions/render")
|
||||
.header("content-type", "application/json")
|
||||
.body(Body::from(
|
||||
json!({
|
||||
"model": "render-model",
|
||||
"messages": [{"role": "user", "content": "hello"}],
|
||||
"max_completion_tokens": 8
|
||||
})
|
||||
.to_string(),
|
||||
))
|
||||
.expect("build request"),
|
||||
)
|
||||
.await
|
||||
.expect("call app");
|
||||
|
||||
assert_eq!(response.status(), StatusCode::BAD_REQUEST);
|
||||
let body = to_bytes(response.into_body(), usize::MAX).await.expect("read body");
|
||||
let body = String::from_utf8(body.to_vec()).expect("decode body");
|
||||
assert!(
|
||||
body.contains("reasoning parser `definitely_missing_reasoning_parser` is not registered")
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn render_completion_returns_generate_request_with_body_request_id() {
|
||||
let mut app = test_render_app();
|
||||
let response = app
|
||||
.call(
|
||||
Request::builder()
|
||||
.method("POST")
|
||||
.uri("/v1/completions/render")
|
||||
.header("content-type", "application/json")
|
||||
.body(Body::from(
|
||||
json!({
|
||||
"request_id": "body-req",
|
||||
"model": "render-model",
|
||||
"prompt": "hello",
|
||||
"max_tokens": 8
|
||||
})
|
||||
.to_string(),
|
||||
))
|
||||
.expect("build request"),
|
||||
)
|
||||
.await
|
||||
.expect("call app");
|
||||
|
||||
assert_eq!(response.status(), StatusCode::OK);
|
||||
let body = to_bytes(response.into_body(), usize::MAX).await.expect("read body");
|
||||
let json: serde_json::Value = serde_json::from_slice(&body).expect("decode json");
|
||||
|
||||
assert_eq!(json[0]["request_id"], "cmpl-body-req");
|
||||
assert_eq!(json[0]["model"], "render-model");
|
||||
assert_eq!(json[0]["stream"], false);
|
||||
assert_eq!(json[0]["sampling_params"]["max_tokens"], 8);
|
||||
assert!(!json[0]["token_ids"].as_array().unwrap().is_empty());
|
||||
assert!(json[0].get("stream_options").is_none());
|
||||
assert!(json[0].get("prompt_token_ids").is_none());
|
||||
assert!(json[0].get("mm_features").is_none());
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
#[serial]
|
||||
async fn render_chat_response_can_be_submitted_to_generate_unchanged() {
|
||||
let mut render_app = test_render_app();
|
||||
let render_response = render_app
|
||||
.call(
|
||||
Request::builder()
|
||||
.method("POST")
|
||||
.uri("/v1/chat/completions/render")
|
||||
.header("content-type", "application/json")
|
||||
.body(Body::from(
|
||||
json!({
|
||||
"request_id": "round-trip",
|
||||
"model": "render-model",
|
||||
"messages": [{"role": "user", "content": "hello"}],
|
||||
"max_completion_tokens": 8,
|
||||
"bad_words": ["blocked"],
|
||||
"vllm_xargs": {"custom": 1}
|
||||
})
|
||||
.to_string(),
|
||||
))
|
||||
.expect("build render request"),
|
||||
)
|
||||
.await
|
||||
.expect("call render app");
|
||||
|
||||
assert_eq!(render_response.status(), StatusCode::OK);
|
||||
let render_body = to_bytes(render_response.into_body(), usize::MAX)
|
||||
.await
|
||||
.expect("read render response");
|
||||
let render_json: serde_json::Value =
|
||||
serde_json::from_slice(&render_body).expect("decode render response");
|
||||
let token_ids: Vec<u32> = serde_json::from_value(render_json["token_ids"].clone())
|
||||
.expect("decode rendered token IDs");
|
||||
|
||||
assert_eq!(
|
||||
render_json["sampling_params"]["bad_words"],
|
||||
json!(["blocked"])
|
||||
);
|
||||
assert_eq!(render_json["sampling_params"]["vllm_xargs"]["custom"], 1);
|
||||
|
||||
let (chat, engine_task) = test_models_with_engine_outputs_and_backend_inner(
|
||||
b"engine-render-generate-round-trip",
|
||||
default_stream_output_specs(),
|
||||
Some(token_ids),
|
||||
Arc::new(FakeChatBackend::new()),
|
||||
)
|
||||
.await;
|
||||
let mut generate_app = build_router(Arc::new(AppState::new(
|
||||
vec!["render-model".to_string()],
|
||||
chat,
|
||||
)));
|
||||
let generate_response = generate_app
|
||||
.call(
|
||||
Request::builder()
|
||||
.method("POST")
|
||||
.uri("/inference/v1/generate")
|
||||
.header("content-type", "application/json")
|
||||
.body(Body::from(render_body))
|
||||
.expect("build generate request"),
|
||||
)
|
||||
.await
|
||||
.expect("call generate app");
|
||||
|
||||
assert_eq!(generate_response.status(), StatusCode::OK);
|
||||
let _ = to_bytes(generate_response.into_body(), usize::MAX)
|
||||
.await
|
||||
.expect("read generate response");
|
||||
engine_task.finish().await;
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
#[serial]
|
||||
async fn list_models_returns_configured_model() {
|
||||
|
||||
Reference in New Issue
Block a user