diff --git a/rust/Cargo.lock b/rust/Cargo.lock index 38f1c25775..3c75e01dc5 100644 --- a/rust/Cargo.lock +++ b/rust/Cargo.lock @@ -5753,6 +5753,7 @@ dependencies = [ "tracing", "tracing-futures", "tracing-subscriber", + "url", "uuid", "validator", "vllm-chat", diff --git a/rust/proto/inference.proto b/rust/proto/inference.proto index 575ba70a19..021d93a1f7 100644 --- a/rust/proto/inference.proto +++ b/rust/proto/inference.proto @@ -47,6 +47,9 @@ message GenerateRequest { int32 priority = 12; optional string session_id = 13; + + // Multimodal inputs aligned with placeholder markers in token_ids. + repeated MediaItem media = 14; } message RandomSampling { @@ -202,3 +205,25 @@ message CandidateTokenInfo { message TokenIds { repeated uint32 ids = 1; } + +// ====================================================================================== +// Media +// ====================================================================================== + +enum Modality { + MODALITY_UNSPECIFIED = 0; + MODALITY_IMAGE = 1; + MODALITY_VIDEO = 2; + MODALITY_AUDIO = 3; +} + +message MediaItem { + Modality modality = 1; + oneof source { + string url = 2; // http:// or https:// + string data_uri = 3; // data: + bytes raw_bytes = 4; + } + string mime_type = 5; + string uuid = 6; +} diff --git a/rust/src/chat/src/lib.rs b/rust/src/chat/src/lib.rs index eb40272604..a0c81b29d6 100644 --- a/rust/src/chat/src/lib.rs +++ b/rust/src/chat/src/lib.rs @@ -21,6 +21,7 @@ pub use event::{ AssistantToolCall, ChatEvent, }; use futures::{StreamExt, TryStreamExt as _}; +pub use llm_multimodal::MediaContentPart; pub use output::{ ChatOutputProcessor, DefaultChatOutputProcessor, DynChatOutputProcessor, HarmonyChatOutputProcessor, @@ -41,6 +42,7 @@ pub use request::{ ChatToolChoice, GenerationPromptMode, ReasoningEffort, SamplingParams, }; pub use stream::{ChatEventStream, ChatEventStreamTrait, CollectedAssistantMessage}; +pub use vllm_engine_core_client::protocol::multimodal::MmFeatures; pub use vllm_llm::FinishReason; mod backend; @@ -55,7 +57,6 @@ mod stream; use vllm_engine_core_client::EngineCoreClient; use vllm_engine_core_client::protocol::dtype::ModelDtype; -use vllm_engine_core_client::protocol::multimodal::MmFeatures; use vllm_engine_core_client::protocol::request::ReasoningParserKwargs; use vllm_llm::Llm; use vllm_text::{Prompt, TextLlm, TextRequest}; @@ -136,6 +137,24 @@ impl ChatRequestProcessor { } } + /// Prepare media for an already-tokenized request. + async fn prepare_media( + &self, + media: Vec, + token_ids: &mut Vec, + ) -> Result> { + if media.is_empty() { + return Ok(None); + } + let info = self + .backend + .multimodal_model_info() + .ok_or(Error::UnsupportedMultimodalRenderer)?; + let model_dtype = self.model_dtype.ok_or(Error::UnsupportedMultimodalRenderer)?; + let features = info.prepare_multimodal(media, token_ids, model_dtype).await?; + Ok(Some(features)) + } + /// Prepare one chat request without submitting it to an engine. pub async fn prepare( &self, @@ -262,6 +281,15 @@ impl ChatLlm { self.processor.backend.multimodal_model_info().is_some() } + /// Prepare media for an already-tokenized request. + pub async fn prepare_media( + &self, + media: Vec, + token_ids: &mut Vec, + ) -> Result> { + self.processor.prepare_media(media, token_ids).await + } + /// Effective tool-call parser name for this model, if parsing is enabled. pub fn tool_call_parser_name(&self) -> Option<&str> { match &self.tool_call_parser { diff --git a/rust/src/chat/src/multimodal.rs b/rust/src/chat/src/multimodal.rs index c0600a6329..edb4722d48 100644 --- a/rust/src/chat/src/multimodal.rs +++ b/rust/src/chat/src/multimodal.rs @@ -696,7 +696,7 @@ impl MultimodalModelInfo { /// `prompt_token_ids` is mutated in place because placeholder expansion /// changes both the final prompt and the offsets recorded in /// `PlaceholderRange`. - async fn prepare_multimodal( + pub(crate) async fn prepare_multimodal( &self, media_parts: Vec, prompt_token_ids: &mut Vec, diff --git a/rust/src/cmd/src/cli.rs b/rust/src/cmd/src/cli.rs index 1572e20467..d5377515cb 100644 --- a/rust/src/cmd/src/cli.rs +++ b/rust/src/cmd/src/cli.rs @@ -154,7 +154,7 @@ pub struct SharedRuntimeArgs { #[arg(long, value_parser = clap::value_parser!(i32).range(-1..), allow_negative_numbers = true)] #[serde(default)] pub max_logprobs: Option, - /// TCP port for the gRPC Generate service. When not set, no gRPC server is + /// TCP port for the gRPC Inference service. When not set, no gRPC server is /// started. #[arg(long)] #[serde(default)] diff --git a/rust/src/server/Cargo.toml b/rust/src/server/Cargo.toml index 1ed397d589..8b8f57a0da 100644 --- a/rust/src/server/Cargo.toml +++ b/rust/src/server/Cargo.toml @@ -42,6 +42,7 @@ tower-http.workspace = true tracing.workspace = true tracing-futures.workspace = true tracing-subscriber.workspace = true +url.workspace = true uuid.workspace = true validator.workspace = true vllm-chat.workspace = true diff --git a/rust/src/server/src/config.rs b/rust/src/server/src/config.rs index 1fb87f2478..76848b3caa 100644 --- a/rust/src/server/src/config.rs +++ b/rust/src/server/src/config.rs @@ -207,7 +207,7 @@ pub struct Config { /// When `true`, suppress periodic stats logging (throughput, queue depth, /// cache usage). pub disable_log_stats: bool, - /// TCP port for the gRPC Generate service. When `None`, no gRPC server is + /// TCP port for the gRPC Inference service. When `None`, no gRPC server is /// started. pub grpc_port: Option, /// Maximum time to wait for active HTTP/gRPC requests to drain on shutdown. diff --git a/rust/src/server/src/grpc/convert.rs b/rust/src/server/src/grpc/convert.rs index 178da515ee..6ac1abdcd8 100644 --- a/rust/src/server/src/grpc/convert.rs +++ b/rust/src/server/src/grpc/convert.rs @@ -5,7 +5,9 @@ //! request/response types. use tonic::Status; +use url::Url; use uuid::Uuid; +use vllm_chat::MediaContentPart; use vllm_engine_core_client::protocol::output::StopReason; use vllm_engine_core_client::protocol::structured_outputs::StructuredOutputsParams; use vllm_text::{ @@ -15,6 +17,78 @@ use vllm_text::{ use super::pb; +pub fn media_parts_from_request( + media: Vec, +) -> Result, Status> { + let mut parts = Vec::with_capacity(media.len()); + for (index, item) in media.into_iter().enumerate() { + match item.modality() { + pb::Modality::Image => {} + pb::Modality::Unspecified => { + return Err(Status::invalid_argument(format!( + "media[{index}].modality is required" + ))); + } + other => { + return Err(Status::unimplemented(format!( + "media[{index}].modality {other:?} is not supported by the gRPC service" + ))); + } + } + let uuid = (!item.uuid.is_empty()).then_some(item.uuid); + let mime_type = (!item.mime_type.is_empty()).then_some(item.mime_type); + let part = match item.source { + Some(pb::media_item::Source::Url(url)) => { + validate_media_uri(index, "url", &url, &["http", "https"])?; + MediaContentPart::ImageUrl { + url, + detail: None, + uuid, + } + } + Some(pb::media_item::Source::DataUri(uri)) => { + validate_media_uri(index, "data_uri", &uri, &["data"])?; + MediaContentPart::ImageUrl { + url: uri, + detail: None, + uuid, + } + } + Some(pb::media_item::Source::RawBytes(bytes)) => MediaContentPart::ImageData { + data: bytes, + mime_type, + uuid, + detail: None, + }, + None => { + return Err(Status::invalid_argument(format!( + "media[{index}].source is required" + ))); + } + }; + parts.push(part); + } + Ok(parts) +} + +fn validate_media_uri( + index: usize, + field: &str, + value: &str, + allowed_schemes: &[&str], +) -> Result<(), Status> { + let uri = Url::parse(value).map_err(|_| { + Status::invalid_argument(format!("media[{index}].{field} is not a valid URI")) + })?; + if !allowed_schemes.contains(&uri.scheme()) { + return Err(Status::invalid_argument(format!( + "media[{index}].{field} must use the {} scheme", + allowed_schemes.join(" or ") + ))); + } + Ok(()) +} + // ======================================================================================== // Request conversion // ======================================================================================== diff --git a/rust/src/server/src/grpc/inference.rs b/rust/src/server/src/grpc/inference.rs index 56fa40924c..3fc56fd0d9 100644 --- a/rust/src/server/src/grpc/inference.rs +++ b/rust/src/server/src/grpc/inference.rs @@ -3,14 +3,18 @@ use std::pin::Pin; use std::sync::Arc; +use std::time::Instant; use futures::{Stream, StreamExt as _}; use thiserror_ext::AsReport as _; use tokio::sync::mpsc; use tokio_stream::wrappers::ReceiverStream; use tonic::{Request, Response, Status}; -use tracing::info; -use vllm_text::{DecodedTextEvent, TextOutputStreamExt as _}; +use tracing::{Span, info, info_span, warn}; +use tracing_futures::Instrument as _; +use uuid::Uuid; +use vllm_llm::current_unix_timestamp_secs; +use vllm_text::{DecodedTextEvent, Prompt, TextOutputStreamExt as _, TextRequest}; use super::convert::{self, ResponseOpts}; use super::{InferenceServer, pb}; @@ -23,10 +27,92 @@ pub struct InferenceServiceImpl { state: Arc, } +struct PreparedGrpcRequest { + text_request: TextRequest, + request_span: Span, + started_at: Instant, +} + impl InferenceServiceImpl { pub fn new(state: Arc) -> Self { Self { state } } + + async fn prepare_request( + &self, + mut proto_request: pb::GenerateRequest, + stream: bool, + rpc: &'static str, + ) -> Result { + let started_at = Instant::now(); + let arrival_time = current_unix_timestamp_secs(); + if proto_request.request_id.is_empty() { + proto_request.request_id = Uuid::new_v4().to_string(); + } + let request_id = proto_request.request_id.clone(); + let model = if proto_request.model.is_empty() { + self.state.primary_model_name().to_string() + } else { + proto_request.model.clone() + }; + let media_count = proto_request.media.len(); + let request_span = info_span!( + "grpc_inference", + %request_id, + rpc, + %model, + media_count, + ); + info!(parent: &request_span, "gRPC inference request received"); + + let result = async { + let media = std::mem::take(&mut proto_request.media); + let mut text_request = + convert::to_text_request(proto_request, stream, self.state.served_model_names())?; + text_request.arrival_time = Some(arrival_time); + + let media = convert::media_parts_from_request(media)?; + if !media.is_empty() { + let Prompt::TokenIds(mut token_ids) = text_request.prompt else { + return Err(Status::invalid_argument( + "multimodal gRPC requests must provide token_ids input", + )); + }; + let mm_features = self + .state + .chat + .prepare_media(media, &mut token_ids) + .await + .map_err(|error| Status::internal(error.to_report_string()))?; + text_request.prompt = Prompt::TokenIds(token_ids); + text_request.mm_features = mm_features; + } + + Ok(text_request) + } + .instrument(request_span.clone()) + .await; + + let text_request = match result { + Ok(text_request) => text_request, + Err(status) => { + warn!( + parent: &request_span, + grpc_code = ?status.code(), + elapsed_ms = started_at.elapsed().as_millis() as u64, + error = %status.message(), + "gRPC inference request preparation failed" + ); + return Err(status); + } + }; + + Ok(PreparedGrpcRequest { + text_request, + request_span, + started_at, + }) + } } #[tonic::async_trait] @@ -41,16 +127,31 @@ impl pb::inference_server::Inference for InferenceServiceImpl { ) -> Result, Status> { let proto_req = request.into_inner(); let response_opts = ResponseOpts::from_proto(proto_req.response.as_ref()); - let text_request = - convert::to_text_request(proto_req, false, self.state.served_model_names())?; + let PreparedGrpcRequest { + text_request, + request_span, + started_at, + } = self.prepare_request(proto_req, false, "Generate").await?; - let request_id = text_request.request_id.clone(); - info!(%request_id, "grpc generate (unary)"); + let stream = self + .state + .chat + .text() + .generate(text_request) + .instrument(request_span.clone()) + .await + .map_err(|error| log_text_error(&request_span, started_at, "submission", error))?; - let stream = self.state.chat.text().generate(text_request).await; - let stream = stream.map_err(text_error_to_status)?; - - let collected = stream.collect_output().await.map_err(text_error_to_status)?; + let collected = stream + .collect_output() + .instrument(request_span.clone()) + .await + .map_err(|error| log_text_error(&request_span, started_at, "collection", error))?; + info!( + parent: &request_span, + elapsed_ms = started_at.elapsed().as_millis() as u64, + "gRPC inference request completed" + ); // Build the single aggregated response. let prompt_info = convert::to_prompt_info( @@ -87,59 +188,73 @@ impl pb::inference_server::Inference for InferenceServiceImpl { ) -> Result, Status> { let proto_req = request.into_inner(); let response_opts = ResponseOpts::from_proto(proto_req.response.as_ref()); - let text_request = - convert::to_text_request(proto_req, true, self.state.served_model_names())?; + let PreparedGrpcRequest { + text_request, + request_span, + started_at, + } = self.prepare_request(proto_req, true, "GenerateStream").await?; - let request_id = text_request.request_id.clone(); - info!(%request_id, "grpc generate (stream)"); - - let stream = self.state.chat.text().generate(text_request).await; - let stream = stream.map_err(text_error_to_status)?; + let stream = self + .state + .chat + .text() + .generate(text_request) + .instrument(request_span.clone()) + .await + .map_err(|error| log_text_error(&request_span, started_at, "submission", error))?; let (tx, rx) = mpsc::channel(32); - tokio::spawn(async move { - futures::pin_mut!(stream); - while let Some(event) = stream.next().await { - let response = match event { - Err(e) => Err(text_error_to_status(e)), - Ok(DecodedTextEvent::Start { - prompt_token_ids, - prompt_logprobs, - }) => { - let prompt_info = convert::to_prompt_info( - &prompt_token_ids, - prompt_logprobs.as_ref(), - &response_opts, - ); - Ok(pb::GenerateResponse { - prompt_info: Some(prompt_info), - outputs: None, - }) - } - Ok(DecodedTextEvent::TextDelta { - delta, - token_ids, - logprobs, - finished, - }) => Ok(pb::GenerateResponse { - prompt_info: None, - outputs: Some(convert::to_sequence_output( - &delta, - &token_ids, - logprobs.as_ref(), - finished.as_ref(), - &response_opts, - )), - }), - }; + let task_span = request_span.clone(); + tokio::spawn( + async move { + futures::pin_mut!(stream); + while let Some(event) = stream.next().await { + let response = match event { + Err(error) => Err(log_text_error(&task_span, started_at, "stream", error)), + Ok(DecodedTextEvent::Start { + prompt_token_ids, + prompt_logprobs, + }) => { + let prompt_info = convert::to_prompt_info( + &prompt_token_ids, + prompt_logprobs.as_ref(), + &response_opts, + ); + Ok(pb::GenerateResponse { + prompt_info: Some(prompt_info), + outputs: None, + }) + } + Ok(DecodedTextEvent::TextDelta { + delta, + token_ids, + logprobs, + finished, + }) => Ok(pb::GenerateResponse { + prompt_info: None, + outputs: Some(convert::to_sequence_output( + &delta, + &token_ids, + logprobs.as_ref(), + finished.as_ref(), + &response_opts, + )), + }), + }; - if tx.send(response).await.is_err() { - // Client disconnected. - break; + if tx.send(response).await.is_err() { + break; + } } + info!( + parent: &task_span, + elapsed_ms = started_at.elapsed().as_millis() as u64, + "gRPC inference stream closed" + ); } - }); + .instrument(request_span), + ); let response_stream = ReceiverStream::new(rx); Ok(Response::new(Box::pin(response_stream))) @@ -154,3 +269,21 @@ fn text_error_to_status(error: vllm_text::Error) -> Status { Status::internal(message) } } + +fn log_text_error( + request_span: &Span, + started_at: Instant, + phase: &'static str, + error: vllm_text::Error, +) -> Status { + let status = text_error_to_status(error); + warn!( + parent: request_span, + phase, + grpc_code = ?status.code(), + elapsed_ms = started_at.elapsed().as_millis() as u64, + error = %status.message(), + "gRPC inference request failed" + ); + status +} diff --git a/rust/src/server/src/grpc/tests.rs b/rust/src/server/src/grpc/tests.rs index d2bfc7249c..fdc98333f1 100644 --- a/rust/src/server/src/grpc/tests.rs +++ b/rust/src/server/src/grpc/tests.rs @@ -1,6 +1,7 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright contributors to the vLLM project +use std::fs; use std::future::Future; use std::io; use std::pin::Pin; @@ -204,6 +205,73 @@ impl ChatRenderer for FakeTextBackend { } } +struct FakeMultimodalBackend { + model_info: vllm_chat::multimodal::MultimodalModelInfo, +} + +impl TextBackend for FakeMultimodalBackend { + fn tokenizer(&self) -> DynTokenizer { + Arc::new(TestTokenizer::new().with_regular_token("<|image_pad|>", QWEN_IMAGE_TOKEN_ID)) + } + + fn model_id(&self) -> &str { + "test-model" + } +} + +impl ChatBackend for FakeMultimodalBackend { + fn chat_renderer(&self) -> DynChatRenderer { + Arc::new(FakeTextBackend) + } + + fn multimodal_model_info(&self) -> Option<&vllm_chat::multimodal::MultimodalModelInfo> { + Some(&self.model_info) + } + + fn new_chat_output_processor( + &self, + request: &mut ChatRequest, + options: NewChatOutputProcessorOptions<'_>, + ) -> vllm_chat::Result { + Ok(Box::new(DefaultChatOutputProcessor::new( + request, + self.model_id(), + self.tokenizer(), + options.tool_call_parser, + options.reasoning_parser, + )?)) + } +} + +const QWEN_IMAGE_TOKEN_ID: u32 = 151655; +const TINY_PNG_DATA_URI: &str = "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNk+A8AAQUBAScY42YAAAAASUVORK5CYII="; + +fn multimodal_backend() -> Arc { + let config_path = std::env::temp_dir().join(format!( + "vllm-grpc-qwen-config-{}.json", + uuid::Uuid::new_v4() + )); + fs::write( + &config_path, + r#"{"model_type":"qwen2_vl","image_token_id":151655}"#, + ) + .expect("write qwen test config"); + let model_info = vllm_chat::multimodal::MultimodalModelInfo::from_paths( + "qwen2-vl-test".to_string(), + Some("qwen2_vl".to_string()), + vllm_chat::multimodal::MultimodalConfigFiles { + config: Some(&config_path), + ..Default::default() + }, + Arc::new(TestTokenizer::new().with_regular_token("<|image_pad|>", QWEN_IMAGE_TOKEN_ID)), + Default::default(), + ) + .expect("load multimodal info") + .expect("qwen multimodal info is registered"); + let _ = fs::remove_file(config_path); + Arc::new(FakeMultimodalBackend { model_info }) +} + /// Build the gRPC service + mock engine that serves a single request with the /// given output specs. Shared by the plaintext and TLS server fixtures. async fn setup_grpc_service( @@ -215,6 +283,24 @@ async fn setup_grpc_service( tokio::sync::watch::Receiver, MockEngineTask, ) { + setup_grpc_service_with_backend(engine_id, output_specs, Arc::new(FakeTextBackend), |_| {}) + .await +} + +async fn setup_grpc_service_with_backend( + engine_id: impl Into, + output_specs: Vec<(Vec, Option)>, + backend: Arc, + check_request: F, +) -> ( + InferenceServer, + ControlServer, + tokio::sync::watch::Receiver, + MockEngineTask, +) +where + F: FnOnce(&EngineCoreRequest) + Send + 'static, +{ let ipc = IpcNamespace::new().expect("create ipc namespace"); let handshake_address = ipc.handshake_endpoint(); let engine_id = engine_id.into(); @@ -227,6 +313,7 @@ async fn setup_grpc_service( let add = recv_engine_message(dealer).await; let request: EngineCoreRequest = rmp_serde::from_slice(&add[1]).expect("decode request"); + check_request(&request); send_outputs( push, engine_outputs_for_request(&request.request_id, output_specs), @@ -248,13 +335,11 @@ async fn setup_grpc_service( .expect("connect client"); let engine_health = client.subscribe_health(); - let chat = ChatLlm::from_shared_backend( - Llm::new(client), - Arc::new(FakeTextBackend) as Arc, - ); + let chat = ChatLlm::from_shared_backend(Llm::new(client), backend); let state = Arc::new(AppState::new(vec!["test-model".to_string()], chat)); ( - InferenceServer::new(InferenceServiceImpl::new(state.clone())), + InferenceServer::new(InferenceServiceImpl::new(state.clone())) + .max_decoding_message_size(crate::DEFAULT_REQUEST_BODY_LIMIT_BYTES), ControlServer::new(ControlServiceImpl::new(state)), engine_health, engine_task, @@ -560,6 +645,142 @@ async fn unary_generate_with_token_ids_prompt() { server_task.abort(); } +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[serial] +async fn unary_generate_prepares_multimodal_input_for_engine_core() { + let (inference_service, control_service, engine_health, engine_task) = + setup_grpc_service_with_backend( + b"engine-grpc-multimodal", + default_stream_output_specs(), + multimodal_backend(), + |request| { + let token_ids = request.prompt_token_ids.as_ref().expect("prompt token ids"); + let features = request.mm_features.as_ref().expect("multimodal features"); + assert_eq!(features.len(), 1); + + let feature = &features[0]; + assert_eq!(feature.modality, "image"); + assert_eq!(feature.identifier, "image-1"); + assert_eq!(feature.mm_position.offset, 1); + assert!(feature.mm_position.length > 1); + assert_eq!(token_ids.len(), feature.mm_position.length + 2); + assert_eq!(token_ids[0], 11); + assert_eq!(token_ids.last(), Some(&12)); + assert!( + token_ids[feature.mm_position.offset + ..feature.mm_position.offset + feature.mm_position.length] + .iter() + .all(|token_id| *token_id == QWEN_IMAGE_TOKEN_ID) + ); + }, + ) + .await; + let (channel, server_task) = start_grpc_test_server( + inference_service, + control_service, + engine_health, + tokio_util::sync::CancellationToken::new(), + ) + .await; + let mut client = InferenceClient::new(channel); + + client + .generate(pb::GenerateRequest { + request_id: "test-multimodal".to_string(), + model: "test-model".to_string(), + prompt: Some(pb::generate_request::Prompt::TokenIds(pb::TokenIds { + ids: vec![11, QWEN_IMAGE_TOKEN_ID, 12], + })), + media: vec![pb::MediaItem { + modality: pb::Modality::Image as i32, + source: Some(pb::media_item::Source::DataUri( + TINY_PNG_DATA_URI.to_string(), + )), + mime_type: String::new(), + uuid: "image-1".to_string(), + }], + stopping: Some(pb::StoppingCriteria { + max_new_tokens: 10, + ..Default::default() + }), + ..Default::default() + }) + .await + .expect("multimodal generate"); + + engine_task.await.expect("mock engine task"); + server_task.abort(); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[serial] +async fn streaming_generate_rejects_text_prompt_with_media() { + let (mut client, server_task, _engine_task) = grpc_test_server( + b"engine-grpc-stream-media-text", + default_stream_output_specs(), + ) + .await; + + let status = client + .generate_stream(pb::GenerateRequest { + request_id: "test-stream-media-text".to_string(), + model: "test-model".to_string(), + prompt: Some(pb::generate_request::Prompt::Text( + "describe this".to_string(), + )), + media: vec![pb::MediaItem { + modality: pb::Modality::Image as i32, + source: Some(pb::media_item::Source::DataUri( + TINY_PNG_DATA_URI.to_string(), + )), + mime_type: String::new(), + uuid: "image-1".to_string(), + }], + ..Default::default() + }) + .await + .expect_err("text prompts with media must be rejected"); + + assert_eq!(status.code(), tonic::Code::InvalidArgument); + assert_eq!( + status.message(), + "multimodal gRPC requests must provide token_ids input" + ); + server_task.abort(); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[serial] +async fn unary_generate_accepts_request_larger_than_tonic_default() { + let (mut client, server_task, _engine_task) = + grpc_test_server(b"engine-grpc-large-media", default_stream_output_specs()).await; + + let status = client + .generate(pb::GenerateRequest { + request_id: "test-large-media".to_string(), + model: "test-model".to_string(), + prompt: Some(pb::generate_request::Prompt::Text( + "describe this".to_string(), + )), + media: vec![pb::MediaItem { + modality: pb::Modality::Image as i32, + source: Some(pb::media_item::Source::RawBytes(vec![0; 5 * 1024 * 1024])), + mime_type: "image/png".to_string(), + uuid: "image-1".to_string(), + }], + ..Default::default() + }) + .await + .expect_err("text prompts with media must be rejected after decoding"); + + assert_eq!(status.code(), tonic::Code::InvalidArgument); + assert_eq!( + status.message(), + "multimodal gRPC requests must provide token_ids input" + ); + server_task.abort(); +} + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] #[serial] async fn unary_generate_returns_token_ids_when_requested() { diff --git a/rust/src/server/src/lib.rs b/rust/src/server/src/lib.rs index 224ec672ae..bff47d5b1a 100644 --- a/rust/src/server/src/lib.rs +++ b/rust/src/server/src/lib.rs @@ -59,6 +59,7 @@ const GRPC_KEEPALIVE_INTERVAL: Duration = Duration::from_secs(7200); /// How long the server waits for a keepalive PING reply before dropping the gRPC /// connection. 20s matches the gRPC-core default. const GRPC_KEEPALIVE_TIMEOUT: Duration = Duration::from_secs(20); +const DEFAULT_REQUEST_BODY_LIMIT_BYTES: usize = 32 * 1024 * 1024; /// Resolve the public model names accepted by the frontend. fn effective_served_model_names(model: &str, served_model_name: &[String]) -> Vec { @@ -215,7 +216,8 @@ where let control_service = grpc::ControlGrpcService::new(grpc::ControlServiceImpl::new(state.clone())); let inference_service = - grpc::InferenceGrpcService::new(grpc::InferenceServiceImpl::new(state.clone())); + grpc::InferenceGrpcService::new(grpc::InferenceServiceImpl::new(state.clone())) + .max_decoding_message_size(DEFAULT_REQUEST_BODY_LIMIT_BYTES); let svc = TonicServer::builder() .http2_keepalive_interval(Some(GRPC_KEEPALIVE_INTERVAL)) .http2_keepalive_timeout(Some(GRPC_KEEPALIVE_TIMEOUT)) diff --git a/rust/src/server/src/routes.rs b/rust/src/server/src/routes.rs index b9157f6ec9..2dec252875 100644 --- a/rust/src/server/src/routes.rs +++ b/rust/src/server/src/routes.rs @@ -27,11 +27,10 @@ use axum::routing::{get, post}; use tower_http::trace::TraceLayer; use tracing::warn; +use crate::DEFAULT_REQUEST_BODY_LIMIT_BYTES; use crate::middleware; use crate::state::AppState; -const DEFAULT_JSON_BODY_LIMIT_BYTES: usize = 32 * 1024 * 1024; - fn server_dev_mode_enabled() -> bool { std::env::var("VLLM_SERVER_DEV_MODE") .ok() @@ -127,7 +126,7 @@ fn build_router_with_options( let enable_api_key_auth = state.has_api_keys(); let mut router = router .with_state(state.clone()) - .layer(DefaultBodyLimit::max(DEFAULT_JSON_BODY_LIMIT_BYTES)) + .layer(DefaultBodyLimit::max(DEFAULT_REQUEST_BODY_LIMIT_BYTES)) .layer(middleware::request_runtime_layer(state.clone())) .layer(from_fn_with_state( state.clone(),