[Rust Frontend][gRPC] Add multimodal image inference (#50368)

Co-authored-by: Bugen Zhao <i@bugenzhao.com>
Signed-off-by: Connor Carpenter <connorc@nvidia.com>
Signed-off-by: Bugen Zhao <i@bugenzhao.com>
This commit is contained in:
Connor Carpenter
2026-08-04 09:34:31 +00:00
committed by GitHub
co-authored by Bugen Zhao
parent 0b1c151cbc
commit e98a8774cb
12 changed files with 553 additions and 69 deletions
+1
View File
@@ -5753,6 +5753,7 @@ dependencies = [
"tracing",
"tracing-futures",
"tracing-subscriber",
"url",
"uuid",
"validator",
"vllm-chat",
+25
View File
@@ -47,6 +47,9 @@ message GenerateRequest {
int32 priority = 12;
optional string session_id = 13;
// Multimodal inputs aligned with placeholder markers in token_ids.
repeated MediaItem media = 14;
}
message RandomSampling {
@@ -202,3 +205,25 @@ message CandidateTokenInfo {
message TokenIds {
repeated uint32 ids = 1;
}
// ======================================================================================
// Media
// ======================================================================================
enum Modality {
MODALITY_UNSPECIFIED = 0;
MODALITY_IMAGE = 1;
MODALITY_VIDEO = 2;
MODALITY_AUDIO = 3;
}
message MediaItem {
Modality modality = 1;
oneof source {
string url = 2; // http:// or https://
string data_uri = 3; // data:
bytes raw_bytes = 4;
}
string mime_type = 5;
string uuid = 6;
}
+29 -1
View File
@@ -21,6 +21,7 @@ pub use event::{
AssistantToolCall, ChatEvent,
};
use futures::{StreamExt, TryStreamExt as _};
pub use llm_multimodal::MediaContentPart;
pub use output::{
ChatOutputProcessor, DefaultChatOutputProcessor, DynChatOutputProcessor,
HarmonyChatOutputProcessor,
@@ -41,6 +42,7 @@ pub use request::{
ChatToolChoice, GenerationPromptMode, ReasoningEffort, SamplingParams,
};
pub use stream::{ChatEventStream, ChatEventStreamTrait, CollectedAssistantMessage};
pub use vllm_engine_core_client::protocol::multimodal::MmFeatures;
pub use vllm_llm::FinishReason;
mod backend;
@@ -55,7 +57,6 @@ mod stream;
use vllm_engine_core_client::EngineCoreClient;
use vllm_engine_core_client::protocol::dtype::ModelDtype;
use vllm_engine_core_client::protocol::multimodal::MmFeatures;
use vllm_engine_core_client::protocol::request::ReasoningParserKwargs;
use vllm_llm::Llm;
use vllm_text::{Prompt, TextLlm, TextRequest};
@@ -136,6 +137,24 @@ impl ChatRequestProcessor {
}
}
/// Prepare media for an already-tokenized request.
async fn prepare_media(
&self,
media: Vec<MediaContentPart>,
token_ids: &mut Vec<u32>,
) -> Result<Option<MmFeatures>> {
if media.is_empty() {
return Ok(None);
}
let info = self
.backend
.multimodal_model_info()
.ok_or(Error::UnsupportedMultimodalRenderer)?;
let model_dtype = self.model_dtype.ok_or(Error::UnsupportedMultimodalRenderer)?;
let features = info.prepare_multimodal(media, token_ids, model_dtype).await?;
Ok(Some(features))
}
/// Prepare one chat request without submitting it to an engine.
pub async fn prepare(
&self,
@@ -262,6 +281,15 @@ impl ChatLlm {
self.processor.backend.multimodal_model_info().is_some()
}
/// Prepare media for an already-tokenized request.
pub async fn prepare_media(
&self,
media: Vec<MediaContentPart>,
token_ids: &mut Vec<u32>,
) -> Result<Option<MmFeatures>> {
self.processor.prepare_media(media, token_ids).await
}
/// Effective tool-call parser name for this model, if parsing is enabled.
pub fn tool_call_parser_name(&self) -> Option<&str> {
match &self.tool_call_parser {
+1 -1
View File
@@ -696,7 +696,7 @@ impl MultimodalModelInfo {
/// `prompt_token_ids` is mutated in place because placeholder expansion
/// changes both the final prompt and the offsets recorded in
/// `PlaceholderRange`.
async fn prepare_multimodal(
pub(crate) async fn prepare_multimodal(
&self,
media_parts: Vec<MediaContentPart>,
prompt_token_ids: &mut Vec<u32>,
+1 -1
View File
@@ -154,7 +154,7 @@ pub struct SharedRuntimeArgs {
#[arg(long, value_parser = clap::value_parser!(i32).range(-1..), allow_negative_numbers = true)]
#[serde(default)]
pub max_logprobs: Option<i32>,
/// TCP port for the gRPC Generate service. When not set, no gRPC server is
/// TCP port for the gRPC Inference service. When not set, no gRPC server is
/// started.
#[arg(long)]
#[serde(default)]
+1
View File
@@ -42,6 +42,7 @@ tower-http.workspace = true
tracing.workspace = true
tracing-futures.workspace = true
tracing-subscriber.workspace = true
url.workspace = true
uuid.workspace = true
validator.workspace = true
vllm-chat.workspace = true
+1 -1
View File
@@ -207,7 +207,7 @@ pub struct Config {
/// When `true`, suppress periodic stats logging (throughput, queue depth,
/// cache usage).
pub disable_log_stats: bool,
/// TCP port for the gRPC Generate service. When `None`, no gRPC server is
/// TCP port for the gRPC Inference service. When `None`, no gRPC server is
/// started.
pub grpc_port: Option<u16>,
/// Maximum time to wait for active HTTP/gRPC requests to drain on shutdown.
+74
View File
@@ -5,7 +5,9 @@
//! request/response types.
use tonic::Status;
use url::Url;
use uuid::Uuid;
use vllm_chat::MediaContentPart;
use vllm_engine_core_client::protocol::output::StopReason;
use vllm_engine_core_client::protocol::structured_outputs::StructuredOutputsParams;
use vllm_text::{
@@ -15,6 +17,78 @@ use vllm_text::{
use super::pb;
pub fn media_parts_from_request(
media: Vec<pb::MediaItem>,
) -> Result<Vec<MediaContentPart>, Status> {
let mut parts = Vec::with_capacity(media.len());
for (index, item) in media.into_iter().enumerate() {
match item.modality() {
pb::Modality::Image => {}
pb::Modality::Unspecified => {
return Err(Status::invalid_argument(format!(
"media[{index}].modality is required"
)));
}
other => {
return Err(Status::unimplemented(format!(
"media[{index}].modality {other:?} is not supported by the gRPC service"
)));
}
}
let uuid = (!item.uuid.is_empty()).then_some(item.uuid);
let mime_type = (!item.mime_type.is_empty()).then_some(item.mime_type);
let part = match item.source {
Some(pb::media_item::Source::Url(url)) => {
validate_media_uri(index, "url", &url, &["http", "https"])?;
MediaContentPart::ImageUrl {
url,
detail: None,
uuid,
}
}
Some(pb::media_item::Source::DataUri(uri)) => {
validate_media_uri(index, "data_uri", &uri, &["data"])?;
MediaContentPart::ImageUrl {
url: uri,
detail: None,
uuid,
}
}
Some(pb::media_item::Source::RawBytes(bytes)) => MediaContentPart::ImageData {
data: bytes,
mime_type,
uuid,
detail: None,
},
None => {
return Err(Status::invalid_argument(format!(
"media[{index}].source is required"
)));
}
};
parts.push(part);
}
Ok(parts)
}
fn validate_media_uri(
index: usize,
field: &str,
value: &str,
allowed_schemes: &[&str],
) -> Result<(), Status> {
let uri = Url::parse(value).map_err(|_| {
Status::invalid_argument(format!("media[{index}].{field} is not a valid URI"))
})?;
if !allowed_schemes.contains(&uri.scheme()) {
return Err(Status::invalid_argument(format!(
"media[{index}].{field} must use the {} scheme",
allowed_schemes.join(" or ")
)));
}
Ok(())
}
// ========================================================================================
// Request conversion
// ========================================================================================
+189 -56
View File
@@ -3,14 +3,18 @@
use std::pin::Pin;
use std::sync::Arc;
use std::time::Instant;
use futures::{Stream, StreamExt as _};
use thiserror_ext::AsReport as _;
use tokio::sync::mpsc;
use tokio_stream::wrappers::ReceiverStream;
use tonic::{Request, Response, Status};
use tracing::info;
use vllm_text::{DecodedTextEvent, TextOutputStreamExt as _};
use tracing::{Span, info, info_span, warn};
use tracing_futures::Instrument as _;
use uuid::Uuid;
use vllm_llm::current_unix_timestamp_secs;
use vllm_text::{DecodedTextEvent, Prompt, TextOutputStreamExt as _, TextRequest};
use super::convert::{self, ResponseOpts};
use super::{InferenceServer, pb};
@@ -23,10 +27,92 @@ pub struct InferenceServiceImpl {
state: Arc<AppState>,
}
struct PreparedGrpcRequest {
text_request: TextRequest,
request_span: Span,
started_at: Instant,
}
impl InferenceServiceImpl {
pub fn new(state: Arc<AppState>) -> Self {
Self { state }
}
async fn prepare_request(
&self,
mut proto_request: pb::GenerateRequest,
stream: bool,
rpc: &'static str,
) -> Result<PreparedGrpcRequest, Status> {
let started_at = Instant::now();
let arrival_time = current_unix_timestamp_secs();
if proto_request.request_id.is_empty() {
proto_request.request_id = Uuid::new_v4().to_string();
}
let request_id = proto_request.request_id.clone();
let model = if proto_request.model.is_empty() {
self.state.primary_model_name().to_string()
} else {
proto_request.model.clone()
};
let media_count = proto_request.media.len();
let request_span = info_span!(
"grpc_inference",
%request_id,
rpc,
%model,
media_count,
);
info!(parent: &request_span, "gRPC inference request received");
let result = async {
let media = std::mem::take(&mut proto_request.media);
let mut text_request =
convert::to_text_request(proto_request, stream, self.state.served_model_names())?;
text_request.arrival_time = Some(arrival_time);
let media = convert::media_parts_from_request(media)?;
if !media.is_empty() {
let Prompt::TokenIds(mut token_ids) = text_request.prompt else {
return Err(Status::invalid_argument(
"multimodal gRPC requests must provide token_ids input",
));
};
let mm_features = self
.state
.chat
.prepare_media(media, &mut token_ids)
.await
.map_err(|error| Status::internal(error.to_report_string()))?;
text_request.prompt = Prompt::TokenIds(token_ids);
text_request.mm_features = mm_features;
}
Ok(text_request)
}
.instrument(request_span.clone())
.await;
let text_request = match result {
Ok(text_request) => text_request,
Err(status) => {
warn!(
parent: &request_span,
grpc_code = ?status.code(),
elapsed_ms = started_at.elapsed().as_millis() as u64,
error = %status.message(),
"gRPC inference request preparation failed"
);
return Err(status);
}
};
Ok(PreparedGrpcRequest {
text_request,
request_span,
started_at,
})
}
}
#[tonic::async_trait]
@@ -41,16 +127,31 @@ impl pb::inference_server::Inference for InferenceServiceImpl {
) -> Result<Response<pb::GenerateResponse>, Status> {
let proto_req = request.into_inner();
let response_opts = ResponseOpts::from_proto(proto_req.response.as_ref());
let text_request =
convert::to_text_request(proto_req, false, self.state.served_model_names())?;
let PreparedGrpcRequest {
text_request,
request_span,
started_at,
} = self.prepare_request(proto_req, false, "Generate").await?;
let request_id = text_request.request_id.clone();
info!(%request_id, "grpc generate (unary)");
let stream = self
.state
.chat
.text()
.generate(text_request)
.instrument(request_span.clone())
.await
.map_err(|error| log_text_error(&request_span, started_at, "submission", error))?;
let stream = self.state.chat.text().generate(text_request).await;
let stream = stream.map_err(text_error_to_status)?;
let collected = stream.collect_output().await.map_err(text_error_to_status)?;
let collected = stream
.collect_output()
.instrument(request_span.clone())
.await
.map_err(|error| log_text_error(&request_span, started_at, "collection", error))?;
info!(
parent: &request_span,
elapsed_ms = started_at.elapsed().as_millis() as u64,
"gRPC inference request completed"
);
// Build the single aggregated response.
let prompt_info = convert::to_prompt_info(
@@ -87,59 +188,73 @@ impl pb::inference_server::Inference for InferenceServiceImpl {
) -> Result<Response<Self::GenerateStreamStream>, Status> {
let proto_req = request.into_inner();
let response_opts = ResponseOpts::from_proto(proto_req.response.as_ref());
let text_request =
convert::to_text_request(proto_req, true, self.state.served_model_names())?;
let PreparedGrpcRequest {
text_request,
request_span,
started_at,
} = self.prepare_request(proto_req, true, "GenerateStream").await?;
let request_id = text_request.request_id.clone();
info!(%request_id, "grpc generate (stream)");
let stream = self.state.chat.text().generate(text_request).await;
let stream = stream.map_err(text_error_to_status)?;
let stream = self
.state
.chat
.text()
.generate(text_request)
.instrument(request_span.clone())
.await
.map_err(|error| log_text_error(&request_span, started_at, "submission", error))?;
let (tx, rx) = mpsc::channel(32);
tokio::spawn(async move {
futures::pin_mut!(stream);
while let Some(event) = stream.next().await {
let response = match event {
Err(e) => Err(text_error_to_status(e)),
Ok(DecodedTextEvent::Start {
prompt_token_ids,
prompt_logprobs,
}) => {
let prompt_info = convert::to_prompt_info(
&prompt_token_ids,
prompt_logprobs.as_ref(),
&response_opts,
);
Ok(pb::GenerateResponse {
prompt_info: Some(prompt_info),
outputs: None,
})
}
Ok(DecodedTextEvent::TextDelta {
delta,
token_ids,
logprobs,
finished,
}) => Ok(pb::GenerateResponse {
prompt_info: None,
outputs: Some(convert::to_sequence_output(
&delta,
&token_ids,
logprobs.as_ref(),
finished.as_ref(),
&response_opts,
)),
}),
};
let task_span = request_span.clone();
tokio::spawn(
async move {
futures::pin_mut!(stream);
while let Some(event) = stream.next().await {
let response = match event {
Err(error) => Err(log_text_error(&task_span, started_at, "stream", error)),
Ok(DecodedTextEvent::Start {
prompt_token_ids,
prompt_logprobs,
}) => {
let prompt_info = convert::to_prompt_info(
&prompt_token_ids,
prompt_logprobs.as_ref(),
&response_opts,
);
Ok(pb::GenerateResponse {
prompt_info: Some(prompt_info),
outputs: None,
})
}
Ok(DecodedTextEvent::TextDelta {
delta,
token_ids,
logprobs,
finished,
}) => Ok(pb::GenerateResponse {
prompt_info: None,
outputs: Some(convert::to_sequence_output(
&delta,
&token_ids,
logprobs.as_ref(),
finished.as_ref(),
&response_opts,
)),
}),
};
if tx.send(response).await.is_err() {
// Client disconnected.
break;
if tx.send(response).await.is_err() {
break;
}
}
info!(
parent: &task_span,
elapsed_ms = started_at.elapsed().as_millis() as u64,
"gRPC inference stream closed"
);
}
});
.instrument(request_span),
);
let response_stream = ReceiverStream::new(rx);
Ok(Response::new(Box::pin(response_stream)))
@@ -154,3 +269,21 @@ fn text_error_to_status(error: vllm_text::Error) -> Status {
Status::internal(message)
}
}
fn log_text_error(
request_span: &Span,
started_at: Instant,
phase: &'static str,
error: vllm_text::Error,
) -> Status {
let status = text_error_to_status(error);
warn!(
parent: request_span,
phase,
grpc_code = ?status.code(),
elapsed_ms = started_at.elapsed().as_millis() as u64,
error = %status.message(),
"gRPC inference request failed"
);
status
}
+226 -5
View File
@@ -1,6 +1,7 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright contributors to the vLLM project
use std::fs;
use std::future::Future;
use std::io;
use std::pin::Pin;
@@ -204,6 +205,73 @@ impl ChatRenderer for FakeTextBackend {
}
}
struct FakeMultimodalBackend {
model_info: vllm_chat::multimodal::MultimodalModelInfo,
}
impl TextBackend for FakeMultimodalBackend {
fn tokenizer(&self) -> DynTokenizer {
Arc::new(TestTokenizer::new().with_regular_token("<|image_pad|>", QWEN_IMAGE_TOKEN_ID))
}
fn model_id(&self) -> &str {
"test-model"
}
}
impl ChatBackend for FakeMultimodalBackend {
fn chat_renderer(&self) -> DynChatRenderer {
Arc::new(FakeTextBackend)
}
fn multimodal_model_info(&self) -> Option<&vllm_chat::multimodal::MultimodalModelInfo> {
Some(&self.model_info)
}
fn new_chat_output_processor(
&self,
request: &mut ChatRequest,
options: NewChatOutputProcessorOptions<'_>,
) -> vllm_chat::Result<DynChatOutputProcessor> {
Ok(Box::new(DefaultChatOutputProcessor::new(
request,
self.model_id(),
self.tokenizer(),
options.tool_call_parser,
options.reasoning_parser,
)?))
}
}
const QWEN_IMAGE_TOKEN_ID: u32 = 151655;
const TINY_PNG_DATA_URI: &str = "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNk+A8AAQUBAScY42YAAAAASUVORK5CYII=";
fn multimodal_backend() -> Arc<dyn ChatTextBackend> {
let config_path = std::env::temp_dir().join(format!(
"vllm-grpc-qwen-config-{}.json",
uuid::Uuid::new_v4()
));
fs::write(
&config_path,
r#"{"model_type":"qwen2_vl","image_token_id":151655}"#,
)
.expect("write qwen test config");
let model_info = vllm_chat::multimodal::MultimodalModelInfo::from_paths(
"qwen2-vl-test".to_string(),
Some("qwen2_vl".to_string()),
vllm_chat::multimodal::MultimodalConfigFiles {
config: Some(&config_path),
..Default::default()
},
Arc::new(TestTokenizer::new().with_regular_token("<|image_pad|>", QWEN_IMAGE_TOKEN_ID)),
Default::default(),
)
.expect("load multimodal info")
.expect("qwen multimodal info is registered");
let _ = fs::remove_file(config_path);
Arc::new(FakeMultimodalBackend { model_info })
}
/// Build the gRPC service + mock engine that serves a single request with the
/// given output specs. Shared by the plaintext and TLS server fixtures.
async fn setup_grpc_service(
@@ -215,6 +283,24 @@ async fn setup_grpc_service(
tokio::sync::watch::Receiver<bool>,
MockEngineTask,
) {
setup_grpc_service_with_backend(engine_id, output_specs, Arc::new(FakeTextBackend), |_| {})
.await
}
async fn setup_grpc_service_with_backend<F>(
engine_id: impl Into<EngineId>,
output_specs: Vec<(Vec<u32>, Option<EngineCoreFinishReason>)>,
backend: Arc<dyn ChatTextBackend>,
check_request: F,
) -> (
InferenceServer<InferenceServiceImpl>,
ControlServer<ControlServiceImpl>,
tokio::sync::watch::Receiver<bool>,
MockEngineTask,
)
where
F: FnOnce(&EngineCoreRequest) + Send + 'static,
{
let ipc = IpcNamespace::new().expect("create ipc namespace");
let handshake_address = ipc.handshake_endpoint();
let engine_id = engine_id.into();
@@ -227,6 +313,7 @@ async fn setup_grpc_service(
let add = recv_engine_message(dealer).await;
let request: EngineCoreRequest =
rmp_serde::from_slice(&add[1]).expect("decode request");
check_request(&request);
send_outputs(
push,
engine_outputs_for_request(&request.request_id, output_specs),
@@ -248,13 +335,11 @@ async fn setup_grpc_service(
.expect("connect client");
let engine_health = client.subscribe_health();
let chat = ChatLlm::from_shared_backend(
Llm::new(client),
Arc::new(FakeTextBackend) as Arc<dyn ChatTextBackend>,
);
let chat = ChatLlm::from_shared_backend(Llm::new(client), backend);
let state = Arc::new(AppState::new(vec!["test-model".to_string()], chat));
(
InferenceServer::new(InferenceServiceImpl::new(state.clone())),
InferenceServer::new(InferenceServiceImpl::new(state.clone()))
.max_decoding_message_size(crate::DEFAULT_REQUEST_BODY_LIMIT_BYTES),
ControlServer::new(ControlServiceImpl::new(state)),
engine_health,
engine_task,
@@ -560,6 +645,142 @@ async fn unary_generate_with_token_ids_prompt() {
server_task.abort();
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
#[serial]
async fn unary_generate_prepares_multimodal_input_for_engine_core() {
let (inference_service, control_service, engine_health, engine_task) =
setup_grpc_service_with_backend(
b"engine-grpc-multimodal",
default_stream_output_specs(),
multimodal_backend(),
|request| {
let token_ids = request.prompt_token_ids.as_ref().expect("prompt token ids");
let features = request.mm_features.as_ref().expect("multimodal features");
assert_eq!(features.len(), 1);
let feature = &features[0];
assert_eq!(feature.modality, "image");
assert_eq!(feature.identifier, "image-1");
assert_eq!(feature.mm_position.offset, 1);
assert!(feature.mm_position.length > 1);
assert_eq!(token_ids.len(), feature.mm_position.length + 2);
assert_eq!(token_ids[0], 11);
assert_eq!(token_ids.last(), Some(&12));
assert!(
token_ids[feature.mm_position.offset
..feature.mm_position.offset + feature.mm_position.length]
.iter()
.all(|token_id| *token_id == QWEN_IMAGE_TOKEN_ID)
);
},
)
.await;
let (channel, server_task) = start_grpc_test_server(
inference_service,
control_service,
engine_health,
tokio_util::sync::CancellationToken::new(),
)
.await;
let mut client = InferenceClient::new(channel);
client
.generate(pb::GenerateRequest {
request_id: "test-multimodal".to_string(),
model: "test-model".to_string(),
prompt: Some(pb::generate_request::Prompt::TokenIds(pb::TokenIds {
ids: vec![11, QWEN_IMAGE_TOKEN_ID, 12],
})),
media: vec![pb::MediaItem {
modality: pb::Modality::Image as i32,
source: Some(pb::media_item::Source::DataUri(
TINY_PNG_DATA_URI.to_string(),
)),
mime_type: String::new(),
uuid: "image-1".to_string(),
}],
stopping: Some(pb::StoppingCriteria {
max_new_tokens: 10,
..Default::default()
}),
..Default::default()
})
.await
.expect("multimodal generate");
engine_task.await.expect("mock engine task");
server_task.abort();
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
#[serial]
async fn streaming_generate_rejects_text_prompt_with_media() {
let (mut client, server_task, _engine_task) = grpc_test_server(
b"engine-grpc-stream-media-text",
default_stream_output_specs(),
)
.await;
let status = client
.generate_stream(pb::GenerateRequest {
request_id: "test-stream-media-text".to_string(),
model: "test-model".to_string(),
prompt: Some(pb::generate_request::Prompt::Text(
"describe this".to_string(),
)),
media: vec![pb::MediaItem {
modality: pb::Modality::Image as i32,
source: Some(pb::media_item::Source::DataUri(
TINY_PNG_DATA_URI.to_string(),
)),
mime_type: String::new(),
uuid: "image-1".to_string(),
}],
..Default::default()
})
.await
.expect_err("text prompts with media must be rejected");
assert_eq!(status.code(), tonic::Code::InvalidArgument);
assert_eq!(
status.message(),
"multimodal gRPC requests must provide token_ids input"
);
server_task.abort();
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
#[serial]
async fn unary_generate_accepts_request_larger_than_tonic_default() {
let (mut client, server_task, _engine_task) =
grpc_test_server(b"engine-grpc-large-media", default_stream_output_specs()).await;
let status = client
.generate(pb::GenerateRequest {
request_id: "test-large-media".to_string(),
model: "test-model".to_string(),
prompt: Some(pb::generate_request::Prompt::Text(
"describe this".to_string(),
)),
media: vec![pb::MediaItem {
modality: pb::Modality::Image as i32,
source: Some(pb::media_item::Source::RawBytes(vec![0; 5 * 1024 * 1024])),
mime_type: "image/png".to_string(),
uuid: "image-1".to_string(),
}],
..Default::default()
})
.await
.expect_err("text prompts with media must be rejected after decoding");
assert_eq!(status.code(), tonic::Code::InvalidArgument);
assert_eq!(
status.message(),
"multimodal gRPC requests must provide token_ids input"
);
server_task.abort();
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
#[serial]
async fn unary_generate_returns_token_ids_when_requested() {
+3 -1
View File
@@ -59,6 +59,7 @@ const GRPC_KEEPALIVE_INTERVAL: Duration = Duration::from_secs(7200);
/// How long the server waits for a keepalive PING reply before dropping the gRPC
/// connection. 20s matches the gRPC-core default.
const GRPC_KEEPALIVE_TIMEOUT: Duration = Duration::from_secs(20);
const DEFAULT_REQUEST_BODY_LIMIT_BYTES: usize = 32 * 1024 * 1024;
/// Resolve the public model names accepted by the frontend.
fn effective_served_model_names(model: &str, served_model_name: &[String]) -> Vec<String> {
@@ -215,7 +216,8 @@ where
let control_service =
grpc::ControlGrpcService::new(grpc::ControlServiceImpl::new(state.clone()));
let inference_service =
grpc::InferenceGrpcService::new(grpc::InferenceServiceImpl::new(state.clone()));
grpc::InferenceGrpcService::new(grpc::InferenceServiceImpl::new(state.clone()))
.max_decoding_message_size(DEFAULT_REQUEST_BODY_LIMIT_BYTES);
let svc = TonicServer::builder()
.http2_keepalive_interval(Some(GRPC_KEEPALIVE_INTERVAL))
.http2_keepalive_timeout(Some(GRPC_KEEPALIVE_TIMEOUT))
+2 -3
View File
@@ -27,11 +27,10 @@ use axum::routing::{get, post};
use tower_http::trace::TraceLayer;
use tracing::warn;
use crate::DEFAULT_REQUEST_BODY_LIMIT_BYTES;
use crate::middleware;
use crate::state::AppState;
const DEFAULT_JSON_BODY_LIMIT_BYTES: usize = 32 * 1024 * 1024;
fn server_dev_mode_enabled() -> bool {
std::env::var("VLLM_SERVER_DEV_MODE")
.ok()
@@ -127,7 +126,7 @@ fn build_router_with_options(
let enable_api_key_auth = state.has_api_keys();
let mut router = router
.with_state(state.clone())
.layer(DefaultBodyLimit::max(DEFAULT_JSON_BODY_LIMIT_BYTES))
.layer(DefaultBodyLimit::max(DEFAULT_REQUEST_BODY_LIMIT_BYTES))
.layer(middleware::request_runtime_layer(state.clone()))
.layer(from_fn_with_state(
state.clone(),