[Rust Frontend][gRPC] Add multimodal image inference (#50368)
Co-authored-by: Bugen Zhao <i@bugenzhao.com> Signed-off-by: Connor Carpenter <connorc@nvidia.com> Signed-off-by: Bugen Zhao <i@bugenzhao.com>
This commit is contained in:
co-authored by
Bugen Zhao
parent
0b1c151cbc
commit
e98a8774cb
@@ -42,6 +42,7 @@ tower-http.workspace = true
|
||||
tracing.workspace = true
|
||||
tracing-futures.workspace = true
|
||||
tracing-subscriber.workspace = true
|
||||
url.workspace = true
|
||||
uuid.workspace = true
|
||||
validator.workspace = true
|
||||
vllm-chat.workspace = true
|
||||
|
||||
@@ -207,7 +207,7 @@ pub struct Config {
|
||||
/// When `true`, suppress periodic stats logging (throughput, queue depth,
|
||||
/// cache usage).
|
||||
pub disable_log_stats: bool,
|
||||
/// TCP port for the gRPC Generate service. When `None`, no gRPC server is
|
||||
/// TCP port for the gRPC Inference service. When `None`, no gRPC server is
|
||||
/// started.
|
||||
pub grpc_port: Option<u16>,
|
||||
/// Maximum time to wait for active HTTP/gRPC requests to drain on shutdown.
|
||||
|
||||
@@ -5,7 +5,9 @@
|
||||
//! request/response types.
|
||||
|
||||
use tonic::Status;
|
||||
use url::Url;
|
||||
use uuid::Uuid;
|
||||
use vllm_chat::MediaContentPart;
|
||||
use vllm_engine_core_client::protocol::output::StopReason;
|
||||
use vllm_engine_core_client::protocol::structured_outputs::StructuredOutputsParams;
|
||||
use vllm_text::{
|
||||
@@ -15,6 +17,78 @@ use vllm_text::{
|
||||
|
||||
use super::pb;
|
||||
|
||||
pub fn media_parts_from_request(
|
||||
media: Vec<pb::MediaItem>,
|
||||
) -> Result<Vec<MediaContentPart>, Status> {
|
||||
let mut parts = Vec::with_capacity(media.len());
|
||||
for (index, item) in media.into_iter().enumerate() {
|
||||
match item.modality() {
|
||||
pb::Modality::Image => {}
|
||||
pb::Modality::Unspecified => {
|
||||
return Err(Status::invalid_argument(format!(
|
||||
"media[{index}].modality is required"
|
||||
)));
|
||||
}
|
||||
other => {
|
||||
return Err(Status::unimplemented(format!(
|
||||
"media[{index}].modality {other:?} is not supported by the gRPC service"
|
||||
)));
|
||||
}
|
||||
}
|
||||
let uuid = (!item.uuid.is_empty()).then_some(item.uuid);
|
||||
let mime_type = (!item.mime_type.is_empty()).then_some(item.mime_type);
|
||||
let part = match item.source {
|
||||
Some(pb::media_item::Source::Url(url)) => {
|
||||
validate_media_uri(index, "url", &url, &["http", "https"])?;
|
||||
MediaContentPart::ImageUrl {
|
||||
url,
|
||||
detail: None,
|
||||
uuid,
|
||||
}
|
||||
}
|
||||
Some(pb::media_item::Source::DataUri(uri)) => {
|
||||
validate_media_uri(index, "data_uri", &uri, &["data"])?;
|
||||
MediaContentPart::ImageUrl {
|
||||
url: uri,
|
||||
detail: None,
|
||||
uuid,
|
||||
}
|
||||
}
|
||||
Some(pb::media_item::Source::RawBytes(bytes)) => MediaContentPart::ImageData {
|
||||
data: bytes,
|
||||
mime_type,
|
||||
uuid,
|
||||
detail: None,
|
||||
},
|
||||
None => {
|
||||
return Err(Status::invalid_argument(format!(
|
||||
"media[{index}].source is required"
|
||||
)));
|
||||
}
|
||||
};
|
||||
parts.push(part);
|
||||
}
|
||||
Ok(parts)
|
||||
}
|
||||
|
||||
fn validate_media_uri(
|
||||
index: usize,
|
||||
field: &str,
|
||||
value: &str,
|
||||
allowed_schemes: &[&str],
|
||||
) -> Result<(), Status> {
|
||||
let uri = Url::parse(value).map_err(|_| {
|
||||
Status::invalid_argument(format!("media[{index}].{field} is not a valid URI"))
|
||||
})?;
|
||||
if !allowed_schemes.contains(&uri.scheme()) {
|
||||
return Err(Status::invalid_argument(format!(
|
||||
"media[{index}].{field} must use the {} scheme",
|
||||
allowed_schemes.join(" or ")
|
||||
)));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
// ========================================================================================
|
||||
// Request conversion
|
||||
// ========================================================================================
|
||||
|
||||
@@ -3,14 +3,18 @@
|
||||
|
||||
use std::pin::Pin;
|
||||
use std::sync::Arc;
|
||||
use std::time::Instant;
|
||||
|
||||
use futures::{Stream, StreamExt as _};
|
||||
use thiserror_ext::AsReport as _;
|
||||
use tokio::sync::mpsc;
|
||||
use tokio_stream::wrappers::ReceiverStream;
|
||||
use tonic::{Request, Response, Status};
|
||||
use tracing::info;
|
||||
use vllm_text::{DecodedTextEvent, TextOutputStreamExt as _};
|
||||
use tracing::{Span, info, info_span, warn};
|
||||
use tracing_futures::Instrument as _;
|
||||
use uuid::Uuid;
|
||||
use vllm_llm::current_unix_timestamp_secs;
|
||||
use vllm_text::{DecodedTextEvent, Prompt, TextOutputStreamExt as _, TextRequest};
|
||||
|
||||
use super::convert::{self, ResponseOpts};
|
||||
use super::{InferenceServer, pb};
|
||||
@@ -23,10 +27,92 @@ pub struct InferenceServiceImpl {
|
||||
state: Arc<AppState>,
|
||||
}
|
||||
|
||||
struct PreparedGrpcRequest {
|
||||
text_request: TextRequest,
|
||||
request_span: Span,
|
||||
started_at: Instant,
|
||||
}
|
||||
|
||||
impl InferenceServiceImpl {
|
||||
pub fn new(state: Arc<AppState>) -> Self {
|
||||
Self { state }
|
||||
}
|
||||
|
||||
async fn prepare_request(
|
||||
&self,
|
||||
mut proto_request: pb::GenerateRequest,
|
||||
stream: bool,
|
||||
rpc: &'static str,
|
||||
) -> Result<PreparedGrpcRequest, Status> {
|
||||
let started_at = Instant::now();
|
||||
let arrival_time = current_unix_timestamp_secs();
|
||||
if proto_request.request_id.is_empty() {
|
||||
proto_request.request_id = Uuid::new_v4().to_string();
|
||||
}
|
||||
let request_id = proto_request.request_id.clone();
|
||||
let model = if proto_request.model.is_empty() {
|
||||
self.state.primary_model_name().to_string()
|
||||
} else {
|
||||
proto_request.model.clone()
|
||||
};
|
||||
let media_count = proto_request.media.len();
|
||||
let request_span = info_span!(
|
||||
"grpc_inference",
|
||||
%request_id,
|
||||
rpc,
|
||||
%model,
|
||||
media_count,
|
||||
);
|
||||
info!(parent: &request_span, "gRPC inference request received");
|
||||
|
||||
let result = async {
|
||||
let media = std::mem::take(&mut proto_request.media);
|
||||
let mut text_request =
|
||||
convert::to_text_request(proto_request, stream, self.state.served_model_names())?;
|
||||
text_request.arrival_time = Some(arrival_time);
|
||||
|
||||
let media = convert::media_parts_from_request(media)?;
|
||||
if !media.is_empty() {
|
||||
let Prompt::TokenIds(mut token_ids) = text_request.prompt else {
|
||||
return Err(Status::invalid_argument(
|
||||
"multimodal gRPC requests must provide token_ids input",
|
||||
));
|
||||
};
|
||||
let mm_features = self
|
||||
.state
|
||||
.chat
|
||||
.prepare_media(media, &mut token_ids)
|
||||
.await
|
||||
.map_err(|error| Status::internal(error.to_report_string()))?;
|
||||
text_request.prompt = Prompt::TokenIds(token_ids);
|
||||
text_request.mm_features = mm_features;
|
||||
}
|
||||
|
||||
Ok(text_request)
|
||||
}
|
||||
.instrument(request_span.clone())
|
||||
.await;
|
||||
|
||||
let text_request = match result {
|
||||
Ok(text_request) => text_request,
|
||||
Err(status) => {
|
||||
warn!(
|
||||
parent: &request_span,
|
||||
grpc_code = ?status.code(),
|
||||
elapsed_ms = started_at.elapsed().as_millis() as u64,
|
||||
error = %status.message(),
|
||||
"gRPC inference request preparation failed"
|
||||
);
|
||||
return Err(status);
|
||||
}
|
||||
};
|
||||
|
||||
Ok(PreparedGrpcRequest {
|
||||
text_request,
|
||||
request_span,
|
||||
started_at,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
#[tonic::async_trait]
|
||||
@@ -41,16 +127,31 @@ impl pb::inference_server::Inference for InferenceServiceImpl {
|
||||
) -> Result<Response<pb::GenerateResponse>, Status> {
|
||||
let proto_req = request.into_inner();
|
||||
let response_opts = ResponseOpts::from_proto(proto_req.response.as_ref());
|
||||
let text_request =
|
||||
convert::to_text_request(proto_req, false, self.state.served_model_names())?;
|
||||
let PreparedGrpcRequest {
|
||||
text_request,
|
||||
request_span,
|
||||
started_at,
|
||||
} = self.prepare_request(proto_req, false, "Generate").await?;
|
||||
|
||||
let request_id = text_request.request_id.clone();
|
||||
info!(%request_id, "grpc generate (unary)");
|
||||
let stream = self
|
||||
.state
|
||||
.chat
|
||||
.text()
|
||||
.generate(text_request)
|
||||
.instrument(request_span.clone())
|
||||
.await
|
||||
.map_err(|error| log_text_error(&request_span, started_at, "submission", error))?;
|
||||
|
||||
let stream = self.state.chat.text().generate(text_request).await;
|
||||
let stream = stream.map_err(text_error_to_status)?;
|
||||
|
||||
let collected = stream.collect_output().await.map_err(text_error_to_status)?;
|
||||
let collected = stream
|
||||
.collect_output()
|
||||
.instrument(request_span.clone())
|
||||
.await
|
||||
.map_err(|error| log_text_error(&request_span, started_at, "collection", error))?;
|
||||
info!(
|
||||
parent: &request_span,
|
||||
elapsed_ms = started_at.elapsed().as_millis() as u64,
|
||||
"gRPC inference request completed"
|
||||
);
|
||||
|
||||
// Build the single aggregated response.
|
||||
let prompt_info = convert::to_prompt_info(
|
||||
@@ -87,59 +188,73 @@ impl pb::inference_server::Inference for InferenceServiceImpl {
|
||||
) -> Result<Response<Self::GenerateStreamStream>, Status> {
|
||||
let proto_req = request.into_inner();
|
||||
let response_opts = ResponseOpts::from_proto(proto_req.response.as_ref());
|
||||
let text_request =
|
||||
convert::to_text_request(proto_req, true, self.state.served_model_names())?;
|
||||
let PreparedGrpcRequest {
|
||||
text_request,
|
||||
request_span,
|
||||
started_at,
|
||||
} = self.prepare_request(proto_req, true, "GenerateStream").await?;
|
||||
|
||||
let request_id = text_request.request_id.clone();
|
||||
info!(%request_id, "grpc generate (stream)");
|
||||
|
||||
let stream = self.state.chat.text().generate(text_request).await;
|
||||
let stream = stream.map_err(text_error_to_status)?;
|
||||
let stream = self
|
||||
.state
|
||||
.chat
|
||||
.text()
|
||||
.generate(text_request)
|
||||
.instrument(request_span.clone())
|
||||
.await
|
||||
.map_err(|error| log_text_error(&request_span, started_at, "submission", error))?;
|
||||
|
||||
let (tx, rx) = mpsc::channel(32);
|
||||
|
||||
tokio::spawn(async move {
|
||||
futures::pin_mut!(stream);
|
||||
while let Some(event) = stream.next().await {
|
||||
let response = match event {
|
||||
Err(e) => Err(text_error_to_status(e)),
|
||||
Ok(DecodedTextEvent::Start {
|
||||
prompt_token_ids,
|
||||
prompt_logprobs,
|
||||
}) => {
|
||||
let prompt_info = convert::to_prompt_info(
|
||||
&prompt_token_ids,
|
||||
prompt_logprobs.as_ref(),
|
||||
&response_opts,
|
||||
);
|
||||
Ok(pb::GenerateResponse {
|
||||
prompt_info: Some(prompt_info),
|
||||
outputs: None,
|
||||
})
|
||||
}
|
||||
Ok(DecodedTextEvent::TextDelta {
|
||||
delta,
|
||||
token_ids,
|
||||
logprobs,
|
||||
finished,
|
||||
}) => Ok(pb::GenerateResponse {
|
||||
prompt_info: None,
|
||||
outputs: Some(convert::to_sequence_output(
|
||||
&delta,
|
||||
&token_ids,
|
||||
logprobs.as_ref(),
|
||||
finished.as_ref(),
|
||||
&response_opts,
|
||||
)),
|
||||
}),
|
||||
};
|
||||
let task_span = request_span.clone();
|
||||
tokio::spawn(
|
||||
async move {
|
||||
futures::pin_mut!(stream);
|
||||
while let Some(event) = stream.next().await {
|
||||
let response = match event {
|
||||
Err(error) => Err(log_text_error(&task_span, started_at, "stream", error)),
|
||||
Ok(DecodedTextEvent::Start {
|
||||
prompt_token_ids,
|
||||
prompt_logprobs,
|
||||
}) => {
|
||||
let prompt_info = convert::to_prompt_info(
|
||||
&prompt_token_ids,
|
||||
prompt_logprobs.as_ref(),
|
||||
&response_opts,
|
||||
);
|
||||
Ok(pb::GenerateResponse {
|
||||
prompt_info: Some(prompt_info),
|
||||
outputs: None,
|
||||
})
|
||||
}
|
||||
Ok(DecodedTextEvent::TextDelta {
|
||||
delta,
|
||||
token_ids,
|
||||
logprobs,
|
||||
finished,
|
||||
}) => Ok(pb::GenerateResponse {
|
||||
prompt_info: None,
|
||||
outputs: Some(convert::to_sequence_output(
|
||||
&delta,
|
||||
&token_ids,
|
||||
logprobs.as_ref(),
|
||||
finished.as_ref(),
|
||||
&response_opts,
|
||||
)),
|
||||
}),
|
||||
};
|
||||
|
||||
if tx.send(response).await.is_err() {
|
||||
// Client disconnected.
|
||||
break;
|
||||
if tx.send(response).await.is_err() {
|
||||
break;
|
||||
}
|
||||
}
|
||||
info!(
|
||||
parent: &task_span,
|
||||
elapsed_ms = started_at.elapsed().as_millis() as u64,
|
||||
"gRPC inference stream closed"
|
||||
);
|
||||
}
|
||||
});
|
||||
.instrument(request_span),
|
||||
);
|
||||
|
||||
let response_stream = ReceiverStream::new(rx);
|
||||
Ok(Response::new(Box::pin(response_stream)))
|
||||
@@ -154,3 +269,21 @@ fn text_error_to_status(error: vllm_text::Error) -> Status {
|
||||
Status::internal(message)
|
||||
}
|
||||
}
|
||||
|
||||
fn log_text_error(
|
||||
request_span: &Span,
|
||||
started_at: Instant,
|
||||
phase: &'static str,
|
||||
error: vllm_text::Error,
|
||||
) -> Status {
|
||||
let status = text_error_to_status(error);
|
||||
warn!(
|
||||
parent: request_span,
|
||||
phase,
|
||||
grpc_code = ?status.code(),
|
||||
elapsed_ms = started_at.elapsed().as_millis() as u64,
|
||||
error = %status.message(),
|
||||
"gRPC inference request failed"
|
||||
);
|
||||
status
|
||||
}
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
// SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
use std::fs;
|
||||
use std::future::Future;
|
||||
use std::io;
|
||||
use std::pin::Pin;
|
||||
@@ -204,6 +205,73 @@ impl ChatRenderer for FakeTextBackend {
|
||||
}
|
||||
}
|
||||
|
||||
struct FakeMultimodalBackend {
|
||||
model_info: vllm_chat::multimodal::MultimodalModelInfo,
|
||||
}
|
||||
|
||||
impl TextBackend for FakeMultimodalBackend {
|
||||
fn tokenizer(&self) -> DynTokenizer {
|
||||
Arc::new(TestTokenizer::new().with_regular_token("<|image_pad|>", QWEN_IMAGE_TOKEN_ID))
|
||||
}
|
||||
|
||||
fn model_id(&self) -> &str {
|
||||
"test-model"
|
||||
}
|
||||
}
|
||||
|
||||
impl ChatBackend for FakeMultimodalBackend {
|
||||
fn chat_renderer(&self) -> DynChatRenderer {
|
||||
Arc::new(FakeTextBackend)
|
||||
}
|
||||
|
||||
fn multimodal_model_info(&self) -> Option<&vllm_chat::multimodal::MultimodalModelInfo> {
|
||||
Some(&self.model_info)
|
||||
}
|
||||
|
||||
fn new_chat_output_processor(
|
||||
&self,
|
||||
request: &mut ChatRequest,
|
||||
options: NewChatOutputProcessorOptions<'_>,
|
||||
) -> vllm_chat::Result<DynChatOutputProcessor> {
|
||||
Ok(Box::new(DefaultChatOutputProcessor::new(
|
||||
request,
|
||||
self.model_id(),
|
||||
self.tokenizer(),
|
||||
options.tool_call_parser,
|
||||
options.reasoning_parser,
|
||||
)?))
|
||||
}
|
||||
}
|
||||
|
||||
const QWEN_IMAGE_TOKEN_ID: u32 = 151655;
|
||||
const TINY_PNG_DATA_URI: &str = "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNk+A8AAQUBAScY42YAAAAASUVORK5CYII=";
|
||||
|
||||
fn multimodal_backend() -> Arc<dyn ChatTextBackend> {
|
||||
let config_path = std::env::temp_dir().join(format!(
|
||||
"vllm-grpc-qwen-config-{}.json",
|
||||
uuid::Uuid::new_v4()
|
||||
));
|
||||
fs::write(
|
||||
&config_path,
|
||||
r#"{"model_type":"qwen2_vl","image_token_id":151655}"#,
|
||||
)
|
||||
.expect("write qwen test config");
|
||||
let model_info = vllm_chat::multimodal::MultimodalModelInfo::from_paths(
|
||||
"qwen2-vl-test".to_string(),
|
||||
Some("qwen2_vl".to_string()),
|
||||
vllm_chat::multimodal::MultimodalConfigFiles {
|
||||
config: Some(&config_path),
|
||||
..Default::default()
|
||||
},
|
||||
Arc::new(TestTokenizer::new().with_regular_token("<|image_pad|>", QWEN_IMAGE_TOKEN_ID)),
|
||||
Default::default(),
|
||||
)
|
||||
.expect("load multimodal info")
|
||||
.expect("qwen multimodal info is registered");
|
||||
let _ = fs::remove_file(config_path);
|
||||
Arc::new(FakeMultimodalBackend { model_info })
|
||||
}
|
||||
|
||||
/// Build the gRPC service + mock engine that serves a single request with the
|
||||
/// given output specs. Shared by the plaintext and TLS server fixtures.
|
||||
async fn setup_grpc_service(
|
||||
@@ -215,6 +283,24 @@ async fn setup_grpc_service(
|
||||
tokio::sync::watch::Receiver<bool>,
|
||||
MockEngineTask,
|
||||
) {
|
||||
setup_grpc_service_with_backend(engine_id, output_specs, Arc::new(FakeTextBackend), |_| {})
|
||||
.await
|
||||
}
|
||||
|
||||
async fn setup_grpc_service_with_backend<F>(
|
||||
engine_id: impl Into<EngineId>,
|
||||
output_specs: Vec<(Vec<u32>, Option<EngineCoreFinishReason>)>,
|
||||
backend: Arc<dyn ChatTextBackend>,
|
||||
check_request: F,
|
||||
) -> (
|
||||
InferenceServer<InferenceServiceImpl>,
|
||||
ControlServer<ControlServiceImpl>,
|
||||
tokio::sync::watch::Receiver<bool>,
|
||||
MockEngineTask,
|
||||
)
|
||||
where
|
||||
F: FnOnce(&EngineCoreRequest) + Send + 'static,
|
||||
{
|
||||
let ipc = IpcNamespace::new().expect("create ipc namespace");
|
||||
let handshake_address = ipc.handshake_endpoint();
|
||||
let engine_id = engine_id.into();
|
||||
@@ -227,6 +313,7 @@ async fn setup_grpc_service(
|
||||
let add = recv_engine_message(dealer).await;
|
||||
let request: EngineCoreRequest =
|
||||
rmp_serde::from_slice(&add[1]).expect("decode request");
|
||||
check_request(&request);
|
||||
send_outputs(
|
||||
push,
|
||||
engine_outputs_for_request(&request.request_id, output_specs),
|
||||
@@ -248,13 +335,11 @@ async fn setup_grpc_service(
|
||||
.expect("connect client");
|
||||
let engine_health = client.subscribe_health();
|
||||
|
||||
let chat = ChatLlm::from_shared_backend(
|
||||
Llm::new(client),
|
||||
Arc::new(FakeTextBackend) as Arc<dyn ChatTextBackend>,
|
||||
);
|
||||
let chat = ChatLlm::from_shared_backend(Llm::new(client), backend);
|
||||
let state = Arc::new(AppState::new(vec!["test-model".to_string()], chat));
|
||||
(
|
||||
InferenceServer::new(InferenceServiceImpl::new(state.clone())),
|
||||
InferenceServer::new(InferenceServiceImpl::new(state.clone()))
|
||||
.max_decoding_message_size(crate::DEFAULT_REQUEST_BODY_LIMIT_BYTES),
|
||||
ControlServer::new(ControlServiceImpl::new(state)),
|
||||
engine_health,
|
||||
engine_task,
|
||||
@@ -560,6 +645,142 @@ async fn unary_generate_with_token_ids_prompt() {
|
||||
server_task.abort();
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
#[serial]
|
||||
async fn unary_generate_prepares_multimodal_input_for_engine_core() {
|
||||
let (inference_service, control_service, engine_health, engine_task) =
|
||||
setup_grpc_service_with_backend(
|
||||
b"engine-grpc-multimodal",
|
||||
default_stream_output_specs(),
|
||||
multimodal_backend(),
|
||||
|request| {
|
||||
let token_ids = request.prompt_token_ids.as_ref().expect("prompt token ids");
|
||||
let features = request.mm_features.as_ref().expect("multimodal features");
|
||||
assert_eq!(features.len(), 1);
|
||||
|
||||
let feature = &features[0];
|
||||
assert_eq!(feature.modality, "image");
|
||||
assert_eq!(feature.identifier, "image-1");
|
||||
assert_eq!(feature.mm_position.offset, 1);
|
||||
assert!(feature.mm_position.length > 1);
|
||||
assert_eq!(token_ids.len(), feature.mm_position.length + 2);
|
||||
assert_eq!(token_ids[0], 11);
|
||||
assert_eq!(token_ids.last(), Some(&12));
|
||||
assert!(
|
||||
token_ids[feature.mm_position.offset
|
||||
..feature.mm_position.offset + feature.mm_position.length]
|
||||
.iter()
|
||||
.all(|token_id| *token_id == QWEN_IMAGE_TOKEN_ID)
|
||||
);
|
||||
},
|
||||
)
|
||||
.await;
|
||||
let (channel, server_task) = start_grpc_test_server(
|
||||
inference_service,
|
||||
control_service,
|
||||
engine_health,
|
||||
tokio_util::sync::CancellationToken::new(),
|
||||
)
|
||||
.await;
|
||||
let mut client = InferenceClient::new(channel);
|
||||
|
||||
client
|
||||
.generate(pb::GenerateRequest {
|
||||
request_id: "test-multimodal".to_string(),
|
||||
model: "test-model".to_string(),
|
||||
prompt: Some(pb::generate_request::Prompt::TokenIds(pb::TokenIds {
|
||||
ids: vec![11, QWEN_IMAGE_TOKEN_ID, 12],
|
||||
})),
|
||||
media: vec![pb::MediaItem {
|
||||
modality: pb::Modality::Image as i32,
|
||||
source: Some(pb::media_item::Source::DataUri(
|
||||
TINY_PNG_DATA_URI.to_string(),
|
||||
)),
|
||||
mime_type: String::new(),
|
||||
uuid: "image-1".to_string(),
|
||||
}],
|
||||
stopping: Some(pb::StoppingCriteria {
|
||||
max_new_tokens: 10,
|
||||
..Default::default()
|
||||
}),
|
||||
..Default::default()
|
||||
})
|
||||
.await
|
||||
.expect("multimodal generate");
|
||||
|
||||
engine_task.await.expect("mock engine task");
|
||||
server_task.abort();
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
#[serial]
|
||||
async fn streaming_generate_rejects_text_prompt_with_media() {
|
||||
let (mut client, server_task, _engine_task) = grpc_test_server(
|
||||
b"engine-grpc-stream-media-text",
|
||||
default_stream_output_specs(),
|
||||
)
|
||||
.await;
|
||||
|
||||
let status = client
|
||||
.generate_stream(pb::GenerateRequest {
|
||||
request_id: "test-stream-media-text".to_string(),
|
||||
model: "test-model".to_string(),
|
||||
prompt: Some(pb::generate_request::Prompt::Text(
|
||||
"describe this".to_string(),
|
||||
)),
|
||||
media: vec![pb::MediaItem {
|
||||
modality: pb::Modality::Image as i32,
|
||||
source: Some(pb::media_item::Source::DataUri(
|
||||
TINY_PNG_DATA_URI.to_string(),
|
||||
)),
|
||||
mime_type: String::new(),
|
||||
uuid: "image-1".to_string(),
|
||||
}],
|
||||
..Default::default()
|
||||
})
|
||||
.await
|
||||
.expect_err("text prompts with media must be rejected");
|
||||
|
||||
assert_eq!(status.code(), tonic::Code::InvalidArgument);
|
||||
assert_eq!(
|
||||
status.message(),
|
||||
"multimodal gRPC requests must provide token_ids input"
|
||||
);
|
||||
server_task.abort();
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
#[serial]
|
||||
async fn unary_generate_accepts_request_larger_than_tonic_default() {
|
||||
let (mut client, server_task, _engine_task) =
|
||||
grpc_test_server(b"engine-grpc-large-media", default_stream_output_specs()).await;
|
||||
|
||||
let status = client
|
||||
.generate(pb::GenerateRequest {
|
||||
request_id: "test-large-media".to_string(),
|
||||
model: "test-model".to_string(),
|
||||
prompt: Some(pb::generate_request::Prompt::Text(
|
||||
"describe this".to_string(),
|
||||
)),
|
||||
media: vec![pb::MediaItem {
|
||||
modality: pb::Modality::Image as i32,
|
||||
source: Some(pb::media_item::Source::RawBytes(vec![0; 5 * 1024 * 1024])),
|
||||
mime_type: "image/png".to_string(),
|
||||
uuid: "image-1".to_string(),
|
||||
}],
|
||||
..Default::default()
|
||||
})
|
||||
.await
|
||||
.expect_err("text prompts with media must be rejected after decoding");
|
||||
|
||||
assert_eq!(status.code(), tonic::Code::InvalidArgument);
|
||||
assert_eq!(
|
||||
status.message(),
|
||||
"multimodal gRPC requests must provide token_ids input"
|
||||
);
|
||||
server_task.abort();
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
#[serial]
|
||||
async fn unary_generate_returns_token_ids_when_requested() {
|
||||
|
||||
@@ -59,6 +59,7 @@ const GRPC_KEEPALIVE_INTERVAL: Duration = Duration::from_secs(7200);
|
||||
/// How long the server waits for a keepalive PING reply before dropping the gRPC
|
||||
/// connection. 20s matches the gRPC-core default.
|
||||
const GRPC_KEEPALIVE_TIMEOUT: Duration = Duration::from_secs(20);
|
||||
const DEFAULT_REQUEST_BODY_LIMIT_BYTES: usize = 32 * 1024 * 1024;
|
||||
|
||||
/// Resolve the public model names accepted by the frontend.
|
||||
fn effective_served_model_names(model: &str, served_model_name: &[String]) -> Vec<String> {
|
||||
@@ -215,7 +216,8 @@ where
|
||||
let control_service =
|
||||
grpc::ControlGrpcService::new(grpc::ControlServiceImpl::new(state.clone()));
|
||||
let inference_service =
|
||||
grpc::InferenceGrpcService::new(grpc::InferenceServiceImpl::new(state.clone()));
|
||||
grpc::InferenceGrpcService::new(grpc::InferenceServiceImpl::new(state.clone()))
|
||||
.max_decoding_message_size(DEFAULT_REQUEST_BODY_LIMIT_BYTES);
|
||||
let svc = TonicServer::builder()
|
||||
.http2_keepalive_interval(Some(GRPC_KEEPALIVE_INTERVAL))
|
||||
.http2_keepalive_timeout(Some(GRPC_KEEPALIVE_TIMEOUT))
|
||||
|
||||
@@ -27,11 +27,10 @@ use axum::routing::{get, post};
|
||||
use tower_http::trace::TraceLayer;
|
||||
use tracing::warn;
|
||||
|
||||
use crate::DEFAULT_REQUEST_BODY_LIMIT_BYTES;
|
||||
use crate::middleware;
|
||||
use crate::state::AppState;
|
||||
|
||||
const DEFAULT_JSON_BODY_LIMIT_BYTES: usize = 32 * 1024 * 1024;
|
||||
|
||||
fn server_dev_mode_enabled() -> bool {
|
||||
std::env::var("VLLM_SERVER_DEV_MODE")
|
||||
.ok()
|
||||
@@ -127,7 +126,7 @@ fn build_router_with_options(
|
||||
let enable_api_key_auth = state.has_api_keys();
|
||||
let mut router = router
|
||||
.with_state(state.clone())
|
||||
.layer(DefaultBodyLimit::max(DEFAULT_JSON_BODY_LIMIT_BYTES))
|
||||
.layer(DefaultBodyLimit::max(DEFAULT_REQUEST_BODY_LIMIT_BYTES))
|
||||
.layer(middleware::request_runtime_layer(state.clone()))
|
||||
.layer(from_fn_with_state(
|
||||
state.clone(),
|
||||
|
||||
Reference in New Issue
Block a user