[Bugfix] Proper input validation for multi-modal encoder-decoder models (#16156)

Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
This commit is contained in:
Cyrus Leung
2025-04-08 09:45:21 -07:00
committed by GitHub
parent dc96fd54c6
commit 4ebc0b9640
10 changed files with 113 additions and 62 deletions
+6 -7
View File
@@ -213,8 +213,12 @@ class MultiModalProfiler(Generic[_I]):
total_len = len(encoder_prompt_token_ids)
# Encoder-decoder multimodal models only support v0
if total_len > seq_len:
processor = cast(EncDecMultiModalProcessor, self.processor)
if processor.pad_dummy_encoder_prompt:
num_tokens_to_pad = max(total_len, seq_len) - total_len
encoder_prompt_token_ids.extend([0] * num_tokens_to_pad)
# NOTE: Whisper allows total_len > seq_len.
elif total_len > seq_len and not envs.VLLM_USE_V1:
# `max_num_batched_tokens` is defined by `SchedulerConfig`
logger.warning_once(
"The encoder sequence length used for profiling ("
@@ -229,11 +233,6 @@ class MultiModalProfiler(Generic[_I]):
"increase `max_model_len`, reduce `max_num_seqs`, "
"and/or reduce `mm_counts`.")
processor = cast(EncDecMultiModalProcessor, self.processor)
if processor.pad_dummy_encoder_prompt:
num_tokens_to_pad = max(total_len, seq_len) - total_len
encoder_prompt_token_ids.extend([0] * num_tokens_to_pad)
return DummyEncoderData(encoder_prompt_token_ids)
def get_decoder_dummy_data(