[Bugfix] Auto-raise max_num_batched_tokens for prefix-LM multimodal models (#43051)

Signed-off-by: Ashwin Giridharan <girida@amazon.com>
Co-authored-by: abinggo <107740309+abinggo@users.noreply.github.com>
This commit is contained in:
Ashwin Giridharan
2026-05-22 21:23:50 -07:00
committed by GitHub
co-authored by abinggo
parent 76ea1d5d2f
commit 84e351555a
2 changed files with 88 additions and 0 deletions
+50
View File
@@ -2439,6 +2439,39 @@ class EngineArgs:
self.reasoning_config = ReasoningConfig()
self.reasoning_config.reasoning_parser = self.reasoning_parser
@staticmethod
def _get_min_mm_batched_tokens(
model_config: ModelConfig,
) -> tuple[int, str] | None:
"""Get the minimum max_num_batched_tokens needed for a multimodal
prefix-LM model to process at least one item of any supported modality.
Returns (token_count, modality_name) for the most expensive modality,
or None if the value cannot be determined at this stage.
"""
try:
from vllm.multimodal import MULTIMODAL_REGISTRY
# get_processing_info returns the model's multimodal processing
# metadata (supported modalities, token limits) without loading
# model weights or generating dummy data.
info = MULTIMODAL_REGISTRY.get_processing_info(model_config)
mm_counts = {modality: 1 for modality in info.supported_mm_limits}
# get_mm_max_tokens_per_item returns pre-computed per-item token
# ceilings for models that override it (e.g., Gemma4), or None
# for models that rely on dummy-input profiling. When None is
# returned we bail out — no dummy generation is triggered here.
max_tokens = info.get_mm_max_tokens_per_item(
seq_len=model_config.max_model_len,
mm_counts=mm_counts,
)
if max_tokens is not None:
modality = max(max_tokens, key=max_tokens.__getitem__)
return (max_tokens[modality], modality)
except Exception as e:
logger.warning("Failed to determine min multimodal batched tokens: %s", e)
return None
def _set_default_max_num_seqs_and_batched_tokens_args(
self,
usage_context: UsageContext | None,
@@ -2489,6 +2522,23 @@ class EngineArgs:
self.max_num_batched_tokens,
)
# For multimodal prefix-LM models (e.g., Gemma 4) that disable
# chunked MM input, a single multimodal item must fit in one batch.
# Raise the floor to accommodate the largest per-item token count.
if model_config.is_multimodal_model and model_config.is_mm_prefix_lm:
result = self._get_min_mm_batched_tokens(model_config)
if result is not None and result[0] > self.max_num_batched_tokens:
mm_min, modality = result
logger.info(
"Raising max_num_batched_tokens from %d to %d to "
"accommodate '%s' input for prefix-LM model %s.",
self.max_num_batched_tokens,
mm_min,
modality,
model_config.model,
)
self.max_num_batched_tokens = mm_min
# When using default settings,
# Ensure max_num_batched_tokens does not exceed model limit.
# Some models (e.g., Whisper) have embeddings tied to max length.