[Bugfix] Auto-raise max_num_batched_tokens for prefix-LM multimodal models (#43051)

Signed-off-by: Ashwin Giridharan <girida@amazon.com>
Co-authored-by: abinggo <107740309+abinggo@users.noreply.github.com>
This commit is contained in:
Ashwin Giridharan
2026-05-22 21:23:50 -07:00
committed by GitHub
co-authored by abinggo
parent 76ea1d5d2f
commit 84e351555a
2 changed files with 88 additions and 0 deletions
+38
View File
@@ -90,3 +90,41 @@ def test_defaults_with_usage_context():
vllm_config = engine_args.create_engine_config(UsageContext.OPENAI_API_SERVER)
assert vllm_config.scheduler_config.max_num_seqs == default_max_num_seqs
assert vllm_config.scheduler_config.max_num_batched_tokens == default_server_tokens # noqa: E501
def test_mm_prefix_lm_raises_batched_tokens_floor():
"""Verify that prefix-LM multimodal models auto-raise
max_num_batched_tokens to fit at least one multimodal item.
Regression test for https://github.com/vllm-project/vllm/issues/42687
"""
from unittest.mock import patch
# Simulate a prefix-LM multimodal model whose largest modality
# (video) requires 2496 tokens — more than the 2048 default.
fake_mm_min = (2496, "video")
engine_args = EngineArgs(
model="facebook/opt-125m",
max_model_len=2048,
enforce_eager=True,
)
with (
patch.object(
type(engine_args),
"_get_min_mm_batched_tokens",
staticmethod(lambda _mc: fake_mm_min),
),
patch(
"vllm.config.ModelConfig.is_multimodal_model",
new_callable=lambda: property(lambda self: True),
),
patch(
"vllm.config.ModelConfig.is_mm_prefix_lm",
new_callable=lambda: property(lambda self: True),
),
):
vllm_config = engine_args.create_engine_config(UsageContext.OPENAI_API_SERVER)
assert vllm_config.scheduler_config.max_num_batched_tokens >= 2496