[Bugfix] Auto-raise max_num_batched_tokens for prefix-LM multimodal models (#43051)
Signed-off-by: Ashwin Giridharan <girida@amazon.com> Co-authored-by: abinggo <107740309+abinggo@users.noreply.github.com>
This commit is contained in:
co-authored by
abinggo
parent
76ea1d5d2f
commit
84e351555a
@@ -90,3 +90,41 @@ def test_defaults_with_usage_context():
|
||||
vllm_config = engine_args.create_engine_config(UsageContext.OPENAI_API_SERVER)
|
||||
assert vllm_config.scheduler_config.max_num_seqs == default_max_num_seqs
|
||||
assert vllm_config.scheduler_config.max_num_batched_tokens == default_server_tokens # noqa: E501
|
||||
|
||||
|
||||
def test_mm_prefix_lm_raises_batched_tokens_floor():
|
||||
"""Verify that prefix-LM multimodal models auto-raise
|
||||
max_num_batched_tokens to fit at least one multimodal item.
|
||||
|
||||
Regression test for https://github.com/vllm-project/vllm/issues/42687
|
||||
"""
|
||||
from unittest.mock import patch
|
||||
|
||||
# Simulate a prefix-LM multimodal model whose largest modality
|
||||
# (video) requires 2496 tokens — more than the 2048 default.
|
||||
fake_mm_min = (2496, "video")
|
||||
|
||||
engine_args = EngineArgs(
|
||||
model="facebook/opt-125m",
|
||||
max_model_len=2048,
|
||||
enforce_eager=True,
|
||||
)
|
||||
|
||||
with (
|
||||
patch.object(
|
||||
type(engine_args),
|
||||
"_get_min_mm_batched_tokens",
|
||||
staticmethod(lambda _mc: fake_mm_min),
|
||||
),
|
||||
patch(
|
||||
"vllm.config.ModelConfig.is_multimodal_model",
|
||||
new_callable=lambda: property(lambda self: True),
|
||||
),
|
||||
patch(
|
||||
"vllm.config.ModelConfig.is_mm_prefix_lm",
|
||||
new_callable=lambda: property(lambda self: True),
|
||||
),
|
||||
):
|
||||
vllm_config = engine_args.create_engine_config(UsageContext.OPENAI_API_SERVER)
|
||||
|
||||
assert vllm_config.scheduler_config.max_num_batched_tokens >= 2496
|
||||
|
||||
Reference in New Issue
Block a user