[VLM] Refactor MultiModalConfig initialization and profiling (#7530)
This commit is contained in:
@@ -86,8 +86,12 @@ def server_function(port):
|
||||
|
||||
ModelRegistry.register_model("OPTForCausalLM", FakeAudioModel)
|
||||
|
||||
with patch("vllm.entrypoints.chat_utils._mm_token_str",
|
||||
lambda *_, **__: "_"):
|
||||
with patch(
|
||||
"vllm.entrypoints.chat_utils._mm_token_str",
|
||||
lambda *_, **__: "_"), patch(
|
||||
"vllm.model_executor.models.ModelRegistry.is_multimodal_model"
|
||||
) as mock:
|
||||
mock.return_value = True
|
||||
sys.argv = ["placeholder.py"] + \
|
||||
(f"--model {MODEL_NAME} --gpu-memory-utilization 0.10 "
|
||||
"--dtype bfloat16 --enforce-eager --api-key token-abc123 "
|
||||
|
||||
@@ -4,7 +4,7 @@ import numpy as np
|
||||
import pytest
|
||||
from transformers import CLIPImageProcessor, LlavaNextImageProcessor
|
||||
|
||||
from vllm.config import ModelConfig, MultiModalConfig
|
||||
from vllm.config import ModelConfig
|
||||
from vllm.multimodal import MultiModalRegistry
|
||||
from vllm.multimodal.utils import rescale_image_size
|
||||
|
||||
@@ -30,10 +30,10 @@ def test_clip_image_processor(image_assets, mm_registry, dtype, size_factor):
|
||||
seed=0,
|
||||
dtype=dtype,
|
||||
revision=None,
|
||||
limit_mm_per_prompt={"image": 1},
|
||||
)
|
||||
mm_config = MultiModalConfig(limit_per_prompt={"image": 1})
|
||||
|
||||
mm_registry.init_mm_limits_per_prompt(model_config, mm_config)
|
||||
mm_registry.init_mm_limits_per_prompt(model_config)
|
||||
|
||||
for asset in image_assets:
|
||||
image = rescale_image_size(asset.pil_image, size_factor)
|
||||
@@ -73,10 +73,10 @@ def test_llava_next_image_processor(image_assets, mm_registry, dtype,
|
||||
seed=0,
|
||||
dtype=dtype,
|
||||
revision=None,
|
||||
limit_mm_per_prompt={"image": 1},
|
||||
)
|
||||
mm_config = MultiModalConfig(limit_per_prompt={"image": 1})
|
||||
|
||||
mm_registry.init_mm_limits_per_prompt(model_config, mm_config)
|
||||
mm_registry.init_mm_limits_per_prompt(model_config)
|
||||
|
||||
for asset in image_assets:
|
||||
image = rescale_image_size(asset.pil_image, size_factor)
|
||||
@@ -115,10 +115,10 @@ def test_mm_limits(image_assets, mm_registry, num_images, limit, is_valid):
|
||||
seed=0,
|
||||
dtype="half",
|
||||
revision=None,
|
||||
limit_mm_per_prompt={"image": limit},
|
||||
)
|
||||
mm_config = MultiModalConfig(limit_per_prompt={"image": limit})
|
||||
|
||||
mm_registry.init_mm_limits_per_prompt(model_config, mm_config)
|
||||
mm_registry.init_mm_limits_per_prompt(model_config)
|
||||
|
||||
image = image_assets[0].pil_image
|
||||
if num_images == 0:
|
||||
@@ -145,10 +145,10 @@ def test_image_mapper_multi(image_assets, mm_registry, num_images):
|
||||
seed=0,
|
||||
dtype="half",
|
||||
revision=None,
|
||||
limit_mm_per_prompt={"image": num_images},
|
||||
)
|
||||
mm_config = MultiModalConfig(limit_per_prompt={"image": num_images})
|
||||
|
||||
mm_registry.init_mm_limits_per_prompt(model_config, mm_config)
|
||||
mm_registry.init_mm_limits_per_prompt(model_config)
|
||||
|
||||
image = image_assets[0].pil_image
|
||||
mm_inputs = {"image": [image] * num_images}
|
||||
|
||||
Reference in New Issue
Block a user