[Model] Fused mm preprocess normalisation on the Device (#50411)
Signed-off-by: wang.yuqi <yuqi.wang@daocloud.io> Signed-off-by: wang.yuqi <noooop@126.com> Co-authored-by: Cyrus Leung <cyrus.tl.leung@gmail.com>
This commit is contained in:
@@ -72,6 +72,7 @@ def vqa_ppl_test(
|
||||
# and avoid batch different requests together.
|
||||
|
||||
model_config = vllm_model.llm.llm_engine.model_config
|
||||
mm_device_do_normalize = model_config.multimodal_config.mm_device_do_normalize
|
||||
|
||||
# Confirm whether vllm is using the correct architecture
|
||||
if model_info.architecture:
|
||||
@@ -147,6 +148,7 @@ def vqa_ppl_test(
|
||||
|
||||
differ = (vllm_ppl - hf_ppl) / hf_ppl
|
||||
print("Model:", model_info.name)
|
||||
print("mm_device_do_normalize:", mm_device_do_normalize)
|
||||
print("VLLM:", f"dtype:{vllm_dtype}", f"head_dtype:{head_dtype}", vllm_ppl)
|
||||
print("Transformers:", hf_dtype, hf_ppl)
|
||||
print("Difference (%):", differ * 100)
|
||||
|
||||
@@ -8,8 +8,8 @@ from tests.models.utils import GenerateModelInfo
|
||||
from .ppl_utils import vqa_ppl_test
|
||||
|
||||
MODELS = [
|
||||
GenerateModelInfo("Qwen/Qwen2-VL-2B-Instruct"),
|
||||
GenerateModelInfo("Qwen/Qwen2.5-VL-3B-Instruct"),
|
||||
GenerateModelInfo("Qwen/Qwen2-VL-2B-Instruct", hf_ppl=41081356.0),
|
||||
GenerateModelInfo("Qwen/Qwen2.5-VL-3B-Instruct", hf_ppl=18330016.0),
|
||||
]
|
||||
|
||||
|
||||
@@ -20,7 +20,14 @@ mm_processor_kwargs = {
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model_info", MODELS)
|
||||
def test_ppl(hf_runner, vllm_runner, model_info: GenerateModelInfo):
|
||||
@pytest.mark.parametrize("mm_device_do_normalize", [True, False])
|
||||
def test_ppl(
|
||||
hf_runner, vllm_runner, model_info: GenerateModelInfo, mm_device_do_normalize: bool
|
||||
):
|
||||
vqa_ppl_test(
|
||||
hf_runner, vllm_runner, model_info, mm_processor_kwargs=mm_processor_kwargs
|
||||
hf_runner,
|
||||
vllm_runner,
|
||||
model_info,
|
||||
vllm_extra_kwargs={"mm_device_do_normalize": mm_device_do_normalize},
|
||||
mm_processor_kwargs=mm_processor_kwargs,
|
||||
)
|
||||
|
||||
@@ -2,9 +2,11 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
from packaging.version import Version
|
||||
from transformers import __version__ as TRANSFORMERS_VERSION
|
||||
|
||||
from vllm.model_executor.models.vision import FusedInputNorm
|
||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
|
||||
from ....conftest import ImageTestAssets
|
||||
@@ -128,3 +130,52 @@ def test_get_image_size_with_most_features(
|
||||
t, h, w = grid_thw[0]
|
||||
tokens = (t * h * w) // (merge_size**2)
|
||||
assert tokens < max_tokens
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"model_id", ["Qwen/Qwen2-VL-2B-Instruct", "Qwen/Qwen2.5-VL-3B-Instruct"]
|
||||
)
|
||||
@pytest.mark.parametrize("num_imgs", [1, 2])
|
||||
def test_mm_device_do_normalize(
|
||||
image_assets: ImageTestAssets,
|
||||
model_id: str,
|
||||
num_imgs: int,
|
||||
):
|
||||
"""Ensure that enable mm_device_do_normalize yields the correct result."""
|
||||
|
||||
ctx = build_model_context(
|
||||
model_id,
|
||||
limit_mm_per_prompt={"image": num_imgs},
|
||||
)
|
||||
ctx.model_config.multimodal_config.mm_device_do_normalize = False
|
||||
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
|
||||
|
||||
# Build the image str / prompt based on the number of images we pass
|
||||
prompt = "<|vision_start|><|image_pad|><|vision_end|>" * num_imgs
|
||||
mm_data = {"image": [image_assets[0].pil_image] * num_imgs}
|
||||
|
||||
processed_inputs_with_normalize = processor(
|
||||
prompt,
|
||||
mm_items=processor.info.parse_mm_data(mm_data),
|
||||
)
|
||||
pixel_values_with_normalize = processed_inputs_with_normalize[
|
||||
"mm_kwargs"
|
||||
].get_data()["pixel_values"]
|
||||
dtype = pixel_values_with_normalize.dtype
|
||||
|
||||
processed_inputs_without_normalize = processor(
|
||||
prompt,
|
||||
mm_items=processor.info.parse_mm_data(mm_data),
|
||||
hf_processor_mm_kwargs={"do_normalize": False, "do_rescale": False},
|
||||
)
|
||||
pixel_values_without_normalize = processed_inputs_without_normalize[
|
||||
"mm_kwargs"
|
||||
].get_data()["pixel_values"]
|
||||
|
||||
ctx.model_config.multimodal_config.mm_device_do_normalize = True
|
||||
input_norm = FusedInputNorm.from_model_config(ctx.model_config)
|
||||
pixel_values_do_input_norm = input_norm(
|
||||
pixel_values_without_normalize.to(dtype), dtype
|
||||
)
|
||||
|
||||
torch.testing.assert_close(pixel_values_with_normalize, pixel_values_do_input_norm)
|
||||
|
||||
Reference in New Issue
Block a user