[Model] Fused mm preprocess normalisation on the Device (#50411)

Signed-off-by: wang.yuqi <yuqi.wang@daocloud.io>
Signed-off-by: wang.yuqi <noooop@126.com>
Co-authored-by: Cyrus Leung <cyrus.tl.leung@gmail.com>
This commit is contained in:
wang.yuqi
2026-08-06 03:11:14 -07:00
committed by GitHub
co-authored by Cyrus Leung
parent 9c22668436
commit 2fa490470d
13 changed files with 359 additions and 11 deletions
@@ -72,6 +72,7 @@ def vqa_ppl_test(
# and avoid batch different requests together.
model_config = vllm_model.llm.llm_engine.model_config
mm_device_do_normalize = model_config.multimodal_config.mm_device_do_normalize
# Confirm whether vllm is using the correct architecture
if model_info.architecture:
@@ -147,6 +148,7 @@ def vqa_ppl_test(
differ = (vllm_ppl - hf_ppl) / hf_ppl
print("Model:", model_info.name)
print("mm_device_do_normalize:", mm_device_do_normalize)
print("VLLM:", f"dtype:{vllm_dtype}", f"head_dtype:{head_dtype}", vllm_ppl)
print("Transformers:", hf_dtype, hf_ppl)
print("Difference (%):", differ * 100)
@@ -8,8 +8,8 @@ from tests.models.utils import GenerateModelInfo
from .ppl_utils import vqa_ppl_test
MODELS = [
GenerateModelInfo("Qwen/Qwen2-VL-2B-Instruct"),
GenerateModelInfo("Qwen/Qwen2.5-VL-3B-Instruct"),
GenerateModelInfo("Qwen/Qwen2-VL-2B-Instruct", hf_ppl=41081356.0),
GenerateModelInfo("Qwen/Qwen2.5-VL-3B-Instruct", hf_ppl=18330016.0),
]
@@ -20,7 +20,14 @@ mm_processor_kwargs = {
@pytest.mark.parametrize("model_info", MODELS)
def test_ppl(hf_runner, vllm_runner, model_info: GenerateModelInfo):
@pytest.mark.parametrize("mm_device_do_normalize", [True, False])
def test_ppl(
hf_runner, vllm_runner, model_info: GenerateModelInfo, mm_device_do_normalize: bool
):
vqa_ppl_test(
hf_runner, vllm_runner, model_info, mm_processor_kwargs=mm_processor_kwargs
hf_runner,
vllm_runner,
model_info,
vllm_extra_kwargs={"mm_device_do_normalize": mm_device_do_normalize},
mm_processor_kwargs=mm_processor_kwargs,
)