[MM][Mistral] Add compile support for Pixtral vision encoders (#57168)
Signed-off-by: Jakub Zakrzewski <jzakrzewski@nvidia.com> Co-authored-by: Nicolò Lucchesi <nicolo.lucchesi@mistral.ai> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
Nicolò Lucchesi
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
93dd80f636
commit
effdea9a53
@@ -1,10 +1,13 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from functools import partial
|
||||
|
||||
import pytest
|
||||
|
||||
from tests.models.utils import dummy_hf_overrides
|
||||
from vllm.compilation.counter import compilation_counter
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.config.compilation import CompilationMode
|
||||
from vllm.config.compilation import CompilationMode, CUDAGraphMode
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
|
||||
@@ -14,6 +17,77 @@ def test_compile():
|
||||
assert not vllm_config.compilation_config.compile_mm_encoder
|
||||
|
||||
|
||||
@pytest.mark.forked
|
||||
@pytest.mark.core_model
|
||||
@pytest.mark.skipif(not current_platform.is_cuda(), reason="Skip if not cuda")
|
||||
@pytest.mark.parametrize(
|
||||
("model", "tokenizer", "config_format", "tokenizer_mode", "model_arch"),
|
||||
[
|
||||
pytest.param(
|
||||
"mistralai/Pixtral-12B-2409",
|
||||
None,
|
||||
"mistral",
|
||||
"mistral",
|
||||
"PixtralForConditionalGeneration",
|
||||
id="pixtral",
|
||||
),
|
||||
pytest.param(
|
||||
"mistralai/Mistral-Small-3.1-24B-Instruct-2503",
|
||||
"mistral-community/pixtral-12b",
|
||||
"hf",
|
||||
"auto",
|
||||
"Mistral3ForConditionalGeneration",
|
||||
id="mistral3-hf",
|
||||
),
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
("compile_mm_encoder", "expected_models"),
|
||||
[
|
||||
# The dummy model has two compiled text components. Enabling encoder
|
||||
# compilation adds its single vision transformer block.
|
||||
pytest.param(False, 2, id="disabled"),
|
||||
pytest.param(True, 3, id="enabled"),
|
||||
],
|
||||
)
|
||||
def test_pixtral_compilation(
|
||||
vllm_runner,
|
||||
monkeypatch,
|
||||
model: str,
|
||||
tokenizer: str | None,
|
||||
config_format: str,
|
||||
tokenizer_mode: str,
|
||||
model_arch: str,
|
||||
compile_mm_encoder: bool,
|
||||
expected_models: int,
|
||||
):
|
||||
"""Test compilation for both Pixtral vision encoder implementations."""
|
||||
monkeypatch.setenv("VLLM_ENABLE_V1_MULTIPROCESSING", "0")
|
||||
monkeypatch.setenv("VLLM_USE_FLASHINFER_SAMPLER", "0")
|
||||
|
||||
with (
|
||||
compilation_counter.expect(num_models_seen=expected_models),
|
||||
vllm_runner(
|
||||
model,
|
||||
tokenizer_name=tokenizer,
|
||||
tokenizer_mode=tokenizer_mode,
|
||||
config_format=config_format,
|
||||
load_format="dummy",
|
||||
hf_overrides=partial(dummy_hf_overrides, model_arch=model_arch),
|
||||
max_model_len=4096,
|
||||
limit_mm_per_prompt={"image": 1},
|
||||
gpu_memory_utilization=0.8,
|
||||
attention_backend="FLASH_ATTN",
|
||||
compilation_config={
|
||||
"mode": CompilationMode.VLLM_COMPILE,
|
||||
"compile_mm_encoder": compile_mm_encoder,
|
||||
"cudagraph_mode": CUDAGraphMode.NONE,
|
||||
},
|
||||
) as _,
|
||||
):
|
||||
pass
|
||||
|
||||
|
||||
# forked needed to workaround https://github.com/vllm-project/vllm/issues/21073
|
||||
@pytest.mark.forked
|
||||
@pytest.mark.skipif(not current_platform.is_cuda(), reason="Skip if not cuda")
|
||||
|
||||
Reference in New Issue
Block a user