[CI] Consolidate multimodal entrypoint tests. (#44819)
Signed-off-by: wang.yuqi <yuqi.wang@daocloud.io>
This commit is contained in:
@@ -1270,6 +1270,20 @@ steps:
|
|||||||
- export VLLM_WORKER_MULTIPROC_METHOD=spawn
|
- export VLLM_WORKER_MULTIPROC_METHOD=spawn
|
||||||
- pytest -v -s entrypoints/speech_to_text
|
- pytest -v -s entrypoints/speech_to_text
|
||||||
|
|
||||||
|
- label: Entrypoints Integration (Multimodal)
|
||||||
|
timeout_in_minutes: 180
|
||||||
|
mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
|
||||||
|
agent_pool: mi300_1
|
||||||
|
fast_check: true
|
||||||
|
torch_nightly: true
|
||||||
|
working_dir: "/vllm-workspace/tests"
|
||||||
|
source_file_dependencies:
|
||||||
|
- vllm/
|
||||||
|
- tests/entrypoints/multimodal
|
||||||
|
commands:
|
||||||
|
- export VLLM_WORKER_MULTIPROC_METHOD=spawn
|
||||||
|
- pytest -v -s entrypoints/multimodal
|
||||||
|
|
||||||
- label: Entrypoints Integration (LLM) # TBD
|
- label: Entrypoints Integration (LLM) # TBD
|
||||||
timeout_in_minutes: 180
|
timeout_in_minutes: 180
|
||||||
mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
|
mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
|
||||||
@@ -1327,7 +1341,7 @@ steps:
|
|||||||
- vllm/platforms/rocm.py
|
- vllm/platforms/rocm.py
|
||||||
commands:
|
commands:
|
||||||
- pytest -v -s entrypoints/openai/tool_parsers
|
- pytest -v -s entrypoints/openai/tool_parsers
|
||||||
- pytest -v -s entrypoints/ --ignore=entrypoints/llm --ignore=entrypoints/openai --ignore=entrypoints/serve --ignore=entrypoints/test_chat_utils.py --ignore=entrypoints/pooling --ignore=entrypoints/speech_to_text --ignore=tests/entrypoints/generate
|
- pytest -v -s entrypoints/ --ignore=entrypoints/llm --ignore=entrypoints/openai --ignore=entrypoints/serve --ignore=entrypoints/test_chat_utils.py --ignore=entrypoints/pooling --ignore=entrypoints/speech_to_text --ignore=entrypoints/pooling --ignore=entrypoints/multimodal --ignore=entrypoints/generate
|
||||||
|
|
||||||
- label: OpenAI API correctness # TBD
|
- label: OpenAI API correctness # TBD
|
||||||
timeout_in_minutes: 180
|
timeout_in_minutes: 180
|
||||||
@@ -2865,6 +2879,20 @@ steps:
|
|||||||
- export VLLM_WORKER_MULTIPROC_METHOD=spawn
|
- export VLLM_WORKER_MULTIPROC_METHOD=spawn
|
||||||
- pytest -v -s entrypoints/speech_to_text
|
- pytest -v -s entrypoints/speech_to_text
|
||||||
|
|
||||||
|
- label: Entrypoints Integration (Multimodal)
|
||||||
|
timeout_in_minutes: 180
|
||||||
|
mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi355]
|
||||||
|
agent_pool: mi355_1
|
||||||
|
fast_check: true
|
||||||
|
torch_nightly: true
|
||||||
|
working_dir: "/vllm-workspace/tests"
|
||||||
|
source_file_dependencies:
|
||||||
|
- vllm/
|
||||||
|
- tests/entrypoints/multimodal
|
||||||
|
commands:
|
||||||
|
- export VLLM_WORKER_MULTIPROC_METHOD=spawn
|
||||||
|
- pytest -v -s entrypoints/multimodal
|
||||||
|
|
||||||
- label: Entrypoints Integration (Pooling) # TBD
|
- label: Entrypoints Integration (Pooling) # TBD
|
||||||
timeout_in_minutes: 180
|
timeout_in_minutes: 180
|
||||||
mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355]
|
mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355]
|
||||||
|
|||||||
@@ -11,7 +11,7 @@ steps:
|
|||||||
- tests/entrypoints/
|
- tests/entrypoints/
|
||||||
commands:
|
commands:
|
||||||
- pytest -v -s entrypoints/openai/tool_parsers
|
- pytest -v -s entrypoints/openai/tool_parsers
|
||||||
- pytest -v -s entrypoints/ --ignore=entrypoints/llm --ignore=entrypoints/openai --ignore=entrypoints/serve --ignore=entrypoints/test_chat_utils.py --ignore=entrypoints/pooling --ignore=entrypoints/speech_to_text --ignore=tests/entrypoints/generate
|
- pytest -v -s entrypoints/ --ignore=entrypoints/llm --ignore=entrypoints/openai --ignore=entrypoints/serve --ignore=entrypoints/test_chat_utils.py --ignore=entrypoints/pooling --ignore=entrypoints/speech_to_text --ignore=entrypoints/multimodal --ignore=entrypoints/generate
|
||||||
|
|
||||||
- label: Entrypoints Integration (LLM)
|
- label: Entrypoints Integration (LLM)
|
||||||
key: entrypoints-integration-llm
|
key: entrypoints-integration-llm
|
||||||
@@ -120,6 +120,18 @@ steps:
|
|||||||
- export VLLM_WORKER_MULTIPROC_METHOD=spawn
|
- export VLLM_WORKER_MULTIPROC_METHOD=spawn
|
||||||
- pytest -v -s entrypoints/speech_to_text
|
- pytest -v -s entrypoints/speech_to_text
|
||||||
|
|
||||||
|
- label: Entrypoints Integration (Multimodal)
|
||||||
|
device: h200_35gb
|
||||||
|
key: entrypoints-integration-multimodal
|
||||||
|
timeout_in_minutes: 50
|
||||||
|
working_dir: "/vllm-workspace/tests"
|
||||||
|
source_file_dependencies:
|
||||||
|
- vllm/
|
||||||
|
- tests/entrypoints/multimodal
|
||||||
|
commands:
|
||||||
|
- export VLLM_WORKER_MULTIPROC_METHOD=spawn
|
||||||
|
- pytest -v -s entrypoints/multimodal
|
||||||
|
|
||||||
- label: Entrypoints Integration (Pooling)
|
- label: Entrypoints Integration (Pooling)
|
||||||
key: entrypoints-integration-pooling
|
key: entrypoints-integration-pooling
|
||||||
timeout_in_minutes: 50
|
timeout_in_minutes: 50
|
||||||
|
|||||||
@@ -4,7 +4,6 @@ import weakref
|
|||||||
|
|
||||||
import pytest
|
import pytest
|
||||||
|
|
||||||
from tests.entrypoints.openai.chat_completion.test_vision import TEST_IMAGE_ASSETS
|
|
||||||
from vllm import LLM
|
from vllm import LLM
|
||||||
from vllm.distributed import cleanup_dist_env_and_memory
|
from vllm.distributed import cleanup_dist_env_and_memory
|
||||||
from vllm.sampling_params import SamplingParams
|
from vllm.sampling_params import SamplingParams
|
||||||
@@ -76,47 +75,6 @@ def test_multi_chat(text_llm):
|
|||||||
assert len(outputs) == 2
|
assert len(outputs) == 2
|
||||||
|
|
||||||
|
|
||||||
@pytest.fixture(scope="function")
|
|
||||||
def vision_llm():
|
|
||||||
# pytest caches the fixture so we use weakref.proxy to
|
|
||||||
# enable garbage collection
|
|
||||||
llm = LLM(
|
|
||||||
model="microsoft/Phi-3.5-vision-instruct",
|
|
||||||
max_model_len=4096,
|
|
||||||
max_num_seqs=5,
|
|
||||||
enforce_eager=True,
|
|
||||||
trust_remote_code=True,
|
|
||||||
limit_mm_per_prompt={"image": 2},
|
|
||||||
seed=0,
|
|
||||||
)
|
|
||||||
|
|
||||||
yield weakref.proxy(llm)
|
|
||||||
|
|
||||||
del llm
|
|
||||||
|
|
||||||
cleanup_dist_env_and_memory()
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.parametrize(
|
|
||||||
"image_urls", [[TEST_IMAGE_ASSETS[0], TEST_IMAGE_ASSETS[1]]], indirect=True
|
|
||||||
)
|
|
||||||
def test_chat_multi_image(vision_llm, image_urls: list[str]):
|
|
||||||
messages = [
|
|
||||||
{
|
|
||||||
"role": "user",
|
|
||||||
"content": [
|
|
||||||
*(
|
|
||||||
{"type": "image_url", "image_url": {"url": image_url}}
|
|
||||||
for image_url in image_urls
|
|
||||||
),
|
|
||||||
{"type": "text", "text": "What's in this image?"},
|
|
||||||
],
|
|
||||||
}
|
|
||||||
]
|
|
||||||
outputs = vision_llm.chat(messages)
|
|
||||||
assert len(outputs) >= 0
|
|
||||||
|
|
||||||
|
|
||||||
def test_llm_chat_tokenization_no_double_bos(text_llm):
|
def test_llm_chat_tokenization_no_double_bos(text_llm):
|
||||||
"""
|
"""
|
||||||
LLM.chat() should not add special tokens when using chat templates.
|
LLM.chat() should not add special tokens when using chat templates.
|
||||||
|
|||||||
@@ -0,0 +1,10 @@
|
|||||||
|
# SPDX-License-Identifier: Apache-2.0
|
||||||
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||||
|
|
||||||
|
# Test different image extensions (JPG/PNG) and formats (gray/RGB/RGBA)
|
||||||
|
TEST_IMAGE_ASSETS = [
|
||||||
|
"2560px-Gfp-wisconsin-madison-the-nature-boardwalk.jpg", # "https://vllm-public-assets.s3.us-west-2.amazonaws.com/vision_model_images/2560px-Gfp-wisconsin-madison-the-nature-boardwalk.jpg"
|
||||||
|
"Grayscale_8bits_palette_sample_image.png", # "https://vllm-public-assets.s3.us-west-2.amazonaws.com/vision_model_images/Grayscale_8bits_palette_sample_image.png",
|
||||||
|
"1280px-Venn_diagram_rgb.svg.png", # "https://vllm-public-assets.s3.us-west-2.amazonaws.com/vision_model_images/1280px-Venn_diagram_rgb.svg.png",
|
||||||
|
"RGBA_comp.png", # "https://vllm-public-assets.s3.us-west-2.amazonaws.com/vision_model_images/RGBA_comp.png",
|
||||||
|
]
|
||||||
@@ -0,0 +1,50 @@
|
|||||||
|
# SPDX-License-Identifier: Apache-2.0
|
||||||
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||||
|
import weakref
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
from tests.entrypoints.multimodal.conftest import TEST_IMAGE_ASSETS
|
||||||
|
from vllm import LLM
|
||||||
|
from vllm.distributed import cleanup_dist_env_and_memory
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture(scope="function")
|
||||||
|
def vision_llm():
|
||||||
|
# pytest caches the fixture so we use weakref.proxy to
|
||||||
|
# enable garbage collection
|
||||||
|
llm = LLM(
|
||||||
|
model="microsoft/Phi-3.5-vision-instruct",
|
||||||
|
max_model_len=4096,
|
||||||
|
max_num_seqs=5,
|
||||||
|
enforce_eager=True,
|
||||||
|
trust_remote_code=True,
|
||||||
|
limit_mm_per_prompt={"image": 2},
|
||||||
|
seed=0,
|
||||||
|
)
|
||||||
|
|
||||||
|
yield weakref.proxy(llm)
|
||||||
|
|
||||||
|
del llm
|
||||||
|
|
||||||
|
cleanup_dist_env_and_memory()
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
"image_urls", [[TEST_IMAGE_ASSETS[0], TEST_IMAGE_ASSETS[1]]], indirect=True
|
||||||
|
)
|
||||||
|
def test_chat_multi_image(vision_llm, image_urls: list[str]):
|
||||||
|
messages = [
|
||||||
|
{
|
||||||
|
"role": "user",
|
||||||
|
"content": [
|
||||||
|
*(
|
||||||
|
{"type": "image_url", "image_url": {"url": image_url}}
|
||||||
|
for image_url in image_urls
|
||||||
|
),
|
||||||
|
{"type": "text", "text": "What's in this image?"},
|
||||||
|
],
|
||||||
|
}
|
||||||
|
]
|
||||||
|
outputs = vision_llm.chat(messages)
|
||||||
|
assert len(outputs) >= 0
|
||||||
+1
-1
@@ -15,7 +15,7 @@ import logging
|
|||||||
import pytest
|
import pytest
|
||||||
import regex as re
|
import regex as re
|
||||||
|
|
||||||
from tests.entrypoints.openai.chat_completion.test_vision import TEST_IMAGE_ASSETS
|
from tests.entrypoints.multimodal.conftest import TEST_IMAGE_ASSETS
|
||||||
from vllm import LLM, SamplingParams
|
from vllm import LLM, SamplingParams
|
||||||
from vllm.renderers.params import ChatParams
|
from vllm.renderers.params import ChatParams
|
||||||
from vllm.v1.metrics import loggers as stat_loggers
|
from vllm.v1.metrics import loggers as stat_loggers
|
||||||
+1
-1
@@ -6,7 +6,7 @@ import logging
|
|||||||
import pytest
|
import pytest
|
||||||
import regex as re
|
import regex as re
|
||||||
|
|
||||||
from tests.entrypoints.openai.chat_completion.test_vision import TEST_IMAGE_ASSETS
|
from tests.entrypoints.multimodal.conftest import TEST_IMAGE_ASSETS
|
||||||
from vllm import LLM
|
from vllm import LLM
|
||||||
from vllm.entrypoints.chat_utils import ChatCompletionMessageParam
|
from vllm.entrypoints.chat_utils import ChatCompletionMessageParam
|
||||||
from vllm.v1.metrics import loggers as stat_loggers
|
from vllm.v1.metrics import loggers as stat_loggers
|
||||||
+1
-1
@@ -52,7 +52,7 @@ async def client_with_image_embeds(server_with_image_embeds):
|
|||||||
@pytest.mark.asyncio
|
@pytest.mark.asyncio
|
||||||
@pytest.mark.parametrize("model_name", [MODEL_NAME])
|
@pytest.mark.parametrize("model_name", [MODEL_NAME])
|
||||||
@pytest.mark.parametrize("dtype", [torch.half, torch.float16, torch.float32])
|
@pytest.mark.parametrize("dtype", [torch.half, torch.float16, torch.float32])
|
||||||
async def test_completions_with_image_embeds(
|
async def test_chat_completions_with_image_embeds(
|
||||||
client_with_image_embeds: openai.AsyncOpenAI,
|
client_with_image_embeds: openai.AsyncOpenAI,
|
||||||
model_name: str,
|
model_name: str,
|
||||||
image_assets: ImageTestAssets,
|
image_assets: ImageTestAssets,
|
||||||
+1
-8
@@ -8,6 +8,7 @@ import pytest
|
|||||||
import pytest_asyncio
|
import pytest_asyncio
|
||||||
from transformers import AutoProcessor
|
from transformers import AutoProcessor
|
||||||
|
|
||||||
|
from tests.entrypoints.multimodal.conftest import TEST_IMAGE_ASSETS
|
||||||
from tests.utils import ROCM_ENV_OVERRIDES, ROCM_EXTRA_ARGS, RemoteOpenAIServer
|
from tests.utils import ROCM_ENV_OVERRIDES, ROCM_EXTRA_ARGS, RemoteOpenAIServer
|
||||||
from vllm.multimodal.media import MediaWithBytes
|
from vllm.multimodal.media import MediaWithBytes
|
||||||
from vllm.multimodal.utils import encode_image_url, fetch_image
|
from vllm.multimodal.utils import encode_image_url, fetch_image
|
||||||
@@ -16,14 +17,6 @@ from vllm.platforms import current_platform
|
|||||||
MODEL_NAME = "microsoft/Phi-3.5-vision-instruct"
|
MODEL_NAME = "microsoft/Phi-3.5-vision-instruct"
|
||||||
MAXIMUM_IMAGES = 2
|
MAXIMUM_IMAGES = 2
|
||||||
|
|
||||||
# Test different image extensions (JPG/PNG) and formats (gray/RGB/RGBA)
|
|
||||||
TEST_IMAGE_ASSETS = [
|
|
||||||
"2560px-Gfp-wisconsin-madison-the-nature-boardwalk.jpg", # "https://vllm-public-assets.s3.us-west-2.amazonaws.com/vision_model_images/2560px-Gfp-wisconsin-madison-the-nature-boardwalk.jpg"
|
|
||||||
"Grayscale_8bits_palette_sample_image.png", # "https://vllm-public-assets.s3.us-west-2.amazonaws.com/vision_model_images/Grayscale_8bits_palette_sample_image.png",
|
|
||||||
"1280px-Venn_diagram_rgb.svg.png", # "https://vllm-public-assets.s3.us-west-2.amazonaws.com/vision_model_images/1280px-Venn_diagram_rgb.svg.png",
|
|
||||||
"RGBA_comp.png", # "https://vllm-public-assets.s3.us-west-2.amazonaws.com/vision_model_images/RGBA_comp.png",
|
|
||||||
]
|
|
||||||
|
|
||||||
# Required terms for beam search validation
|
# Required terms for beam search validation
|
||||||
# Each entry is a list of term groups - ALL groups must match
|
# Each entry is a list of term groups - ALL groups must match
|
||||||
# Each group is a list of alternatives - at least ONE term in the group must appear
|
# Each group is a list of alternatives - at least ONE term in the group must appear
|
||||||
+1
-7
@@ -7,19 +7,13 @@ import openai
|
|||||||
import pytest
|
import pytest
|
||||||
import pytest_asyncio
|
import pytest_asyncio
|
||||||
|
|
||||||
|
from tests.entrypoints.multimodal.conftest import TEST_IMAGE_ASSETS
|
||||||
from tests.utils import RemoteOpenAIServer
|
from tests.utils import RemoteOpenAIServer
|
||||||
from vllm.multimodal.utils import encode_image_url
|
from vllm.multimodal.utils import encode_image_url
|
||||||
|
|
||||||
# Use a small vision model for testing
|
# Use a small vision model for testing
|
||||||
MODEL_NAME = "Qwen/Qwen2.5-VL-3B-Instruct"
|
MODEL_NAME = "Qwen/Qwen2.5-VL-3B-Instruct"
|
||||||
MAXIMUM_IMAGES = 2
|
MAXIMUM_IMAGES = 2
|
||||||
# Test different image extensions (JPG/PNG) and formats (gray/RGB/RGBA)
|
|
||||||
TEST_IMAGE_ASSETS = [
|
|
||||||
"2560px-Gfp-wisconsin-madison-the-nature-boardwalk.jpg", # "https://vllm-public-assets.s3.us-west-2.amazonaws.com/vision_model_images/2560px-Gfp-wisconsin-madison-the-nature-boardwalk.jpg"
|
|
||||||
"Grayscale_8bits_palette_sample_image.png", # "https://vllm-public-assets.s3.us-west-2.amazonaws.com/vision_model_images/Grayscale_8bits_palette_sample_image.png",
|
|
||||||
"1280px-Venn_diagram_rgb.svg.png", # "https://vllm-public-assets.s3.us-west-2.amazonaws.com/vision_model_images/1280px-Venn_diagram_rgb.svg.png",
|
|
||||||
"RGBA_comp.png", # "https://vllm-public-assets.s3.us-west-2.amazonaws.com/vision_model_images/RGBA_comp.png",
|
|
||||||
]
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.fixture(scope="module")
|
@pytest.fixture(scope="module")
|
||||||
Reference in New Issue
Block a user