From 8eb35c5217ef98ab06c88f42ef4e41737dd2d1f3 Mon Sep 17 00:00:00 2001 From: "Kevin H. Luu" Date: Wed, 12 Aug 2026 19:08:59 -0700 Subject: [PATCH] [CI] Mirror external test assets in vLLM S3 (#52064) Signed-off-by: khluu Co-authored-by: OpenAI Codex --- .../openai/chat_completion/test_video.py | 7 ++-- .../pooling/classify/test_online_vision.py | 5 ++- tests/evals/gsm8k/gsm8k_eval.py | 6 ++- .../pooling/test_jinavl_reranker.py | 42 +++++++------------ tests/multimodal/media/test_connector.py | 11 +++-- 5 files changed, 30 insertions(+), 41 deletions(-) diff --git a/tests/entrypoints/multimodal/openai/chat_completion/test_video.py b/tests/entrypoints/multimodal/openai/chat_completion/test_video.py index a5827c9f9c..a8c6e472b9 100644 --- a/tests/entrypoints/multimodal/openai/chat_completion/test_video.py +++ b/tests/entrypoints/multimodal/openai/chat_completion/test_video.py @@ -8,6 +8,7 @@ import pytest import pytest_asyncio from tests.utils import RemoteOpenAIServer +from vllm.assets.base import VLLM_S3_BUCKET_URL from vllm.multimodal.utils import encode_video_url, fetch_video from vllm.platforms import current_platform @@ -15,9 +16,9 @@ MODEL_NAME = "llava-hf/llava-onevision-qwen2-0.5b-ov-hf" MAXIMUM_VIDEOS = 3 TEST_VIDEO_URLS = [ - "https://www.bogotobogo.com/python/OpenCV_Python/images/mean_shift_tracking/slow_traffic_small.mp4", - "https://github.com/opencv/opencv/raw/refs/tags/4.12.0/samples/data/vtest.avi", - "https://github.com/opencv/opencv/raw/refs/tags/4.12.0/samples/data/Megamind.avi", + f"{VLLM_S3_BUCKET_URL}/multimodal_asset/slow_traffic_small.mp4", + f"{VLLM_S3_BUCKET_URL}/multimodal_asset/vtest.avi", + f"{VLLM_S3_BUCKET_URL}/multimodal_asset/Megamind.avi", ] diff --git a/tests/entrypoints/pooling/classify/test_online_vision.py b/tests/entrypoints/pooling/classify/test_online_vision.py index ce60e01ebe..0d361b0914 100644 --- a/tests/entrypoints/pooling/classify/test_online_vision.py +++ b/tests/entrypoints/pooling/classify/test_online_vision.py @@ -6,6 +6,7 @@ import pytest import requests from tests.utils import RemoteOpenAIServer +from vllm.assets.base import VLLM_S3_BUCKET_URL from vllm.entrypoints.pooling.classify.protocol import ClassificationResponse from vllm.multimodal.utils import encode_image_url, fetch_image @@ -14,9 +15,9 @@ MAXIMUM_VIDEOS = 1 HF_OVERRIDES = {"architectures": ["Qwen2_5_VLForSequenceClassification"]} input_text = "This product was excellent and exceeded my expectations" -image_url = "https://vllm-public-assets.s3.us-west-2.amazonaws.com/multimodal_asset/cat_snow.jpg" +image_url = f"{VLLM_S3_BUCKET_URL}/multimodal_asset/cat_snow.jpg" image_base64 = {"url": encode_image_url(fetch_image(image_url))} -video_url = "https://www.bogotobogo.com/python/OpenCV_Python/images/mean_shift_tracking/slow_traffic_small.mp4" +video_url = f"{VLLM_S3_BUCKET_URL}/multimodal_asset/slow_traffic_small.mp4" @pytest.fixture(scope="module") diff --git a/tests/evals/gsm8k/gsm8k_eval.py b/tests/evals/gsm8k/gsm8k_eval.py index eeb88c2e97..b5a1e6a9cc 100644 --- a/tests/evals/gsm8k/gsm8k_eval.py +++ b/tests/evals/gsm8k/gsm8k_eval.py @@ -20,6 +20,8 @@ import regex as re import requests from tqdm.asyncio import tqdm +from vllm.assets.base import VLLM_S3_BUCKET_URL + INVALID = -9999999 @@ -44,8 +46,8 @@ def download_and_cache_file(url: str, filename: str | None = None) -> str: def load_gsm8k_data() -> tuple[list[dict], list[dict]]: """Load GSM8K train and test data""" - train_url = "https://raw.githubusercontent.com/openai/grade-school-math/master/grade_school_math/data/train.jsonl" - test_url = "https://raw.githubusercontent.com/openai/grade-school-math/master/grade_school_math/data/test.jsonl" + train_url = f"{VLLM_S3_BUCKET_URL}/ci-datasets/gsm8k/train.jsonl" + test_url = f"{VLLM_S3_BUCKET_URL}/ci-datasets/gsm8k/test.jsonl" train_file = download_and_cache_file(train_url) test_file = download_and_cache_file(test_url) diff --git a/tests/models/multimodal/pooling/test_jinavl_reranker.py b/tests/models/multimodal/pooling/test_jinavl_reranker.py index 18a02625ea..aa039735ce 100644 --- a/tests/models/multimodal/pooling/test_jinavl_reranker.py +++ b/tests/models/multimodal/pooling/test_jinavl_reranker.py @@ -6,6 +6,7 @@ import transformers from packaging import version from transformers import AutoModel +from vllm.assets.base import VLLM_S3_BUCKET_URL from vllm.entrypoints.chat_utils import ( ChatCompletionContentPartImageEmbedsParam, ChatCompletionContentPartImageParam, @@ -34,6 +35,11 @@ CHECKPOINT_TO_HF_MAPPER = { "model.": "model.language_model.", } +HANDELSBLATT_IMAGE_URL = ( + f"{VLLM_S3_BUCKET_URL}/multimodal_asset/jinavl-handelsblatt-preview.png" +) +PAPER_IMAGE_URL = f"{VLLM_S3_BUCKET_URL}/multimodal_asset/jinavl-paper-11.png" + # Shared long text for test data LONG_TEXT_DOC = """We present ReaderLM-v2, a compact 1.5 billion parameter language model designed for efficient web content extraction. Our model processes documents up to 512K tokens, transforming messy HTML @@ -50,12 +56,8 @@ lower computational requirements.""" # noqa: E501 TEXT_IMAGE_TEST_DATA = { "query": [{"text": "slm markdown"}], "documents": [ - { - "image": "https://raw.githubusercontent.com/jina-ai/multimodal-reranker-test/main/handelsblatt-preview.png" - }, - { - "image": "https://raw.githubusercontent.com/jina-ai/multimodal-reranker-test/main/paper-11.png" - }, + {"image": HANDELSBLATT_IMAGE_URL}, + {"image": PAPER_IMAGE_URL}, ], } @@ -68,11 +70,7 @@ TEXT_TEXT_TEST_DATA = { } IMAGE_TEXT_TEST_DATA = { - "query": [ - { - "image": "https://raw.githubusercontent.com/jina-ai/multimodal-reranker-test/main/paper-11.png" - } - ], + "query": [{"image": PAPER_IMAGE_URL}], "documents": [ {"text": LONG_TEXT_DOC}, {"text": "数据提取么?为什么不用正则啊,你用正则不就全解决了么?"}, @@ -80,18 +78,10 @@ IMAGE_TEXT_TEST_DATA = { } IMAGE_IMAGE_TEST_DATA = { - "query": [ - { - "image": "https://raw.githubusercontent.com/jina-ai/multimodal-reranker-test/main/paper-11.png" - } - ], + "query": [{"image": PAPER_IMAGE_URL}], "documents": [ - { - "image": "https://raw.githubusercontent.com/jina-ai/multimodal-reranker-test/main/handelsblatt-preview.png" - }, - { - "image": "https://raw.githubusercontent.com/jina-ai/multimodal-reranker-test/main/paper-11.png" - }, + {"image": HANDELSBLATT_IMAGE_URL}, + {"image": PAPER_IMAGE_URL}, ], } @@ -99,13 +89,9 @@ TEXT_MIXED_DOCS_TEST_DATA = { "query": [{"text": "slm markdown"}], "documents": [ {"text": LONG_TEXT_DOC}, - { - "image": "https://raw.githubusercontent.com/jina-ai/multimodal-reranker-test/main/paper-11.png" - }, + {"image": PAPER_IMAGE_URL}, {"text": "数据提取么？为什么不用正则啊,你用正则不就全解决了么?"}, - { - "image": "https://raw.githubusercontent.com/jina-ai/multimodal-reranker-test/main/handelsblatt-preview.png" - }, + {"image": HANDELSBLATT_IMAGE_URL}, ], } diff --git a/tests/multimodal/media/test_connector.py b/tests/multimodal/media/test_connector.py index ddf6dbcef8..b222d57d3c 100644 --- a/tests/multimodal/media/test_connector.py +++ b/tests/multimodal/media/test_connector.py @@ -17,6 +17,7 @@ import requests import torch from PIL import Image, ImageChops +from vllm.assets.base import VLLM_S3_BUCKET_URL from vllm.multimodal.image import convert_image_mode from vllm.multimodal.inputs import PlaceholderRange from vllm.multimodal.media import MediaConnector @@ -30,8 +31,8 @@ TEST_IMAGE_ASSETS = [ ] TEST_VIDEO_URLS = [ - "https://www.bogotobogo.com/python/OpenCV_Python/images/mean_shift_tracking/slow_traffic_small.mp4", - "https://github.com/opencv/opencv/raw/refs/tags/4.12.0/samples/data/vtest.avi", + f"{VLLM_S3_BUCKET_URL}/multimodal_asset/slow_traffic_small.mp4", + f"{VLLM_S3_BUCKET_URL}/multimodal_asset/vtest.avi", ] @@ -76,8 +77,7 @@ async def test_fetch_image_base64( connector = MediaConnector( # Domain restriction should not apply to data URLs. allowed_media_domains=[ - "www.bogotobogo.com", - "github.com", + VLLM_S3_BUCKET_URL.removeprefix("https://"), ] ) url_image = url_images[raw_image_url] @@ -390,8 +390,7 @@ async def test_allowed_media_domains(video_url: str, num_frames: int): } }, allowed_media_domains=[ - "www.bogotobogo.com", - "github.com", + VLLM_S3_BUCKET_URL.removeprefix("https://"), ], )