[CI] Mirror external test assets in vLLM S3 (#52064)

Signed-off-by: khluu <khluu000@gmail.com>
Co-authored-by: OpenAI Codex <noreply@openai.com>
This commit is contained in:
Kevin H. Luu
2026-08-13 02:08:59 +00:00
committed by GitHub
co-authored by OpenAI Codex
parent b369f10d5c
commit 8eb35c5217
5 changed files with 30 additions and 41 deletions
@@ -8,6 +8,7 @@ import pytest
import pytest_asyncio import pytest_asyncio
from tests.utils import RemoteOpenAIServer from tests.utils import RemoteOpenAIServer
from vllm.assets.base import VLLM_S3_BUCKET_URL
from vllm.multimodal.utils import encode_video_url, fetch_video from vllm.multimodal.utils import encode_video_url, fetch_video
from vllm.platforms import current_platform from vllm.platforms import current_platform
@@ -15,9 +16,9 @@ MODEL_NAME = "llava-hf/llava-onevision-qwen2-0.5b-ov-hf"
MAXIMUM_VIDEOS = 3 MAXIMUM_VIDEOS = 3
TEST_VIDEO_URLS = [ TEST_VIDEO_URLS = [
"https://www.bogotobogo.com/python/OpenCV_Python/images/mean_shift_tracking/slow_traffic_small.mp4", f"{VLLM_S3_BUCKET_URL}/multimodal_asset/slow_traffic_small.mp4",
"https://github.com/opencv/opencv/raw/refs/tags/4.12.0/samples/data/vtest.avi", f"{VLLM_S3_BUCKET_URL}/multimodal_asset/vtest.avi",
"https://github.com/opencv/opencv/raw/refs/tags/4.12.0/samples/data/Megamind.avi", f"{VLLM_S3_BUCKET_URL}/multimodal_asset/Megamind.avi",
] ]
@@ -6,6 +6,7 @@ import pytest
import requests import requests
from tests.utils import RemoteOpenAIServer from tests.utils import RemoteOpenAIServer
from vllm.assets.base import VLLM_S3_BUCKET_URL
from vllm.entrypoints.pooling.classify.protocol import ClassificationResponse from vllm.entrypoints.pooling.classify.protocol import ClassificationResponse
from vllm.multimodal.utils import encode_image_url, fetch_image from vllm.multimodal.utils import encode_image_url, fetch_image
@@ -14,9 +15,9 @@ MAXIMUM_VIDEOS = 1
HF_OVERRIDES = {"architectures": ["Qwen2_5_VLForSequenceClassification"]} HF_OVERRIDES = {"architectures": ["Qwen2_5_VLForSequenceClassification"]}
input_text = "This product was excellent and exceeded my expectations" input_text = "This product was excellent and exceeded my expectations"
image_url = "https://vllm-public-assets.s3.us-west-2.amazonaws.com/multimodal_asset/cat_snow.jpg" image_url = f"{VLLM_S3_BUCKET_URL}/multimodal_asset/cat_snow.jpg"
image_base64 = {"url": encode_image_url(fetch_image(image_url))} image_base64 = {"url": encode_image_url(fetch_image(image_url))}
video_url = "https://www.bogotobogo.com/python/OpenCV_Python/images/mean_shift_tracking/slow_traffic_small.mp4" video_url = f"{VLLM_S3_BUCKET_URL}/multimodal_asset/slow_traffic_small.mp4"
@pytest.fixture(scope="module") @pytest.fixture(scope="module")
+4 -2
View File
@@ -20,6 +20,8 @@ import regex as re
import requests import requests
from tqdm.asyncio import tqdm from tqdm.asyncio import tqdm
from vllm.assets.base import VLLM_S3_BUCKET_URL
INVALID = -9999999 INVALID = -9999999
@@ -44,8 +46,8 @@ def download_and_cache_file(url: str, filename: str | None = None) -> str:
def load_gsm8k_data() -> tuple[list[dict], list[dict]]: def load_gsm8k_data() -> tuple[list[dict], list[dict]]:
"""Load GSM8K train and test data""" """Load GSM8K train and test data"""
train_url = "https://raw.githubusercontent.com/openai/grade-school-math/master/grade_school_math/data/train.jsonl" train_url = f"{VLLM_S3_BUCKET_URL}/ci-datasets/gsm8k/train.jsonl"
test_url = "https://raw.githubusercontent.com/openai/grade-school-math/master/grade_school_math/data/test.jsonl" test_url = f"{VLLM_S3_BUCKET_URL}/ci-datasets/gsm8k/test.jsonl"
train_file = download_and_cache_file(train_url) train_file = download_and_cache_file(train_url)
test_file = download_and_cache_file(test_url) test_file = download_and_cache_file(test_url)
@@ -6,6 +6,7 @@ import transformers
from packaging import version from packaging import version
from transformers import AutoModel from transformers import AutoModel
from vllm.assets.base import VLLM_S3_BUCKET_URL
from vllm.entrypoints.chat_utils import ( from vllm.entrypoints.chat_utils import (
ChatCompletionContentPartImageEmbedsParam, ChatCompletionContentPartImageEmbedsParam,
ChatCompletionContentPartImageParam, ChatCompletionContentPartImageParam,
@@ -34,6 +35,11 @@ CHECKPOINT_TO_HF_MAPPER = {
"model.": "model.language_model.", "model.": "model.language_model.",
} }
HANDELSBLATT_IMAGE_URL = (
f"{VLLM_S3_BUCKET_URL}/multimodal_asset/jinavl-handelsblatt-preview.png"
)
PAPER_IMAGE_URL = f"{VLLM_S3_BUCKET_URL}/multimodal_asset/jinavl-paper-11.png"
# Shared long text for test data # Shared long text for test data
LONG_TEXT_DOC = """We present ReaderLM-v2, a compact 1.5 billion parameter language model designed for efficient LONG_TEXT_DOC = """We present ReaderLM-v2, a compact 1.5 billion parameter language model designed for efficient
web content extraction. Our model processes documents up to 512K tokens, transforming messy HTML web content extraction. Our model processes documents up to 512K tokens, transforming messy HTML
@@ -50,12 +56,8 @@ lower computational requirements.""" # noqa: E501
TEXT_IMAGE_TEST_DATA = { TEXT_IMAGE_TEST_DATA = {
"query": [{"text": "slm markdown"}], "query": [{"text": "slm markdown"}],
"documents": [ "documents": [
{ {"image": HANDELSBLATT_IMAGE_URL},
"image": "https://raw.githubusercontent.com/jina-ai/multimodal-reranker-test/main/handelsblatt-preview.png" {"image": PAPER_IMAGE_URL},
},
{
"image": "https://raw.githubusercontent.com/jina-ai/multimodal-reranker-test/main/paper-11.png"
},
], ],
} }
@@ -68,11 +70,7 @@ TEXT_TEXT_TEST_DATA = {
} }
IMAGE_TEXT_TEST_DATA = { IMAGE_TEXT_TEST_DATA = {
"query": [ "query": [{"image": PAPER_IMAGE_URL}],
{
"image": "https://raw.githubusercontent.com/jina-ai/multimodal-reranker-test/main/paper-11.png"
}
],
"documents": [ "documents": [
{"text": LONG_TEXT_DOC}, {"text": LONG_TEXT_DOC},
{"text": "数据提取么?为什么不用正则啊,你用正则不就全解决了么?"}, {"text": "数据提取么?为什么不用正则啊,你用正则不就全解决了么?"},
@@ -80,18 +78,10 @@ IMAGE_TEXT_TEST_DATA = {
} }
IMAGE_IMAGE_TEST_DATA = { IMAGE_IMAGE_TEST_DATA = {
"query": [ "query": [{"image": PAPER_IMAGE_URL}],
{
"image": "https://raw.githubusercontent.com/jina-ai/multimodal-reranker-test/main/paper-11.png"
}
],
"documents": [ "documents": [
{ {"image": HANDELSBLATT_IMAGE_URL},
"image": "https://raw.githubusercontent.com/jina-ai/multimodal-reranker-test/main/handelsblatt-preview.png" {"image": PAPER_IMAGE_URL},
},
{
"image": "https://raw.githubusercontent.com/jina-ai/multimodal-reranker-test/main/paper-11.png"
},
], ],
} }
@@ -99,13 +89,9 @@ TEXT_MIXED_DOCS_TEST_DATA = {
"query": [{"text": "slm markdown"}], "query": [{"text": "slm markdown"}],
"documents": [ "documents": [
{"text": LONG_TEXT_DOC}, {"text": LONG_TEXT_DOC},
{ {"image": PAPER_IMAGE_URL},
"image": "https://raw.githubusercontent.com/jina-ai/multimodal-reranker-test/main/paper-11.png"
},
{"text": "数据提取么?为什么不用正则啊,你用正则不就全解决了么?"}, {"text": "数据提取么?为什么不用正则啊,你用正则不就全解决了么?"},
{ {"image": HANDELSBLATT_IMAGE_URL},
"image": "https://raw.githubusercontent.com/jina-ai/multimodal-reranker-test/main/handelsblatt-preview.png"
},
], ],
} }
+5 -6
View File
@@ -17,6 +17,7 @@ import requests
import torch import torch
from PIL import Image, ImageChops from PIL import Image, ImageChops
from vllm.assets.base import VLLM_S3_BUCKET_URL
from vllm.multimodal.image import convert_image_mode from vllm.multimodal.image import convert_image_mode
from vllm.multimodal.inputs import PlaceholderRange from vllm.multimodal.inputs import PlaceholderRange
from vllm.multimodal.media import MediaConnector from vllm.multimodal.media import MediaConnector
@@ -30,8 +31,8 @@ TEST_IMAGE_ASSETS = [
] ]
TEST_VIDEO_URLS = [ TEST_VIDEO_URLS = [
"https://www.bogotobogo.com/python/OpenCV_Python/images/mean_shift_tracking/slow_traffic_small.mp4", f"{VLLM_S3_BUCKET_URL}/multimodal_asset/slow_traffic_small.mp4",
"https://github.com/opencv/opencv/raw/refs/tags/4.12.0/samples/data/vtest.avi", f"{VLLM_S3_BUCKET_URL}/multimodal_asset/vtest.avi",
] ]
@@ -76,8 +77,7 @@ async def test_fetch_image_base64(
connector = MediaConnector( connector = MediaConnector(
# Domain restriction should not apply to data URLs. # Domain restriction should not apply to data URLs.
allowed_media_domains=[ allowed_media_domains=[
"www.bogotobogo.com", VLLM_S3_BUCKET_URL.removeprefix("https://"),
"github.com",
] ]
) )
url_image = url_images[raw_image_url] url_image = url_images[raw_image_url]
@@ -390,8 +390,7 @@ async def test_allowed_media_domains(video_url: str, num_frames: int):
} }
}, },
allowed_media_domains=[ allowed_media_domains=[
"www.bogotobogo.com", VLLM_S3_BUCKET_URL.removeprefix("https://"),
"github.com",
], ],
) )