From 68ea76e78005bb53d9ef6b855b6c23411f5bb1e1 Mon Sep 17 00:00:00 2001 From: Isotr0py Date: Fri, 10 Jul 2026 19:17:42 +0800 Subject: [PATCH] [Misc] Remove dead code in ViT functionality test (#48220) Signed-off-by: Isotr0py --- .../test_vit_backend_functionality.py | 438 ------------------ 1 file changed, 438 deletions(-) delete mode 100644 tests/models/multimodal/generation/test_vit_backend_functionality.py diff --git a/tests/models/multimodal/generation/test_vit_backend_functionality.py b/tests/models/multimodal/generation/test_vit_backend_functionality.py deleted file mode 100644 index ad912067a1..0000000000 --- a/tests/models/multimodal/generation/test_vit_backend_functionality.py +++ /dev/null @@ -1,438 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -""" -Consolidated test for ViT attention backend functionality across multiple models. - -This test validates that each multimodal model can successfully generate outputs -using different ViT attention backends. Tests are parametrized by model and backend. -""" - -from typing import Any - -import pytest -from transformers import AutoProcessor - -from vllm import LLM, SamplingParams -from vllm.multimodal.utils import encode_image_url -from vllm.multimodal.video import sample_frames_from_video -from vllm.platforms import current_platform -from vllm.v1.attention.backends.registry import AttentionBackendEnum - -from ....utils import create_new_process_for_each_test -from ...utils import dummy_hf_overrides - -# Dots.OCR prompt from official repository -# https://github.com/rednote-hilab/dots.ocr/blob/d72d1d8c5bdd0362eb264f714cdbd1e5daa7cdff/dots_ocr/utils/prompts.py#L3 -# ruff: noqa: E501 -DOTS_OCR_PROMPT = """Please output the layout information from the PDF image, including each layout element's bbox, its category, and the corresponding text content within the bbox. - -1. Bbox format: [x1, y1, x2, y2] - -2. Layout Categories: The possible categories are ['Caption', 'Footnote', 'Formula', 'List-item', 'Page-footer', 'Page-header', 'Picture', 'Section-header', 'Table', 'Text', 'Title']. - -3. Text Extraction & Formatting Rules: - - Picture: For the 'Picture' category, the text field should be omitted. - - Formula: Format its text as LaTeX. - - Table: Format its text as HTML. - - All Others (Text, Title, etc.): Format their text as Markdown. - -4. Constraints: - - The output text must be the original text from the image, with no translation. - - All layout elements must be sorted according to human reading order. - -5. Final Output: The entire output must be a single JSON object. -""" - -VIDEO_PLACEHOLDER = "<|vision_start|><|video_pad|><|vision_end|>" - - -# Model configurations -MODEL_CONFIGS: dict[str, dict[str, Any]] = { - "dots_ocr": { - "model_name": "rednote-hilab/dots.ocr", - "interface": "llm_chat", - "max_model_len": 32768, - "max_num_seqs": 1, - "limit_mm_per_prompt": {"image": 1}, - "sampling_params": { - "temperature": 0.1, - "max_tokens": 16384, - "top_p": 0.9, - "stop_token_ids": None, - }, - "use_specific_image": "stop_sign", - "prompt_builder": "build_dots_ocr_prompt", - "output_validator": lambda x: len(x) > 10 and "stop" in x.lower(), - }, - "ernie45_vl": { - "model_name": "baidu/ERNIE-4.5-VL-28B-A3B-PT", - "interface": "llm_generate", - "max_model_len": 16384, - "max_num_seqs": 2, - "sampling_params": { - "temperature": 0.0, - "max_tokens": 256, - "stop_token_ids": None, - }, - "use_processor": True, - "question": "What is the content of each image?", - }, - "glm4_1v": { - "model_name": "zai-org/GLM-4.1V-9B-Thinking", - "interface": "llm_generate", - "max_model_len": 32768, - "max_num_seqs": 2, - "sampling_params": { - "temperature": 0.0, - "max_tokens": 256, - "stop_token_ids": None, - }, - "use_processor": True, - "question": "What is the content of each image?", - }, - "glm_ocr": { - "model_name": "zai-org/GLM-OCR", - "interface": "llm_generate", - "max_model_len": 131072, - "max_num_seqs": 2, - "sampling_params": { - "temperature": 0.0, - "max_tokens": 256, - "stop_token_ids": None, - }, - "use_processor": True, - "question": "Text Recognition:", - }, - "keye_vl": { - "model_name": "Kwai-Keye/Keye-VL-8B-Preview", - "interface": "llm_generate", - "max_model_len": 8192, - "max_num_seqs": 5, - "sampling_params": { - "temperature": 0.0, - "max_tokens": 256, - "stop_token_ids": None, - }, - "supported_backends": { - AttentionBackendEnum.FLASH_ATTN, - AttentionBackendEnum.ROCM_AITER_FA, - }, - "use_processor": True, - "question": "What is the content of each image?", - }, - "ovis2_5": { - "model_name": "AIDC-AI/Ovis2.5-2B", - "interface": "llm_generate", - "max_model_len": 8192, - "max_num_seqs": 2, - "sampling_params": { - "temperature": 0.0, - "max_tokens": 256, - "stop_token_ids": None, - }, - "prompt_builder": "build_ovis_prompt", - "question": "What is the content of each image?", - }, - "qwen2_5_vl": { - "model_name": "Qwen/Qwen2.5-VL-3B-Instruct", - "interface": "vllm_runner", - "media_type": "video", - "max_model_len": 4000, - "max_num_seqs": 1, - "limit_mm_per_prompt": {"video": 1}, - "sampling_params": { - "max_tokens": 128, - }, - "runner_kwargs": { - "runner": "generate", - "dtype": "bfloat16", - }, - "video_params": { - "num_frames": 16, - "pruning_rates": [0.0, 0.75], - }, - }, - "qwen2_5_omni": { - "model_name": "Qwen/Qwen2.5-Omni-3B", - "interface": "llm_generate", - "max_model_len": 32768, - "max_num_seqs": 2, - "limit_mm_per_prompt": {"image": 3, "video": 3, "audio": 3}, - "sampling_params": { - "temperature": 0.6, - "top_p": 0.95, - "top_k": 20, - "max_tokens": 16384, - }, - "use_processor": True, - "question": "What is the content of each image?", - }, - "qwen3_omni": { - "model_name": "Qwen/Qwen3-Omni-30B-A3B-Instruct", - "interface": "llm_generate", - "max_model_len": 32768, - "max_num_seqs": 2, - "limit_mm_per_prompt": {"image": 3, "video": 3, "audio": 3}, - "sampling_params": { - "temperature": 0.6, - "top_p": 0.95, - "top_k": 20, - "max_tokens": 16384, - }, - "use_processor": True, - "question": "What is the content of each image?", - }, -} - - -# Prompt builder functions -def build_dots_ocr_prompt(images, config): - """Build Dots.OCR specific prompt with OCR instructions.""" - # Use only stop_sign image for Dots.OCR - image = images[0] # Already filtered to stop_sign - image_url = encode_image_url(image) - - placeholders = [{"type": "image_url", "image_url": {"url": image_url}}] - messages = [ - { - "role": "user", - "content": [ - *placeholders, - { - "type": "text", - "text": f"<|img|><|imgpad|><|endofimg|>{DOTS_OCR_PROMPT}", - }, - ], - }, - ] - - return messages - - -def build_processor_prompt(images, config): - """Build prompt using AutoProcessor.apply_chat_template().""" - processor = AutoProcessor.from_pretrained( - config["model_name"], trust_remote_code=True - ) - - image_urls = [encode_image_url(img) for img in images] - placeholders = [{"type": "image", "image": url} for url in image_urls] - messages = [ - { - "role": "user", - "content": [ - *placeholders, - {"type": "text", "text": config["question"]}, - ], - }, - ] - - return processor.apply_chat_template( - messages, tokenize=False, add_generation_prompt=True - ) - - -def build_ovis_prompt(images, config): - """Build Ovis2.5 specific prompt with custom format.""" - image_urls = [encode_image_url(img) for img in images] - - placeholders = "\n".join( - f"Image-{i}: \n" for i, _ in enumerate(image_urls, start=1) - ) - - return ( - f"<|im_start|>user\n\n{placeholders}\n{config['question']}<|im_end|>\n" - "<|im_start|>assistant\n" - ) - - -def build_qwen2_5_video_prompt(): - """Build Qwen2.5-VL video prompt with EVS placeholder.""" - return ( - f"<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n" - f"<|im_start|>user\n{VIDEO_PLACEHOLDER}" - "Describe this video with a short sentence (no more than 20 words)" - "<|im_end|><|im_start|>assistant\n" - ) - - -# Handler functions -def run_llm_generate_test(config, mm_encoder_attn_backend, image_assets): - """Standard LLM.generate() interface handler.""" - images = [asset.pil_image for asset in image_assets] - - # Build prompt - if config.get("use_processor"): - prompt = build_processor_prompt(images, config) - else: - prompt_builder_name = config.get("prompt_builder", "build_ovis_prompt") - prompt_builder = globals()[prompt_builder_name] - prompt = prompt_builder(images, config) - - # Determine limit_mm_per_prompt - limit_mm_per_prompt = config.get("limit_mm_per_prompt", {"image": len(images)}) - - # Create engine - llm = LLM( - model=config["model_name"], - trust_remote_code=True, - max_model_len=config["max_model_len"], - max_num_seqs=config["max_num_seqs"], - limit_mm_per_prompt=limit_mm_per_prompt, - mm_encoder_attn_backend=mm_encoder_attn_backend, - hf_overrides=dummy_hf_overrides, - load_format="dummy", - seed=42, - ) - - # Generate - sampling_params = SamplingParams(**config["sampling_params"]) - outputs = llm.generate( - { - "prompt": prompt, - "multi_modal_data": {"image": images}, - }, - sampling_params=sampling_params, - ) - - # Validate - for o in outputs: - generated_text = o.outputs[0].text - validator = config.get("output_validator", lambda x: len(x) > 10) - assert validator(generated_text), ( - f"Validation failed for {config['model_name']}: {generated_text}" - ) - - -def run_llm_chat_test(config, mm_encoder_attn_backend, image_assets): - """LLM.chat() interface handler for Dots.OCR.""" - # Filter to stop_sign image only - stop_sign_image = [ - asset.pil_image for asset in image_assets if asset.name == "stop_sign" - ][0] - - # Build messages - messages = build_dots_ocr_prompt([stop_sign_image], config) - - # Create engine - llm = LLM( - model=config["model_name"], - trust_remote_code=True, - max_model_len=config["max_model_len"], - max_num_seqs=config["max_num_seqs"], - limit_mm_per_prompt=config["limit_mm_per_prompt"], - mm_encoder_attn_backend=mm_encoder_attn_backend, - hf_overrides=dummy_hf_overrides, - load_format="dummy", - seed=42, - ) - - # Generate using chat - sampling_params = SamplingParams(**config["sampling_params"]) - outputs = llm.chat(messages=messages, sampling_params=sampling_params) - - # Validate - for o in outputs: - generated_text = o.outputs[0].text - validator = config.get("output_validator", lambda x: len(x) > 10) - assert validator(generated_text), ( - f"Validation failed for {config['model_name']}: {generated_text}" - ) - - -def run_video_test(config, mm_encoder_attn_backend, video_assets, vllm_runner): - """Video test with EVS (Efficient Video Sampling) handler.""" - for pruning_rate in config["video_params"]["pruning_rates"]: - num_frames = config["video_params"]["num_frames"] - - # Sample frames from video - sampled_vids = [ - sample_frames_from_video(asset.np_ndarrays, num_frames) - for asset in video_assets - ] - - # Build prompt and prepare video - prompt = build_qwen2_5_video_prompt() - prompts = [prompt] - videos = [sampled_vids[0]] - - # Run with vllm_runner context manager - with vllm_runner( - config["model_name"], - max_model_len=config["max_model_len"], - max_num_seqs=config["max_num_seqs"], - limit_mm_per_prompt=config["limit_mm_per_prompt"], - tensor_parallel_size=1, - video_pruning_rate=pruning_rate, - mm_encoder_attn_backend=mm_encoder_attn_backend, - hf_overrides=dummy_hf_overrides, - load_format="dummy", - **config["runner_kwargs"], - ) as vllm_model: - outputs = vllm_model.generate_greedy( - prompts, - config["sampling_params"]["max_tokens"], - videos=videos, - ) - - # Validate output - assert len(outputs) == 1, f"Expected 1 output, got {len(outputs)}" - output_ids, output_text = outputs[0] - assert len(output_ids) > 0, "Generated no output IDs" - assert len(output_text) > 0, "Generated empty text" - assert isinstance(output_text, str), ( - f"Output is not string: {type(output_text)}" - ) - - -# Main test function -@pytest.mark.parametrize("model_key", list(MODEL_CONFIGS.keys())) -@pytest.mark.parametrize( - "mm_encoder_attn_backend", - [None] + current_platform.get_supported_vit_attn_backends(), -) -@pytest.mark.skip(reason="Broken test due to memory segmentation fault") -@create_new_process_for_each_test() -def test_vit_backend_functionality( - model_key: str, - mm_encoder_attn_backend: AttentionBackendEnum | None, - image_assets, - video_assets, - vllm_runner, - request, -): - """Test ViT attention backend functionality for multimodal models. - - This test validates that each model can successfully generate outputs - using different ViT attention backends. The test: - 1. Filters unsupported backends per model - 2. Applies appropriate GPU marks - 3. Routes to the correct test handler based on interface - 4. Validates output meets minimum requirements - """ - config = MODEL_CONFIGS[model_key] - - # Step 1: Backend filtering - if ( - "supported_backends" in config - and mm_encoder_attn_backend is not None - and mm_encoder_attn_backend not in config["supported_backends"] - ): - pytest.skip( - f"{model_key} does not support {mm_encoder_attn_backend} backend now." - ) - - # Step 2: Apply GPU marks dynamically - if "gpu_marks" in config: - for mark in config["gpu_marks"]: - request.applymarker(mark) - - # Step 3: Route to appropriate handler - if config.get("media_type") == "video": - run_video_test(config, mm_encoder_attn_backend, video_assets, vllm_runner) - elif config["interface"] == "llm_chat": - run_llm_chat_test(config, mm_encoder_attn_backend, image_assets) - elif config["interface"] == "llm_generate": - run_llm_generate_test(config, mm_encoder_attn_backend, image_assets) - else: - raise ValueError(f"Unknown interface: {config['interface']}")