diff --git a/docs/design/cuda_graphs_multimodal.md b/docs/design/cuda_graphs_multimodal.md index f44ef359df..74f244d02f 100644 --- a/docs/design/cuda_graphs_multimodal.md +++ b/docs/design/cuda_graphs_multimodal.md @@ -85,8 +85,9 @@ Models opt-in to encoder CUDA Graphs by implementing the [SupportsEncoderCudaGra | Architecture | Models | CG for Image | CG for Video | | ------------ | ------ | ------------ | ------------ | -| `Qwen3VLForConditionalGeneration` | `Qwen3-VL` | ✅︎ | ✅︎ | | `Qwen2_5_VLForConditionalGeneration` | `Qwen2.5-VL` | ✅︎ | ✅︎ | +| `Qwen3VLForConditionalGeneration` | `Qwen3-VL` | ✅︎ | ✅︎ | +| `Qwen3_5ForConditionalGeneration` | `Qwen3.5` | ✅︎ | ✅︎ | !!! note Encoder CUDA Graphs have currently been tested with `--mm-encoder-attn-backend=FLASH_ATTN` and `--mm-encoder-attn-backend=FLASHINFER` on Blackwell GPUs. diff --git a/examples/generate/multimodal/vision_language_offline.py b/examples/generate/multimodal/vision_language_offline.py index 794f20dd0a..694467d7c5 100644 --- a/examples/generate/multimodal/vision_language_offline.py +++ b/examples/generate/multimodal/vision_language_offline.py @@ -2179,6 +2179,92 @@ def run_qwen3_vl_moe(questions: list[str], modality: str) -> ModelRequestData: ) +# Qwen3.5-Dense +def run_qwen3_5(questions: list[str], modality: str) -> ModelRequestData: + model_name = "Qwen/Qwen3.5-4B" + + mm_limit = {"image": 1, "video": 1} if modality == "image+video" else {modality: 1} + engine_args = EngineArgs( + model=model_name, + max_model_len=4096, + max_num_seqs=5, + mm_processor_kwargs={ + "min_pixels": 28 * 28, + "max_pixels": 1280 * 28 * 28, + "fps": 1, + }, + limit_mm_per_prompt=mm_limit, + ) + + image_placeholder = "<|vision_start|><|image_pad|><|vision_end|>" + video_placeholder = "<|vision_start|><|video_pad|><|vision_end|>" + + if modality == "image": + placeholder = image_placeholder + elif modality == "video": + placeholder = video_placeholder + elif modality == "image+video": + placeholder = image_placeholder + video_placeholder + + prompts = [ + ( + "<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n" + f"<|im_start|>user\n{placeholder}" + f"{question}<|im_end|>\n" + "<|im_start|>assistant\n" + ) + for question in questions + ] + + return ModelRequestData( + engine_args=engine_args, + prompts=prompts, + ) + + +# Qwen3.5-MoE +def run_qwen3_5_moe(questions: list[str], modality: str) -> ModelRequestData: + model_name = "Qwen/Qwen3.5-35B-A3B" + + mm_limit = {"image": 1, "video": 1} if modality == "image+video" else {modality: 1} + engine_args = EngineArgs( + model=model_name, + max_model_len=4096, + max_num_seqs=5, + mm_processor_kwargs={ + "min_pixels": 28 * 28, + "max_pixels": 1280 * 28 * 28, + "fps": 1, + }, + limit_mm_per_prompt=mm_limit, + ) + + image_placeholder = "<|vision_start|><|image_pad|><|vision_end|>" + video_placeholder = "<|vision_start|><|video_pad|><|vision_end|>" + + if modality == "image": + placeholder = image_placeholder + elif modality == "video": + placeholder = video_placeholder + elif modality == "image+video": + placeholder = image_placeholder + video_placeholder + + prompts = [ + ( + "<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n" + f"<|im_start|>user\n{placeholder}" + f"{question}<|im_end|>\n" + "<|im_start|>assistant\n" + ) + for question in questions + ] + + return ModelRequestData( + engine_args=engine_args, + prompts=prompts, + ) + + # R-4B def run_r_vl(questions: list[str], modality: str) -> ModelRequestData: assert modality == "image" @@ -2442,6 +2528,8 @@ model_example_map = { "qwen2_5_omni": run_qwen2_5_omni, "qwen3_vl": run_qwen3_vl, "qwen3_vl_moe": run_qwen3_vl_moe, + "qwen3_5": run_qwen3_5, + "qwen3_5_moe": run_qwen3_5_moe, "rvl": run_r_vl, "skywork_chat": run_skyworkr1v, "smolvlm": run_smolvlm, @@ -2460,13 +2548,17 @@ MODELS_NEED_VIDEO_METADATA = [ "molmo2", "qwen3_vl", "qwen3_vl_moe", + "qwen3_5", + "qwen3_5_moe", ] MODELS_SUPPORT_VIT_CUDA_GRAPH = [ + "qwen2_5_vl", "qwen3_vl", "qwen3_vl_moe", - "qwen2_5_vl", + "qwen3_5", + "qwen3_5_moe", ] diff --git a/tests/models/multimodal/generation/test_vit_cudagraph.py b/tests/models/multimodal/generation/test_vit_cudagraph.py index fb7bdfc862..8ea19c2c07 100644 --- a/tests/models/multimodal/generation/test_vit_cudagraph.py +++ b/tests/models/multimodal/generation/test_vit_cudagraph.py @@ -42,6 +42,18 @@ def qwen_vl_chat_template(content: str) -> str: MODEL_CONFIGS: dict[str, VitCudagraphTestConfig] = { + "qwen2_5_vl": VitCudagraphTestConfig( + model="Qwen/Qwen2.5-VL-3B-Instruct", + image_prompt=qwen_vl_chat_template( + "<|vision_start|><|image_pad|><|vision_end|>What is in this image?" + ), + video_prompt=qwen_vl_chat_template( + "<|vision_start|><|video_pad|><|vision_end|>" + "Describe this video in one sentence." + ), + needs_video_metadata=False, + marks=[pytest.mark.core_model], + ), "qwen3_vl": VitCudagraphTestConfig( model="Qwen/Qwen3-VL-2B-Instruct", image_prompt=qwen_vl_chat_template( @@ -54,8 +66,8 @@ MODEL_CONFIGS: dict[str, VitCudagraphTestConfig] = { needs_video_metadata=True, marks=[pytest.mark.core_model], ), - "qwen2_5_vl": VitCudagraphTestConfig( - model="Qwen/Qwen2.5-VL-3B-Instruct", + "qwen3_5": VitCudagraphTestConfig( + model="Qwen/Qwen3.5-0.8B", image_prompt=qwen_vl_chat_template( "<|vision_start|><|image_pad|><|vision_end|>What is in this image?" ), @@ -63,7 +75,7 @@ MODEL_CONFIGS: dict[str, VitCudagraphTestConfig] = { "<|vision_start|><|video_pad|><|vision_end|>" "Describe this video in one sentence." ), - needs_video_metadata=False, + needs_video_metadata=True, marks=[pytest.mark.core_model], ), } diff --git a/vllm/model_executor/models/qwen3_5.py b/vllm/model_executor/models/qwen3_5.py index dc761e69b4..57dd18b4af 100644 --- a/vllm/model_executor/models/qwen3_5.py +++ b/vllm/model_executor/models/qwen3_5.py @@ -565,6 +565,7 @@ class Qwen3_5ForConditionalGeneration(Qwen3VLForConditionalGeneration, IsHybrid) multimodal_config = vllm_config.model_config.multimodal_config self.config = config + self.model_config = vllm_config.model_config self.multimodal_config = multimodal_config self.use_data_parallel = multimodal_config.mm_encoder_tp_mode == "data" # Qwen3.5 does not support multimodal pruning (EVS). @@ -778,6 +779,7 @@ class Qwen3_5MoeForConditionalGeneration( multimodal_config = vllm_config.model_config.multimodal_config self.config = config + self.model_config = vllm_config.model_config self.multimodal_config = multimodal_config self.use_data_parallel = multimodal_config.mm_encoder_tp_mode == "data" # Qwen3.5 does not support multimodal pruning (EVS).