[MM][Perf][CG] Support ViT full CUDA graph for Qwen3.5 (#42151)
Signed-off-by: shen-shanshan <467638484@qq.com>
This commit is contained in:
@@ -85,8 +85,9 @@ Models opt-in to encoder CUDA Graphs by implementing the [SupportsEncoderCudaGra
|
|||||||
|
|
||||||
| Architecture | Models | CG for Image | CG for Video |
|
| Architecture | Models | CG for Image | CG for Video |
|
||||||
| ------------ | ------ | ------------ | ------------ |
|
| ------------ | ------ | ------------ | ------------ |
|
||||||
| `Qwen3VLForConditionalGeneration` | `Qwen3-VL` | ✅︎ | ✅︎ |
|
|
||||||
| `Qwen2_5_VLForConditionalGeneration` | `Qwen2.5-VL` | ✅︎ | ✅︎ |
|
| `Qwen2_5_VLForConditionalGeneration` | `Qwen2.5-VL` | ✅︎ | ✅︎ |
|
||||||
|
| `Qwen3VLForConditionalGeneration` | `Qwen3-VL` | ✅︎ | ✅︎ |
|
||||||
|
| `Qwen3_5ForConditionalGeneration` | `Qwen3.5` | ✅︎ | ✅︎ |
|
||||||
|
|
||||||
!!! note
|
!!! note
|
||||||
Encoder CUDA Graphs have currently been tested with `--mm-encoder-attn-backend=FLASH_ATTN` and `--mm-encoder-attn-backend=FLASHINFER` on Blackwell GPUs.
|
Encoder CUDA Graphs have currently been tested with `--mm-encoder-attn-backend=FLASH_ATTN` and `--mm-encoder-attn-backend=FLASHINFER` on Blackwell GPUs.
|
||||||
|
|||||||
@@ -2179,6 +2179,92 @@ def run_qwen3_vl_moe(questions: list[str], modality: str) -> ModelRequestData:
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
# Qwen3.5-Dense
|
||||||
|
def run_qwen3_5(questions: list[str], modality: str) -> ModelRequestData:
|
||||||
|
model_name = "Qwen/Qwen3.5-4B"
|
||||||
|
|
||||||
|
mm_limit = {"image": 1, "video": 1} if modality == "image+video" else {modality: 1}
|
||||||
|
engine_args = EngineArgs(
|
||||||
|
model=model_name,
|
||||||
|
max_model_len=4096,
|
||||||
|
max_num_seqs=5,
|
||||||
|
mm_processor_kwargs={
|
||||||
|
"min_pixels": 28 * 28,
|
||||||
|
"max_pixels": 1280 * 28 * 28,
|
||||||
|
"fps": 1,
|
||||||
|
},
|
||||||
|
limit_mm_per_prompt=mm_limit,
|
||||||
|
)
|
||||||
|
|
||||||
|
image_placeholder = "<|vision_start|><|image_pad|><|vision_end|>"
|
||||||
|
video_placeholder = "<|vision_start|><|video_pad|><|vision_end|>"
|
||||||
|
|
||||||
|
if modality == "image":
|
||||||
|
placeholder = image_placeholder
|
||||||
|
elif modality == "video":
|
||||||
|
placeholder = video_placeholder
|
||||||
|
elif modality == "image+video":
|
||||||
|
placeholder = image_placeholder + video_placeholder
|
||||||
|
|
||||||
|
prompts = [
|
||||||
|
(
|
||||||
|
"<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n"
|
||||||
|
f"<|im_start|>user\n{placeholder}"
|
||||||
|
f"{question}<|im_end|>\n"
|
||||||
|
"<|im_start|>assistant\n"
|
||||||
|
)
|
||||||
|
for question in questions
|
||||||
|
]
|
||||||
|
|
||||||
|
return ModelRequestData(
|
||||||
|
engine_args=engine_args,
|
||||||
|
prompts=prompts,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
# Qwen3.5-MoE
|
||||||
|
def run_qwen3_5_moe(questions: list[str], modality: str) -> ModelRequestData:
|
||||||
|
model_name = "Qwen/Qwen3.5-35B-A3B"
|
||||||
|
|
||||||
|
mm_limit = {"image": 1, "video": 1} if modality == "image+video" else {modality: 1}
|
||||||
|
engine_args = EngineArgs(
|
||||||
|
model=model_name,
|
||||||
|
max_model_len=4096,
|
||||||
|
max_num_seqs=5,
|
||||||
|
mm_processor_kwargs={
|
||||||
|
"min_pixels": 28 * 28,
|
||||||
|
"max_pixels": 1280 * 28 * 28,
|
||||||
|
"fps": 1,
|
||||||
|
},
|
||||||
|
limit_mm_per_prompt=mm_limit,
|
||||||
|
)
|
||||||
|
|
||||||
|
image_placeholder = "<|vision_start|><|image_pad|><|vision_end|>"
|
||||||
|
video_placeholder = "<|vision_start|><|video_pad|><|vision_end|>"
|
||||||
|
|
||||||
|
if modality == "image":
|
||||||
|
placeholder = image_placeholder
|
||||||
|
elif modality == "video":
|
||||||
|
placeholder = video_placeholder
|
||||||
|
elif modality == "image+video":
|
||||||
|
placeholder = image_placeholder + video_placeholder
|
||||||
|
|
||||||
|
prompts = [
|
||||||
|
(
|
||||||
|
"<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n"
|
||||||
|
f"<|im_start|>user\n{placeholder}"
|
||||||
|
f"{question}<|im_end|>\n"
|
||||||
|
"<|im_start|>assistant\n"
|
||||||
|
)
|
||||||
|
for question in questions
|
||||||
|
]
|
||||||
|
|
||||||
|
return ModelRequestData(
|
||||||
|
engine_args=engine_args,
|
||||||
|
prompts=prompts,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
# R-4B
|
# R-4B
|
||||||
def run_r_vl(questions: list[str], modality: str) -> ModelRequestData:
|
def run_r_vl(questions: list[str], modality: str) -> ModelRequestData:
|
||||||
assert modality == "image"
|
assert modality == "image"
|
||||||
@@ -2442,6 +2528,8 @@ model_example_map = {
|
|||||||
"qwen2_5_omni": run_qwen2_5_omni,
|
"qwen2_5_omni": run_qwen2_5_omni,
|
||||||
"qwen3_vl": run_qwen3_vl,
|
"qwen3_vl": run_qwen3_vl,
|
||||||
"qwen3_vl_moe": run_qwen3_vl_moe,
|
"qwen3_vl_moe": run_qwen3_vl_moe,
|
||||||
|
"qwen3_5": run_qwen3_5,
|
||||||
|
"qwen3_5_moe": run_qwen3_5_moe,
|
||||||
"rvl": run_r_vl,
|
"rvl": run_r_vl,
|
||||||
"skywork_chat": run_skyworkr1v,
|
"skywork_chat": run_skyworkr1v,
|
||||||
"smolvlm": run_smolvlm,
|
"smolvlm": run_smolvlm,
|
||||||
@@ -2460,13 +2548,17 @@ MODELS_NEED_VIDEO_METADATA = [
|
|||||||
"molmo2",
|
"molmo2",
|
||||||
"qwen3_vl",
|
"qwen3_vl",
|
||||||
"qwen3_vl_moe",
|
"qwen3_vl_moe",
|
||||||
|
"qwen3_5",
|
||||||
|
"qwen3_5_moe",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
|
||||||
MODELS_SUPPORT_VIT_CUDA_GRAPH = [
|
MODELS_SUPPORT_VIT_CUDA_GRAPH = [
|
||||||
|
"qwen2_5_vl",
|
||||||
"qwen3_vl",
|
"qwen3_vl",
|
||||||
"qwen3_vl_moe",
|
"qwen3_vl_moe",
|
||||||
"qwen2_5_vl",
|
"qwen3_5",
|
||||||
|
"qwen3_5_moe",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -42,6 +42,18 @@ def qwen_vl_chat_template(content: str) -> str:
|
|||||||
|
|
||||||
|
|
||||||
MODEL_CONFIGS: dict[str, VitCudagraphTestConfig] = {
|
MODEL_CONFIGS: dict[str, VitCudagraphTestConfig] = {
|
||||||
|
"qwen2_5_vl": VitCudagraphTestConfig(
|
||||||
|
model="Qwen/Qwen2.5-VL-3B-Instruct",
|
||||||
|
image_prompt=qwen_vl_chat_template(
|
||||||
|
"<|vision_start|><|image_pad|><|vision_end|>What is in this image?"
|
||||||
|
),
|
||||||
|
video_prompt=qwen_vl_chat_template(
|
||||||
|
"<|vision_start|><|video_pad|><|vision_end|>"
|
||||||
|
"Describe this video in one sentence."
|
||||||
|
),
|
||||||
|
needs_video_metadata=False,
|
||||||
|
marks=[pytest.mark.core_model],
|
||||||
|
),
|
||||||
"qwen3_vl": VitCudagraphTestConfig(
|
"qwen3_vl": VitCudagraphTestConfig(
|
||||||
model="Qwen/Qwen3-VL-2B-Instruct",
|
model="Qwen/Qwen3-VL-2B-Instruct",
|
||||||
image_prompt=qwen_vl_chat_template(
|
image_prompt=qwen_vl_chat_template(
|
||||||
@@ -54,8 +66,8 @@ MODEL_CONFIGS: dict[str, VitCudagraphTestConfig] = {
|
|||||||
needs_video_metadata=True,
|
needs_video_metadata=True,
|
||||||
marks=[pytest.mark.core_model],
|
marks=[pytest.mark.core_model],
|
||||||
),
|
),
|
||||||
"qwen2_5_vl": VitCudagraphTestConfig(
|
"qwen3_5": VitCudagraphTestConfig(
|
||||||
model="Qwen/Qwen2.5-VL-3B-Instruct",
|
model="Qwen/Qwen3.5-0.8B",
|
||||||
image_prompt=qwen_vl_chat_template(
|
image_prompt=qwen_vl_chat_template(
|
||||||
"<|vision_start|><|image_pad|><|vision_end|>What is in this image?"
|
"<|vision_start|><|image_pad|><|vision_end|>What is in this image?"
|
||||||
),
|
),
|
||||||
@@ -63,7 +75,7 @@ MODEL_CONFIGS: dict[str, VitCudagraphTestConfig] = {
|
|||||||
"<|vision_start|><|video_pad|><|vision_end|>"
|
"<|vision_start|><|video_pad|><|vision_end|>"
|
||||||
"Describe this video in one sentence."
|
"Describe this video in one sentence."
|
||||||
),
|
),
|
||||||
needs_video_metadata=False,
|
needs_video_metadata=True,
|
||||||
marks=[pytest.mark.core_model],
|
marks=[pytest.mark.core_model],
|
||||||
),
|
),
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -565,6 +565,7 @@ class Qwen3_5ForConditionalGeneration(Qwen3VLForConditionalGeneration, IsHybrid)
|
|||||||
multimodal_config = vllm_config.model_config.multimodal_config
|
multimodal_config = vllm_config.model_config.multimodal_config
|
||||||
|
|
||||||
self.config = config
|
self.config = config
|
||||||
|
self.model_config = vllm_config.model_config
|
||||||
self.multimodal_config = multimodal_config
|
self.multimodal_config = multimodal_config
|
||||||
self.use_data_parallel = multimodal_config.mm_encoder_tp_mode == "data"
|
self.use_data_parallel = multimodal_config.mm_encoder_tp_mode == "data"
|
||||||
# Qwen3.5 does not support multimodal pruning (EVS).
|
# Qwen3.5 does not support multimodal pruning (EVS).
|
||||||
@@ -778,6 +779,7 @@ class Qwen3_5MoeForConditionalGeneration(
|
|||||||
multimodal_config = vllm_config.model_config.multimodal_config
|
multimodal_config = vllm_config.model_config.multimodal_config
|
||||||
|
|
||||||
self.config = config
|
self.config = config
|
||||||
|
self.model_config = vllm_config.model_config
|
||||||
self.multimodal_config = multimodal_config
|
self.multimodal_config = multimodal_config
|
||||||
self.use_data_parallel = multimodal_config.mm_encoder_tp_mode == "data"
|
self.use_data_parallel = multimodal_config.mm_encoder_tp_mode == "data"
|
||||||
# Qwen3.5 does not support multimodal pruning (EVS).
|
# Qwen3.5 does not support multimodal pruning (EVS).
|
||||||
|
|||||||
Reference in New Issue
Block a user