[Fix] Misc Fixes in ViT CUDA Graph (#38040)

Signed-off-by: Baorun Mu <bmu@nvidia.com>
This commit is contained in:
Baorun (Lauren) Mu
2026-05-14 23:49:06 +08:00
committed by GitHub
parent b8a25d0e12
commit a7737cb4f3
4 changed files with 242 additions and 21 deletions
+10 -9
View File
@@ -1768,14 +1768,12 @@ class Qwen3VLForConditionalGeneration(
EncoderCudaGraphConfig,
)
modalities = ["image"]
# NOTE: When EVS (Efficient Video Sampling) pruning is enabled, the number
# of tokens becomes data-dependent (i.e., the retained tokens are
# dynamically selected based on inter-frame differences) and therefore
# cannot be captured by CUDA Graphs. As a result, video CUDA Graphs are
# only enabled when EVS is disabled.
if not self.is_multimodal_pruning_enabled:
modalities.append("video")
# When EVS pruning is enabled, embed_multimodal post-processes both
# image and video embeddings (mrope positions are appended for image,
# prune+append for video). The encoder CUDA graph path bypasses that
# post-process, producing inconsistent embedding formats vs eager. So
# disable CUDA graph for all modalities when pruning is on.
modalities = [] if self.is_multimodal_pruning_enabled else ["image", "video"]
return EncoderCudaGraphConfig(
modalities=modalities,
@@ -1923,7 +1921,10 @@ class Qwen3VLForConditionalGeneration(
)
spatial_merge_size = self.visual.spatial_merge_size
per_mm_item_output = token_budget // max_batch_size
# Ceil so the buffer fits the worst case of one item using the full
# budget. Floor under-allocates when budget is not a multiple of
# max_batch_size.
per_mm_item_output = (token_budget + max_batch_size - 1) // max_batch_size
frames_per_item = max_frames_per_batch // max_batch_size
if frames_per_item > 1: