[Fix] Misc Fixes in ViT CUDA Graph (#38040)
Signed-off-by: Baorun Mu <bmu@nvidia.com>
This commit is contained in:
@@ -1768,14 +1768,12 @@ class Qwen3VLForConditionalGeneration(
|
||||
EncoderCudaGraphConfig,
|
||||
)
|
||||
|
||||
modalities = ["image"]
|
||||
# NOTE: When EVS (Efficient Video Sampling) pruning is enabled, the number
|
||||
# of tokens becomes data-dependent (i.e., the retained tokens are
|
||||
# dynamically selected based on inter-frame differences) and therefore
|
||||
# cannot be captured by CUDA Graphs. As a result, video CUDA Graphs are
|
||||
# only enabled when EVS is disabled.
|
||||
if not self.is_multimodal_pruning_enabled:
|
||||
modalities.append("video")
|
||||
# When EVS pruning is enabled, embed_multimodal post-processes both
|
||||
# image and video embeddings (mrope positions are appended for image,
|
||||
# prune+append for video). The encoder CUDA graph path bypasses that
|
||||
# post-process, producing inconsistent embedding formats vs eager. So
|
||||
# disable CUDA graph for all modalities when pruning is on.
|
||||
modalities = [] if self.is_multimodal_pruning_enabled else ["image", "video"]
|
||||
|
||||
return EncoderCudaGraphConfig(
|
||||
modalities=modalities,
|
||||
@@ -1923,7 +1921,10 @@ class Qwen3VLForConditionalGeneration(
|
||||
)
|
||||
|
||||
spatial_merge_size = self.visual.spatial_merge_size
|
||||
per_mm_item_output = token_budget // max_batch_size
|
||||
# Ceil so the buffer fits the worst case of one item using the full
|
||||
# budget. Floor under-allocates when budget is not a multiple of
|
||||
# max_batch_size.
|
||||
per_mm_item_output = (token_budget + max_batch_size - 1) // max_batch_size
|
||||
|
||||
frames_per_item = max_frames_per_batch // max_batch_size
|
||||
if frames_per_item > 1:
|
||||
|
||||
Reference in New Issue
Block a user