Support temporal compression for Nemotron-3-VL videos (#36808)

Signed-off-by: Collin McCarthy <cmccarthy@nvidia.com>
This commit is contained in:
Collin McCarthy
2026-03-19 08:02:19 +00:00
committed by GitHub
parent d3cc379567
commit 0b6d52629f
5 changed files with 552 additions and 129 deletions
+13 -3
View File
@@ -420,8 +420,9 @@ class GPUModelRunner(
self.is_multimodal_raw_input_only_model = (
model_config.is_multimodal_raw_input_only_model
)
# This will be overridden in load_model()
# These will be overridden in load_model()
self.is_multimodal_pruning_enabled = False
self.requires_sequential_video_encoding = False
# Set to True after init_routed_experts_capturer() completes.
# Prevents routed experts code from running during profiling/dummy run.
self.routed_experts_initialized = False
@@ -2625,17 +2626,23 @@ class GPUModelRunner(
):
batch_outputs: MultiModalEmbeddings
# EVS-related change.
# EVS and dynamic res video related change.
# (ekhvedchenia): Temporary hack to limit peak memory usage when
# processing multimodal data. This solves the issue with scheduler
# putting too many video samples into a single batch. Scheduler
# uses pruned vision tokens count to compare it versus compute
# budget which is incorrect (Either input media size or non-pruned
# output vision tokens count should be considered)
# dynamic res video for nemotron temporarily uses this hack via
# requires_sequential_video_encoding
# because it doesn't yet support video batching.
# TODO(ywang96): Fix memory profiling to take EVS into account and
# remove this hack.
if (
self.is_multimodal_pruning_enabled
(
self.is_multimodal_pruning_enabled
or self.requires_sequential_video_encoding
)
and modality == "video"
and num_items > 1
):
@@ -4609,6 +4616,9 @@ class GPUModelRunner(
and mm_config is not None
and mm_config.is_multimodal_pruning_enabled()
)
self.requires_sequential_video_encoding = hasattr(
self.get_model(), "requires_sequential_video_encoding"
) # Temporary hack for dynamic res video w/o support for bs>1 yet
if (
is_mixture_of_experts(self.model)