Nemotron Nano V2 VL + EVS Video Support (#27107)

Signed-off-by: Eugene Khvedchenia <ekhvedchenia@nvidia.com>
Signed-off-by: Natan Bagrov <nbagrov@nvidia.com>
Signed-off-by: Roger Wang <hey@rogerw.io>
Co-authored-by: Natan Bagrov <nbagrov@nvidia.com>
Co-authored-by: Roger Wang <hey@rogerw.io>
This commit is contained in:
Eugene Khvedchenya
2025-10-20 22:19:11 +08:00
committed by GitHub
co-authored by Natan Bagrov Roger Wang
parent 1c691f4a71
commit e93ff6c8b9
5 changed files with 317 additions and 106 deletions
+21 -9
View File
@@ -1735,20 +1735,32 @@ class GPUModelRunner(LoRAModelRunnerMixin, KVConnectorModelRunnerMixin):
pin_memory=self.pin_memory,
merge_by_field_config=model.merge_by_field_config,
):
curr_group_outputs = []
# EVS-related change.
# (ekhvedchenia): Temporary hack to limit peak memory usage when
# processing multimodal data.This solves the issue with scheduler
# processing multimodal data. This solves the issue with scheduler
# putting too many video samples into a single batch. Scheduler
# uses pruned vision tokens count to compare it versus compute
# budget which is incorrect (Either input media size or non-pruned
# output vision tokens count should be considered)
curr_group_outputs = []
if self.is_multimodal_pruning_enabled and modality == "video":
micro_batch_size = 1
for i in range(0, num_items, micro_batch_size):
micro_batch_mm_inputs = dict(
(k, v[i : i + micro_batch_size])
for k, v in mm_kwargs_group.items()
# TODO(ywang96): Fix memory profiling to take EVS into account and
# remove this hack.
if (
self.is_multimodal_pruning_enabled
and modality == "video"
and num_items > 1
):
for video_mm_kwargs_item in filter(
lambda item: item.modality == "video", mm_kwargs
):
_, _, micro_batch_mm_inputs = next(
group_mm_kwargs_by_modality(
[video_mm_kwargs_item],
device=self.device,
pin_memory=self.pin_memory,
merge_by_field_config=model.merge_by_field_config,
)
)
micro_batch_outputs = model.get_multimodal_embeddings(