Nemotron Nano V2 VL + EVS Video Support (#27107)
Signed-off-by: Eugene Khvedchenia <ekhvedchenia@nvidia.com> Signed-off-by: Natan Bagrov <nbagrov@nvidia.com> Signed-off-by: Roger Wang <hey@rogerw.io> Co-authored-by: Natan Bagrov <nbagrov@nvidia.com> Co-authored-by: Roger Wang <hey@rogerw.io>
This commit is contained in:
co-authored by
Natan Bagrov
Roger Wang
parent
1c691f4a71
commit
e93ff6c8b9
@@ -1735,20 +1735,32 @@ class GPUModelRunner(LoRAModelRunnerMixin, KVConnectorModelRunnerMixin):
|
||||
pin_memory=self.pin_memory,
|
||||
merge_by_field_config=model.merge_by_field_config,
|
||||
):
|
||||
curr_group_outputs = []
|
||||
|
||||
# EVS-related change.
|
||||
# (ekhvedchenia): Temporary hack to limit peak memory usage when
|
||||
# processing multimodal data.This solves the issue with scheduler
|
||||
# processing multimodal data. This solves the issue with scheduler
|
||||
# putting too many video samples into a single batch. Scheduler
|
||||
# uses pruned vision tokens count to compare it versus compute
|
||||
# budget which is incorrect (Either input media size or non-pruned
|
||||
# output vision tokens count should be considered)
|
||||
curr_group_outputs = []
|
||||
|
||||
if self.is_multimodal_pruning_enabled and modality == "video":
|
||||
micro_batch_size = 1
|
||||
for i in range(0, num_items, micro_batch_size):
|
||||
micro_batch_mm_inputs = dict(
|
||||
(k, v[i : i + micro_batch_size])
|
||||
for k, v in mm_kwargs_group.items()
|
||||
# TODO(ywang96): Fix memory profiling to take EVS into account and
|
||||
# remove this hack.
|
||||
if (
|
||||
self.is_multimodal_pruning_enabled
|
||||
and modality == "video"
|
||||
and num_items > 1
|
||||
):
|
||||
for video_mm_kwargs_item in filter(
|
||||
lambda item: item.modality == "video", mm_kwargs
|
||||
):
|
||||
_, _, micro_batch_mm_inputs = next(
|
||||
group_mm_kwargs_by_modality(
|
||||
[video_mm_kwargs_item],
|
||||
device=self.device,
|
||||
pin_memory=self.pin_memory,
|
||||
merge_by_field_config=model.merge_by_field_config,
|
||||
)
|
||||
)
|
||||
|
||||
micro_batch_outputs = model.get_multimodal_embeddings(
|
||||
|
||||
Reference in New Issue
Block a user