Fix Nano Nemotron VL regressions (#38655)
Signed-off-by: Netanel Haber <58652339+netanel-haber@users.noreply.github.com>
This commit is contained in:
@@ -44,15 +44,19 @@ class ExtractorConfig:
|
||||
subsampling_factor: int
|
||||
subsampling_conv_kernel_size: int
|
||||
subsampling_conv_stride: int
|
||||
hop_length: int = 160
|
||||
"""Default `160`: Matches HF default"""
|
||||
clip_duration_s: int = 30
|
||||
clip_min_duration_s: float = 0.1
|
||||
|
||||
@staticmethod
|
||||
def from_hf_config(config: PretrainedConfig) -> "ExtractorConfig":
|
||||
assert isinstance(config, PretrainedConfig)
|
||||
hop_length = int(getattr(config, "hop_length", ExtractorConfig.hop_length))
|
||||
return ExtractorConfig(
|
||||
feature_size=config.num_mel_bins,
|
||||
sampling_rate=config.sampling_rate,
|
||||
hop_length=hop_length,
|
||||
subsampling_factor=config.subsampling_factor,
|
||||
subsampling_conv_kernel_size=config.subsampling_conv_kernel_size,
|
||||
subsampling_conv_stride=config.subsampling_conv_stride,
|
||||
|
||||
@@ -356,15 +356,6 @@ class DynamicResolutionImageTiler:
|
||||
feature_sizes.append(param.num_embeddings)
|
||||
return images, feature_sizes
|
||||
|
||||
feature_size_cache: dict[Image.Image, int] = {}
|
||||
|
||||
@classmethod
|
||||
def get_cached_feature_size(cls, image: Image.Image) -> int:
|
||||
feature_size = cls.feature_size_cache[id(image)]
|
||||
# hard assert that we only use the feature size once
|
||||
del cls.feature_size_cache[id(image)]
|
||||
return feature_size
|
||||
|
||||
@dataclass
|
||||
class DynamicResolutionParams:
|
||||
media: Image.Image
|
||||
@@ -519,7 +510,6 @@ class DynamicResolutionImageTiler:
|
||||
param, token_count = self.process_media(media, tokens_for_media)
|
||||
params.append(param)
|
||||
token_counts.append(token_count)
|
||||
self.feature_size_cache[id(param.media)] = param.num_embeddings
|
||||
|
||||
# Step 2: Check if total tokens is within budget
|
||||
total_tokens = sum(token_counts)
|
||||
@@ -857,13 +847,12 @@ class NanoNemotronVLProcessor(BaseNanoNemotronVLProcessor):
|
||||
|
||||
@property
|
||||
def supports_video(self) -> bool:
|
||||
return self.video_token_id is not None
|
||||
return True
|
||||
|
||||
@property
|
||||
def video_token_id(self) -> int | None:
|
||||
if self.video_token is None:
|
||||
return None
|
||||
return self.tokenizer.get_vocab().get(self.video_token, None)
|
||||
def video_token_id(self) -> int:
|
||||
assert self.video_token is not None
|
||||
return self.tokenizer.get_vocab()[self.video_token]
|
||||
|
||||
@property
|
||||
def image_token_id(self) -> int:
|
||||
@@ -1055,6 +1044,13 @@ class NanoNemotronVLProcessor(BaseNanoNemotronVLProcessor):
|
||||
text_inputs = self.tokenizer(text, add_special_tokens=False)
|
||||
|
||||
combined_inputs = {**text_inputs, **video_inputs, **audio_inputs}
|
||||
frames_indices = combined_inputs.get("frames_indices")
|
||||
ragged_frames_indices = (
|
||||
isinstance(frames_indices, list)
|
||||
and len({len(frame_indices) for frame_indices in frames_indices}) > 1
|
||||
)
|
||||
if ragged_frames_indices:
|
||||
combined_inputs.pop("frames_indices")
|
||||
|
||||
if self.dynamic_tiler is None:
|
||||
batch = BatchFeature(
|
||||
@@ -1066,6 +1062,12 @@ class NanoNemotronVLProcessor(BaseNanoNemotronVLProcessor):
|
||||
# allow images to be exempt from the BatchFeature validation:
|
||||
# We will .stack() them in _parse_and_validate_image_input
|
||||
batch.update(image_inputs)
|
||||
if ragged_frames_indices:
|
||||
assert isinstance(frames_indices, list)
|
||||
batch["frames_indices"] = [
|
||||
torch.as_tensor(frame_indices, dtype=torch.int64)
|
||||
for frame_indices in frames_indices
|
||||
]
|
||||
return batch
|
||||
|
||||
def get_image_repl(
|
||||
|
||||
Reference in New Issue
Block a user