[ROCm][CI] Gate incompatible HF references on Transformers v5 (#41532)

Signed-off-by: Andreas Karatzas <akaratza@amd.com>
This commit is contained in:
Andreas Karatzas
2026-06-16 20:34:11 +08:00
committed by GitHub
parent 405c7cf283
commit 7e179e4bc0
10 changed files with 217 additions and 145 deletions
+27 -29
View File
@@ -202,7 +202,7 @@ class AudioFlamingo3ProcessingInfo(BaseProcessingInfo):
)
def get_supported_mm_limits(self) -> Mapping[str, int | None]:
return {"audio": None}
return {"audio": 1}
class AudioFlamingo3DummyInputsBuilder(
@@ -379,32 +379,37 @@ class AudioFlamingo3MultiModalProcessor(
mm_kwargs: Mapping[str, Any],
tok_kwargs: Mapping[str, object],
) -> BatchFeature:
audios = mm_data.pop("audios", [])
if audios:
mm_data["audio"] = audios
processor_mm_data = dict(mm_data)
audios = processor_mm_data.pop("audios", None)
if audios is not None:
processor_mm_data["audio"] = audios
if not mm_data.get("audio", []):
prompt_ids = self.info.get_tokenizer().encode(prompt)
prompt_ids = self._apply_hf_processor_tokens_only(prompt_ids)
return BatchFeature(dict(input_ids=[prompt_ids]), tensor_type="pt")
outputs = super()._call_hf_processor(
prompt=prompt,
mm_data=processor_mm_data,
mm_kwargs=mm_kwargs,
tok_kwargs=tok_kwargs,
)
if "input_features_mask" in outputs:
outputs["feature_attention_mask"] = outputs.pop("input_features_mask")
audio_data = processor_mm_data.get("audio")
if audio_data is None:
return outputs
audio_list = audio_data if isinstance(audio_data, list) else [audio_data]
if len(audio_list) == 0:
return outputs
processor = self.info.get_hf_processor(**mm_kwargs)
feature_extractor = processor.feature_extractor
mm_kwargs = dict(
**mm_kwargs,
sampling_rate=feature_extractor.sampling_rate,
)
audio_list = mm_data.get("audio")
if not isinstance(audio_list, list):
audio_list = [audio_list]
chunk_counts = []
sampling_rate = feature_extractor.sampling_rate
chunk_length = feature_extractor.chunk_length
window_size = int(sampling_rate * chunk_length)
max_windows = int(processor.max_audio_len // chunk_length)
chunk_counts = []
for audio in audio_list:
# audio is numpy array or list
n_samples = len(audio) if isinstance(audio, list) else audio.shape[0]
@@ -414,18 +419,7 @@ class AudioFlamingo3MultiModalProcessor(
n_win = max_windows
chunk_counts.append(n_win)
outputs = super()._call_hf_processor(
prompt=prompt,
mm_data=mm_data,
mm_kwargs=mm_kwargs,
tok_kwargs=tok_kwargs,
)
if "input_features_mask" in outputs:
outputs["feature_attention_mask"] = outputs.pop("input_features_mask")
outputs["chunk_counts"] = torch.tensor(chunk_counts, dtype=torch.long)
return outputs
def _get_mm_fields_config(
@@ -611,6 +605,10 @@ class AudioFlamingo3ForConditionalGeneration(
input_features: torch.Tensor,
feature_attention_mask: torch.Tensor,
) -> torch.Tensor:
input_features = input_features.to(
dtype=self.audio_tower.conv1.weight.dtype,
device=self.audio_tower.conv1.weight.device,
)
audio_attention_mask = _build_audio_encoder_attention_mask(
feature_attention_mask,
dtype=self.audio_tower.conv1.weight.dtype,
+67 -60
View File
@@ -120,9 +120,11 @@ class MusicFlamingoRotaryEmbedding(nn.Module):
) -> tuple["torch.Tensor", float]:
del seq_len
base = config.rope_parameters["rope_theta"]
dim = getattr(config, "head_dim", None) or (
partial_rotary_factor = config.rope_parameters.get("partial_rotary_factor", 1.0)
head_dim = getattr(config, "head_dim", None) or (
config.hidden_size // config.num_attention_heads
)
dim = int(head_dim * partial_rotary_factor)
attention_factor = 1.0
inv_freq = 1.0 / (
@@ -148,21 +150,40 @@ class MusicFlamingoRotaryEmbedding(nn.Module):
position_angles = torch.repeat_interleave(position_angles, 2, dim=-1)
return position_angles.to(dtype=inv_freq.dtype)
def _restore_fp32_rope_buffers(self) -> None:
rope_init_fn: Callable = self.compute_default_rope_parameters
if self.rope_type != "default":
rope_init_fn = ROPE_INIT_FUNCTIONS[self.rope_type]
inv_freq, self.attention_scaling = rope_init_fn(
self.config, self.inv_freq.device
)
self.inv_freq = inv_freq
self.original_inv_freq = inv_freq.clone()
self.position_angles = self._compute_position_angles(inv_freq)
def _apply(self, fn):
super()._apply(fn)
self._restore_fp32_rope_buffers()
return self
@torch.no_grad()
def forward(self, timestamps: Tensor, seq_len: int) -> tuple[Tensor, Tensor]:
batch_positions = torch.arange(
timestamps.shape[0],
window_starts = timestamps[:, 0].to(
device=self.inv_freq.device,
dtype=self.inv_freq.dtype,
)
batch_positions = batch_positions / self.max_seq_len_cached
batch_freqs = batch_positions.unsqueeze(-1) * self.inv_freq
batch_freqs = torch.repeat_interleave(batch_freqs, 2, dim=-1)
window_duration = self.config.audio_frame_step * 4 * seq_len
window_positions = (
torch.round(window_starts / window_duration) / self.max_seq_len_cached
)
window_freqs = window_positions.unsqueeze(-1) * self.inv_freq
window_freqs = torch.repeat_interleave(window_freqs, 2, dim=-1)
batch_freqs = batch_freqs[:, None, :]
window_freqs = window_freqs[:, None, :]
time_freqs = self.position_angles[:seq_len][None, :, :]
batch_freqs, time_freqs = broadcast_tensors(batch_freqs, time_freqs)
freqs = torch.cat((batch_freqs, time_freqs), dim=-1)
window_freqs, time_freqs = broadcast_tensors(window_freqs, time_freqs)
freqs = torch.cat((window_freqs, time_freqs), dim=-1)
angle = (-timestamps * 2 * pi).to(freqs)
freqs = freqs * angle.unsqueeze(-1)
return freqs.cos(), freqs.sin()
@@ -170,7 +191,7 @@ class MusicFlamingoRotaryEmbedding(nn.Module):
class MusicFlamingoFeatureInputs(AudioFlamingo3FeatureInputs):
rote_timestamps: Annotated[
torch.Tensor,
torch.Tensor | None,
TensorShape(
"num_chunks",
"num_audio_time_steps",
@@ -205,6 +226,7 @@ class MusicFlamingoProcessingInfo(AudioFlamingo3ProcessingInfo):
feature_extractor = self.get_feature_extractor()
return MusicFlamingoMultiModalDataParser(
target_sr=feature_extractor.sampling_rate,
audio_resample_method="soxr",
expected_hidden_size=self._get_expected_hidden_size(),
)
@@ -268,49 +290,6 @@ class MusicFlamingoMultiModalDataParser(AudioFlamingo3MultiModalDataParser):
class MusicFlamingoMultiModalProcessor(AudioFlamingo3MultiModalProcessor):
def _call_hf_processor(
self,
prompt: str,
mm_data: dict[str, object],
mm_kwargs: Mapping[str, Any],
tok_kwargs: Mapping[str, object],
) -> BatchFeature:
outputs = super()._call_hf_processor(
prompt=prompt,
mm_data=mm_data,
mm_kwargs=mm_kwargs,
tok_kwargs=tok_kwargs,
)
audio_data = mm_data.get("audio")
if audio_data is None:
return outputs
audio_list = audio_data if isinstance(audio_data, list) else [audio_data]
if len(audio_list) == 0:
return outputs
processor = self.info.get_hf_processor(**mm_kwargs)
feature_extractor = processor.feature_extractor
sampling_rate = feature_extractor.sampling_rate
chunk_length = feature_extractor.chunk_length
window_size = int(sampling_rate * chunk_length)
max_windows = int(processor.max_audio_len // chunk_length)
chunk_counts = []
for audio in audio_list:
n_samples = len(audio) if isinstance(audio, list) else audio.shape[0]
n_win = max(1, (n_samples + window_size - 1) // window_size)
chunk_counts.append(min(n_win, max_windows))
outputs["chunk_counts"] = torch.tensor(chunk_counts, dtype=torch.long)
if "rote_timestamps" not in outputs:
raise KeyError(
"MusicFlamingoProcessor output must include `rote_timestamps`."
)
return outputs
def _get_mm_fields_config(
self,
hf_inputs: BatchFeature,
@@ -405,6 +384,32 @@ class MusicFlamingoForConditionalGeneration(AudioFlamingo3ForConditionalGenerati
rote_timestamps=rote_timestamps,
)
def _build_audio_timestamps(
self,
chunk_counts: list[int],
seq_len: int,
device: torch.device,
) -> torch.Tensor:
audio_embed_frame_step = self.config.audio_frame_step * 4
frame_offsets = (
torch.arange(seq_len, device=device, dtype=torch.float32)
* audio_embed_frame_step
)
if not chunk_counts:
return frame_offsets.new_empty((0, seq_len))
window_indices = torch.cat(
[
torch.arange(count, device=device, dtype=torch.float32)
for count in chunk_counts
]
)
return (
window_indices.unsqueeze(1) * seq_len * audio_embed_frame_step
+ frame_offsets
)
def _process_audio_input(
self, audio_input: MusicFlamingoInputs
) -> torch.Tensor | tuple[torch.Tensor, ...]:
@@ -412,13 +417,6 @@ class MusicFlamingoForConditionalGeneration(AudioFlamingo3ForConditionalGenerati
return super()._process_audio_input(audio_input)
rote_timestamps = audio_input["rote_timestamps"]
if rote_timestamps is None:
raise ValueError(
"MusicFlamingo audio feature inputs must include `rote_timestamps`."
)
if isinstance(rote_timestamps, list):
rote_timestamps = torch.cat(rote_timestamps, dim=0)
(
input_features,
feature_attention_mask,
@@ -428,6 +426,15 @@ class MusicFlamingoForConditionalGeneration(AudioFlamingo3ForConditionalGenerati
input_features,
feature_attention_mask,
)
if rote_timestamps is None:
rote_timestamps = self._build_audio_timestamps(
chunk_counts,
seq_len=hidden_states.shape[-2],
device=hidden_states.device,
)
elif isinstance(rote_timestamps, list):
rote_timestamps = torch.cat(rote_timestamps, dim=0)
cos, sin = self.pos_emb(
rote_timestamps.to(hidden_states.device),
seq_len=hidden_states.shape[-2],