[ROCm][CI] Gate incompatible HF references on Transformers v5 (#41532)
Signed-off-by: Andreas Karatzas <akaratza@amd.com>
This commit is contained in:
@@ -1240,6 +1240,7 @@ setup(
|
||||
"av",
|
||||
"scipy",
|
||||
"soundfile",
|
||||
"soxr",
|
||||
"mistral_common[audio]",
|
||||
], # Required for audio processing
|
||||
"video": [], # Kept for backwards compatibility
|
||||
|
||||
@@ -148,7 +148,11 @@ def test_models(
|
||||
"def add(a, b):\n return a + b\n\ndef sub(a, b):\n return a - "
|
||||
)
|
||||
|
||||
with hf_runner(model) as hf_model:
|
||||
with hf_runner(
|
||||
model,
|
||||
revision=model_info.revision,
|
||||
trust_remote_code=model_info.trust_remote_code,
|
||||
) as hf_model:
|
||||
hf_outputs = hf_model.generate_greedy_logprobs_limit(
|
||||
example_prompts, max_tokens, num_logprobs
|
||||
)
|
||||
@@ -184,6 +188,7 @@ def test_models(
|
||||
model,
|
||||
tokenizer_name=model_info.tokenizer or model,
|
||||
tokenizer_mode=model_info.tokenizer_mode,
|
||||
revision=model_info.revision,
|
||||
trust_remote_code=model_info.trust_remote_code,
|
||||
# Remove the effects of batch variance on ROCm since batch invariance
|
||||
# is not yet supported.
|
||||
|
||||
@@ -59,6 +59,12 @@ def get_fixture_path(filename):
|
||||
)
|
||||
|
||||
|
||||
def load_expected_fixture(filename):
|
||||
fixture_path = get_fixture_path(filename)
|
||||
with open(fixture_path) as f:
|
||||
return json.load(f)
|
||||
|
||||
|
||||
def assert_output_matches(output, expected_text, expected_token_ids):
|
||||
generated = output.outputs[0]
|
||||
assert generated.text == expected_text
|
||||
@@ -76,7 +82,7 @@ def llm():
|
||||
model_info.check_transformers_version(on_fail="skip")
|
||||
|
||||
try:
|
||||
return LLM(
|
||||
llm = LLM(
|
||||
model=MODEL_NAME,
|
||||
dtype="bfloat16",
|
||||
enforce_eager=True,
|
||||
@@ -86,14 +92,19 @@ def llm():
|
||||
except Exception as e:
|
||||
pytest.skip(f"Failed to load model {MODEL_NAME}: {e}")
|
||||
|
||||
# ROCm may compile decoder kernels on the first inference pass; warm up
|
||||
# once so exact fixture assertions cover the steady-state path.
|
||||
llm.chat(
|
||||
messages=SINGLE_CONVERSATION,
|
||||
sampling_params=SamplingParams(temperature=0.0, max_tokens=1),
|
||||
use_tqdm=False,
|
||||
)
|
||||
|
||||
return llm
|
||||
|
||||
|
||||
def test_single_generation(llm):
|
||||
fixture_path = get_fixture_path("expected_results_single.json")
|
||||
if not os.path.exists(fixture_path):
|
||||
pytest.skip(f"Fixture not found: {fixture_path}")
|
||||
|
||||
with open(fixture_path) as f:
|
||||
expected = json.load(f)
|
||||
expected = load_expected_fixture("expected_results_single.json")
|
||||
|
||||
outputs = llm.chat(
|
||||
messages=SINGLE_CONVERSATION,
|
||||
@@ -108,12 +119,7 @@ def test_single_generation(llm):
|
||||
|
||||
|
||||
def test_batched_generation(llm):
|
||||
fixture_path = get_fixture_path("expected_results_batched.json")
|
||||
if not os.path.exists(fixture_path):
|
||||
pytest.skip(f"Fixture not found: {fixture_path}")
|
||||
|
||||
with open(fixture_path) as f:
|
||||
expected = json.load(f)
|
||||
expected = load_expected_fixture("expected_results_batched.json")
|
||||
|
||||
outputs = llm.chat(
|
||||
messages=BATCHED_CONVERSATIONS,
|
||||
|
||||
@@ -42,15 +42,30 @@ class MockAudioFlamingo3Processor:
|
||||
self.audio_token_id = 12345
|
||||
self.max_audio_len = 60
|
||||
self.feature_extractor = MockFeatureExtractor()
|
||||
self.tokenizer = self._tokenize
|
||||
|
||||
def __call__(self, text=None, audios=None, **kwargs):
|
||||
return {"input_ids": [1, 2, 3], "input_features": [np.zeros((3000, 80))]}
|
||||
def __call__(self, text=None, audio=None, **kwargs):
|
||||
return {
|
||||
"input_ids": torch.tensor([[1, 2, 3]], dtype=torch.long),
|
||||
"input_features": torch.zeros((3, 80, 3000)),
|
||||
"input_features_mask": torch.ones((3, 3000), dtype=torch.long),
|
||||
}
|
||||
|
||||
def _tokenize(self, text, **kwargs):
|
||||
return {"input_ids": torch.tensor([[1, 2, 3]], dtype=torch.long)}
|
||||
|
||||
|
||||
class MockFeatureExtractor:
|
||||
def __init__(self):
|
||||
self.sampling_rate = 16000
|
||||
self.chunk_length = 30
|
||||
self.hop_length = 160
|
||||
|
||||
def __call__(self, audios, **kwargs):
|
||||
return {
|
||||
"input_features": torch.zeros((len(audios), 80, 3000)),
|
||||
"attention_mask": torch.ones((len(audios), 3000), dtype=torch.long),
|
||||
}
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
@@ -60,6 +75,9 @@ def mock_ctx():
|
||||
ctx = MagicMock()
|
||||
ctx.get_hf_config.return_value = config
|
||||
ctx.get_hf_processor.return_value = MockAudioFlamingo3Processor()
|
||||
ctx.call_hf_processor.side_effect = lambda processor, data, kwargs: processor(
|
||||
**data, **kwargs
|
||||
)
|
||||
ctx.model_config.hf_config = config
|
||||
return ctx
|
||||
|
||||
@@ -89,21 +107,14 @@ def test_audio_chunk_counting(mock_ctx):
|
||||
mm_data = {"audio": [audio_1, audio_2]}
|
||||
prompt = "<|user|>Listen.<|end|>"
|
||||
|
||||
from vllm.multimodal.processing import BaseMultiModalProcessor
|
||||
processed = processor._call_hf_processor(prompt, mm_data, {}, {})
|
||||
|
||||
def mock_base_call(self, prompt, mm_data, mm_kwargs, tok_kwargs):
|
||||
return {"input_ids": [1, 2, 3], "input_features": torch.randn(1, 80, 3000)}
|
||||
chunk_counts = processed["chunk_counts"]
|
||||
|
||||
with pytest.MonkeyPatch.context() as mp:
|
||||
mp.setattr(BaseMultiModalProcessor, "_call_hf_processor", mock_base_call)
|
||||
|
||||
processed = processor._call_hf_processor(prompt, mm_data, {}, {})
|
||||
|
||||
chunk_counts = processed["chunk_counts"]
|
||||
|
||||
assert chunk_counts[0].item() == 1
|
||||
assert chunk_counts[1].item() == 2
|
||||
assert len(chunk_counts) == 2
|
||||
assert chunk_counts[0].item() == 1
|
||||
assert chunk_counts[1].item() == 2
|
||||
assert len(chunk_counts) == 2
|
||||
assert processed["feature_attention_mask"].shape == (3, 3000)
|
||||
|
||||
|
||||
def test_dummy_data_generation(mock_ctx):
|
||||
|
||||
@@ -49,6 +49,13 @@ class MockMusicFlamingoProcessor:
|
||||
self.max_audio_len = 1200
|
||||
self.feature_extractor = MockFeatureExtractor()
|
||||
|
||||
def __call__(self, text=None, audio=None, **kwargs):
|
||||
return {
|
||||
"input_ids": torch.tensor([[1, 2, 3]], dtype=torch.long),
|
||||
"input_features": torch.zeros((3, 80, 3000)),
|
||||
"input_features_mask": torch.ones((3, 3000), dtype=torch.long),
|
||||
}
|
||||
|
||||
|
||||
class MockFeatureExtractor:
|
||||
def __init__(self):
|
||||
@@ -63,6 +70,9 @@ def mock_ctx():
|
||||
ctx = MagicMock()
|
||||
ctx.get_hf_config.return_value = config
|
||||
ctx.get_hf_processor.return_value = MockMusicFlamingoProcessor()
|
||||
ctx.call_hf_processor.side_effect = lambda processor, data, kwargs: processor(
|
||||
**data, **kwargs
|
||||
)
|
||||
ctx.model_config.hf_config = config
|
||||
return ctx
|
||||
|
||||
@@ -73,7 +83,7 @@ def check_transformers_version():
|
||||
model_info.check_transformers_version(on_fail="skip")
|
||||
|
||||
|
||||
def test_musicflamingo_chunk_counting_uses_rote_timestamps(mock_ctx, monkeypatch):
|
||||
def test_musicflamingo_chunk_counting_without_rote_timestamps(mock_ctx):
|
||||
from vllm.model_executor.models.musicflamingo import (
|
||||
MusicFlamingoDummyInputsBuilder,
|
||||
MusicFlamingoMultiModalProcessor,
|
||||
@@ -92,24 +102,13 @@ def test_musicflamingo_chunk_counting_uses_rote_timestamps(mock_ctx, monkeypatch
|
||||
mm_data = {"audio": [audio_1, audio_2]}
|
||||
prompt = "<|user|>Listen.<|end|>"
|
||||
|
||||
from vllm.multimodal.processing import BaseMultiModalProcessor
|
||||
|
||||
def mock_base_call(self, prompt, mm_data, mm_kwargs, tok_kwargs):
|
||||
del self, prompt, mm_data, mm_kwargs, tok_kwargs
|
||||
return {
|
||||
"input_ids": [1, 2, 3],
|
||||
"input_features": torch.randn(3, 80, 3000),
|
||||
"rote_timestamps": torch.randn(3, 750),
|
||||
}
|
||||
|
||||
monkeypatch.setattr(BaseMultiModalProcessor, "_call_hf_processor", mock_base_call)
|
||||
|
||||
processed = processor._call_hf_processor(prompt, mm_data, {}, {})
|
||||
|
||||
chunk_counts = processed["chunk_counts"]
|
||||
|
||||
assert chunk_counts.tolist() == [1, 2]
|
||||
assert "rote_timestamps" in processed
|
||||
assert "rote_timestamps" not in processed
|
||||
assert processed["feature_attention_mask"].shape == (3, 3000)
|
||||
|
||||
|
||||
def test_musicflamingo_dummy_text_uses_plain_audio_tokens(mock_ctx):
|
||||
|
||||
+24
-10
@@ -145,22 +145,28 @@ class _HfExamplesInfo:
|
||||
# Only check the base version for the min/max version, otherwise preview
|
||||
# models cannot be run because `x.yy.0.dev0`<`x.yy.0`
|
||||
if min_version and Version(cur_base_version) < Version(min_version):
|
||||
is_version_valid = not check_min_version
|
||||
is_version_valid = False
|
||||
should_check_version = check_min_version
|
||||
msg += f">={min_version}` is required to run this model."
|
||||
elif max_version and Version(cur_base_version) > Version(max_version):
|
||||
is_version_valid = not check_max_version
|
||||
is_version_valid = False
|
||||
should_check_version = check_max_version
|
||||
msg += f"<={max_version}` is required to run this model."
|
||||
else:
|
||||
is_version_valid = True
|
||||
should_check_version = False
|
||||
|
||||
# check if Transformers version breaks the corresponding model runner,
|
||||
# skip test when model runner not compatible
|
||||
is_reason_valid = not (
|
||||
check_version_reason
|
||||
and self.transformers_version_reason
|
||||
# Reasons explain a known incompatibility with a violated version
|
||||
# bound. They should not skip models when the installed version is
|
||||
# already within the requested range.
|
||||
is_reason_applicable = (
|
||||
not is_version_valid
|
||||
and self.transformers_version_reason is not None
|
||||
and check_version_reason in self.transformers_version_reason
|
||||
)
|
||||
is_transformers_valid = is_version_valid and is_reason_valid
|
||||
is_transformers_valid = is_version_valid or (
|
||||
not should_check_version and not is_reason_applicable
|
||||
)
|
||||
if is_transformers_valid:
|
||||
return None
|
||||
elif self.transformers_version_reason:
|
||||
@@ -405,7 +411,15 @@ _TEXT_GENERATION_EXAMPLE_MODELS = {
|
||||
"openbmb/MiniCPM3-4B", trust_remote_code=True
|
||||
),
|
||||
"MiniCPM4ForCausalLM": _HfExamplesInfo(
|
||||
"openbmb/MiniCPM4.1-8B", trust_remote_code=True
|
||||
"openbmb/MiniCPM4.1-8B",
|
||||
min_transformers_version="4.56",
|
||||
max_transformers_version="4.57",
|
||||
transformers_version_reason={
|
||||
"hf": "HF remote code imports removed `is_torch_fx_available`; "
|
||||
"the upstream compatibility shim request was closed as not planned: "
|
||||
"https://github.com/huggingface/transformers/issues/44561"
|
||||
},
|
||||
trust_remote_code=True,
|
||||
),
|
||||
"MiniMaxForCausalLM": _HfExamplesInfo("MiniMaxAI/MiniMax-Text-01-hf"),
|
||||
"MiniMaxText01ForCausalLM": _HfExamplesInfo(
|
||||
@@ -786,7 +800,7 @@ _MULTIMODAL_EXAMPLE_MODELS = {
|
||||
),
|
||||
"MusicFlamingoForConditionalGeneration": _HfExamplesInfo(
|
||||
"nvidia/music-flamingo-2601-hf",
|
||||
min_transformers_version="5.3.0",
|
||||
min_transformers_version="5.5.0",
|
||||
transformers_version_reason={
|
||||
"vllm": "Needs https://github.com/huggingface/transformers/pull/43538"
|
||||
},
|
||||
|
||||
@@ -202,7 +202,7 @@ class AudioFlamingo3ProcessingInfo(BaseProcessingInfo):
|
||||
)
|
||||
|
||||
def get_supported_mm_limits(self) -> Mapping[str, int | None]:
|
||||
return {"audio": None}
|
||||
return {"audio": 1}
|
||||
|
||||
|
||||
class AudioFlamingo3DummyInputsBuilder(
|
||||
@@ -379,32 +379,37 @@ class AudioFlamingo3MultiModalProcessor(
|
||||
mm_kwargs: Mapping[str, Any],
|
||||
tok_kwargs: Mapping[str, object],
|
||||
) -> BatchFeature:
|
||||
audios = mm_data.pop("audios", [])
|
||||
if audios:
|
||||
mm_data["audio"] = audios
|
||||
processor_mm_data = dict(mm_data)
|
||||
audios = processor_mm_data.pop("audios", None)
|
||||
if audios is not None:
|
||||
processor_mm_data["audio"] = audios
|
||||
|
||||
if not mm_data.get("audio", []):
|
||||
prompt_ids = self.info.get_tokenizer().encode(prompt)
|
||||
prompt_ids = self._apply_hf_processor_tokens_only(prompt_ids)
|
||||
return BatchFeature(dict(input_ids=[prompt_ids]), tensor_type="pt")
|
||||
outputs = super()._call_hf_processor(
|
||||
prompt=prompt,
|
||||
mm_data=processor_mm_data,
|
||||
mm_kwargs=mm_kwargs,
|
||||
tok_kwargs=tok_kwargs,
|
||||
)
|
||||
|
||||
if "input_features_mask" in outputs:
|
||||
outputs["feature_attention_mask"] = outputs.pop("input_features_mask")
|
||||
|
||||
audio_data = processor_mm_data.get("audio")
|
||||
if audio_data is None:
|
||||
return outputs
|
||||
|
||||
audio_list = audio_data if isinstance(audio_data, list) else [audio_data]
|
||||
if len(audio_list) == 0:
|
||||
return outputs
|
||||
|
||||
processor = self.info.get_hf_processor(**mm_kwargs)
|
||||
feature_extractor = processor.feature_extractor
|
||||
mm_kwargs = dict(
|
||||
**mm_kwargs,
|
||||
sampling_rate=feature_extractor.sampling_rate,
|
||||
)
|
||||
|
||||
audio_list = mm_data.get("audio")
|
||||
if not isinstance(audio_list, list):
|
||||
audio_list = [audio_list]
|
||||
|
||||
chunk_counts = []
|
||||
sampling_rate = feature_extractor.sampling_rate
|
||||
chunk_length = feature_extractor.chunk_length
|
||||
window_size = int(sampling_rate * chunk_length)
|
||||
max_windows = int(processor.max_audio_len // chunk_length)
|
||||
|
||||
chunk_counts = []
|
||||
for audio in audio_list:
|
||||
# audio is numpy array or list
|
||||
n_samples = len(audio) if isinstance(audio, list) else audio.shape[0]
|
||||
@@ -414,18 +419,7 @@ class AudioFlamingo3MultiModalProcessor(
|
||||
n_win = max_windows
|
||||
chunk_counts.append(n_win)
|
||||
|
||||
outputs = super()._call_hf_processor(
|
||||
prompt=prompt,
|
||||
mm_data=mm_data,
|
||||
mm_kwargs=mm_kwargs,
|
||||
tok_kwargs=tok_kwargs,
|
||||
)
|
||||
|
||||
if "input_features_mask" in outputs:
|
||||
outputs["feature_attention_mask"] = outputs.pop("input_features_mask")
|
||||
|
||||
outputs["chunk_counts"] = torch.tensor(chunk_counts, dtype=torch.long)
|
||||
|
||||
return outputs
|
||||
|
||||
def _get_mm_fields_config(
|
||||
@@ -611,6 +605,10 @@ class AudioFlamingo3ForConditionalGeneration(
|
||||
input_features: torch.Tensor,
|
||||
feature_attention_mask: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
input_features = input_features.to(
|
||||
dtype=self.audio_tower.conv1.weight.dtype,
|
||||
device=self.audio_tower.conv1.weight.device,
|
||||
)
|
||||
audio_attention_mask = _build_audio_encoder_attention_mask(
|
||||
feature_attention_mask,
|
||||
dtype=self.audio_tower.conv1.weight.dtype,
|
||||
|
||||
@@ -120,9 +120,11 @@ class MusicFlamingoRotaryEmbedding(nn.Module):
|
||||
) -> tuple["torch.Tensor", float]:
|
||||
del seq_len
|
||||
base = config.rope_parameters["rope_theta"]
|
||||
dim = getattr(config, "head_dim", None) or (
|
||||
partial_rotary_factor = config.rope_parameters.get("partial_rotary_factor", 1.0)
|
||||
head_dim = getattr(config, "head_dim", None) or (
|
||||
config.hidden_size // config.num_attention_heads
|
||||
)
|
||||
dim = int(head_dim * partial_rotary_factor)
|
||||
attention_factor = 1.0
|
||||
|
||||
inv_freq = 1.0 / (
|
||||
@@ -148,21 +150,40 @@ class MusicFlamingoRotaryEmbedding(nn.Module):
|
||||
position_angles = torch.repeat_interleave(position_angles, 2, dim=-1)
|
||||
return position_angles.to(dtype=inv_freq.dtype)
|
||||
|
||||
def _restore_fp32_rope_buffers(self) -> None:
|
||||
rope_init_fn: Callable = self.compute_default_rope_parameters
|
||||
if self.rope_type != "default":
|
||||
rope_init_fn = ROPE_INIT_FUNCTIONS[self.rope_type]
|
||||
inv_freq, self.attention_scaling = rope_init_fn(
|
||||
self.config, self.inv_freq.device
|
||||
)
|
||||
|
||||
self.inv_freq = inv_freq
|
||||
self.original_inv_freq = inv_freq.clone()
|
||||
self.position_angles = self._compute_position_angles(inv_freq)
|
||||
|
||||
def _apply(self, fn):
|
||||
super()._apply(fn)
|
||||
self._restore_fp32_rope_buffers()
|
||||
return self
|
||||
|
||||
@torch.no_grad()
|
||||
def forward(self, timestamps: Tensor, seq_len: int) -> tuple[Tensor, Tensor]:
|
||||
batch_positions = torch.arange(
|
||||
timestamps.shape[0],
|
||||
window_starts = timestamps[:, 0].to(
|
||||
device=self.inv_freq.device,
|
||||
dtype=self.inv_freq.dtype,
|
||||
)
|
||||
batch_positions = batch_positions / self.max_seq_len_cached
|
||||
batch_freqs = batch_positions.unsqueeze(-1) * self.inv_freq
|
||||
batch_freqs = torch.repeat_interleave(batch_freqs, 2, dim=-1)
|
||||
window_duration = self.config.audio_frame_step * 4 * seq_len
|
||||
window_positions = (
|
||||
torch.round(window_starts / window_duration) / self.max_seq_len_cached
|
||||
)
|
||||
window_freqs = window_positions.unsqueeze(-1) * self.inv_freq
|
||||
window_freqs = torch.repeat_interleave(window_freqs, 2, dim=-1)
|
||||
|
||||
batch_freqs = batch_freqs[:, None, :]
|
||||
window_freqs = window_freqs[:, None, :]
|
||||
time_freqs = self.position_angles[:seq_len][None, :, :]
|
||||
batch_freqs, time_freqs = broadcast_tensors(batch_freqs, time_freqs)
|
||||
freqs = torch.cat((batch_freqs, time_freqs), dim=-1)
|
||||
window_freqs, time_freqs = broadcast_tensors(window_freqs, time_freqs)
|
||||
freqs = torch.cat((window_freqs, time_freqs), dim=-1)
|
||||
angle = (-timestamps * 2 * pi).to(freqs)
|
||||
freqs = freqs * angle.unsqueeze(-1)
|
||||
return freqs.cos(), freqs.sin()
|
||||
@@ -170,7 +191,7 @@ class MusicFlamingoRotaryEmbedding(nn.Module):
|
||||
|
||||
class MusicFlamingoFeatureInputs(AudioFlamingo3FeatureInputs):
|
||||
rote_timestamps: Annotated[
|
||||
torch.Tensor,
|
||||
torch.Tensor | None,
|
||||
TensorShape(
|
||||
"num_chunks",
|
||||
"num_audio_time_steps",
|
||||
@@ -205,6 +226,7 @@ class MusicFlamingoProcessingInfo(AudioFlamingo3ProcessingInfo):
|
||||
feature_extractor = self.get_feature_extractor()
|
||||
return MusicFlamingoMultiModalDataParser(
|
||||
target_sr=feature_extractor.sampling_rate,
|
||||
audio_resample_method="soxr",
|
||||
expected_hidden_size=self._get_expected_hidden_size(),
|
||||
)
|
||||
|
||||
@@ -268,49 +290,6 @@ class MusicFlamingoMultiModalDataParser(AudioFlamingo3MultiModalDataParser):
|
||||
|
||||
|
||||
class MusicFlamingoMultiModalProcessor(AudioFlamingo3MultiModalProcessor):
|
||||
def _call_hf_processor(
|
||||
self,
|
||||
prompt: str,
|
||||
mm_data: dict[str, object],
|
||||
mm_kwargs: Mapping[str, Any],
|
||||
tok_kwargs: Mapping[str, object],
|
||||
) -> BatchFeature:
|
||||
outputs = super()._call_hf_processor(
|
||||
prompt=prompt,
|
||||
mm_data=mm_data,
|
||||
mm_kwargs=mm_kwargs,
|
||||
tok_kwargs=tok_kwargs,
|
||||
)
|
||||
|
||||
audio_data = mm_data.get("audio")
|
||||
if audio_data is None:
|
||||
return outputs
|
||||
|
||||
audio_list = audio_data if isinstance(audio_data, list) else [audio_data]
|
||||
if len(audio_list) == 0:
|
||||
return outputs
|
||||
|
||||
processor = self.info.get_hf_processor(**mm_kwargs)
|
||||
feature_extractor = processor.feature_extractor
|
||||
sampling_rate = feature_extractor.sampling_rate
|
||||
chunk_length = feature_extractor.chunk_length
|
||||
window_size = int(sampling_rate * chunk_length)
|
||||
max_windows = int(processor.max_audio_len // chunk_length)
|
||||
|
||||
chunk_counts = []
|
||||
for audio in audio_list:
|
||||
n_samples = len(audio) if isinstance(audio, list) else audio.shape[0]
|
||||
n_win = max(1, (n_samples + window_size - 1) // window_size)
|
||||
chunk_counts.append(min(n_win, max_windows))
|
||||
outputs["chunk_counts"] = torch.tensor(chunk_counts, dtype=torch.long)
|
||||
|
||||
if "rote_timestamps" not in outputs:
|
||||
raise KeyError(
|
||||
"MusicFlamingoProcessor output must include `rote_timestamps`."
|
||||
)
|
||||
|
||||
return outputs
|
||||
|
||||
def _get_mm_fields_config(
|
||||
self,
|
||||
hf_inputs: BatchFeature,
|
||||
@@ -405,6 +384,32 @@ class MusicFlamingoForConditionalGeneration(AudioFlamingo3ForConditionalGenerati
|
||||
rote_timestamps=rote_timestamps,
|
||||
)
|
||||
|
||||
def _build_audio_timestamps(
|
||||
self,
|
||||
chunk_counts: list[int],
|
||||
seq_len: int,
|
||||
device: torch.device,
|
||||
) -> torch.Tensor:
|
||||
audio_embed_frame_step = self.config.audio_frame_step * 4
|
||||
frame_offsets = (
|
||||
torch.arange(seq_len, device=device, dtype=torch.float32)
|
||||
* audio_embed_frame_step
|
||||
)
|
||||
|
||||
if not chunk_counts:
|
||||
return frame_offsets.new_empty((0, seq_len))
|
||||
|
||||
window_indices = torch.cat(
|
||||
[
|
||||
torch.arange(count, device=device, dtype=torch.float32)
|
||||
for count in chunk_counts
|
||||
]
|
||||
)
|
||||
return (
|
||||
window_indices.unsqueeze(1) * seq_len * audio_embed_frame_step
|
||||
+ frame_offsets
|
||||
)
|
||||
|
||||
def _process_audio_input(
|
||||
self, audio_input: MusicFlamingoInputs
|
||||
) -> torch.Tensor | tuple[torch.Tensor, ...]:
|
||||
@@ -412,13 +417,6 @@ class MusicFlamingoForConditionalGeneration(AudioFlamingo3ForConditionalGenerati
|
||||
return super()._process_audio_input(audio_input)
|
||||
|
||||
rote_timestamps = audio_input["rote_timestamps"]
|
||||
if rote_timestamps is None:
|
||||
raise ValueError(
|
||||
"MusicFlamingo audio feature inputs must include `rote_timestamps`."
|
||||
)
|
||||
if isinstance(rote_timestamps, list):
|
||||
rote_timestamps = torch.cat(rote_timestamps, dim=0)
|
||||
|
||||
(
|
||||
input_features,
|
||||
feature_attention_mask,
|
||||
@@ -428,6 +426,15 @@ class MusicFlamingoForConditionalGeneration(AudioFlamingo3ForConditionalGenerati
|
||||
input_features,
|
||||
feature_attention_mask,
|
||||
)
|
||||
if rote_timestamps is None:
|
||||
rote_timestamps = self._build_audio_timestamps(
|
||||
chunk_counts,
|
||||
seq_len=hidden_states.shape[-2],
|
||||
device=hidden_states.device,
|
||||
)
|
||||
elif isinstance(rote_timestamps, list):
|
||||
rote_timestamps = torch.cat(rote_timestamps, dim=0)
|
||||
|
||||
cos, sin = self.pos_emb(
|
||||
rote_timestamps.to(hidden_states.device),
|
||||
seq_len=hidden_states.shape[-2],
|
||||
|
||||
@@ -21,6 +21,11 @@ try:
|
||||
except ImportError:
|
||||
scipy_signal = PlaceholderModule("scipy").placeholder_attr("signal") # type: ignore[assignment]
|
||||
|
||||
try:
|
||||
import soxr as soxr
|
||||
except ImportError:
|
||||
soxr = PlaceholderModule("soxr") # type: ignore[assignment]
|
||||
|
||||
|
||||
# ============================================================
|
||||
# Aligned with `librosa.get_duration` function
|
||||
@@ -245,13 +250,37 @@ def resample_audio_scipy(
|
||||
)
|
||||
|
||||
|
||||
def resample_audio_soxr(
|
||||
audio: npt.NDArray[np.floating],
|
||||
*,
|
||||
orig_sr: float,
|
||||
target_sr: float,
|
||||
) -> npt.NDArray[np.floating]:
|
||||
orig_sr_int = int(round(orig_sr))
|
||||
target_sr_int = int(round(target_sr))
|
||||
|
||||
if orig_sr_int == target_sr_int:
|
||||
return audio
|
||||
|
||||
if audio.ndim == 2:
|
||||
return np.stack(
|
||||
[
|
||||
resample_audio_soxr(ch, orig_sr=orig_sr, target_sr=target_sr)
|
||||
for ch in audio
|
||||
],
|
||||
axis=0,
|
||||
)
|
||||
|
||||
return soxr.resample(audio, orig_sr_int, target_sr_int)
|
||||
|
||||
|
||||
class AudioResampler:
|
||||
"""Resample audio data to a target sample rate."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
target_sr: float | None = None,
|
||||
method: Literal["pyav", "scipy"] = "pyav",
|
||||
method: Literal["pyav", "scipy", "soxr"] = "pyav",
|
||||
):
|
||||
self.target_sr = target_sr
|
||||
self.method = method
|
||||
@@ -279,10 +308,12 @@ class AudioResampler:
|
||||
return resample_audio_scipy(
|
||||
audio, orig_sr=orig_sr, target_sr=self.target_sr
|
||||
)
|
||||
elif self.method == "soxr":
|
||||
return resample_audio_soxr(audio, orig_sr=orig_sr, target_sr=self.target_sr)
|
||||
else:
|
||||
raise ValueError(
|
||||
f"Invalid resampling method: {self.method}. "
|
||||
"Supported methods are 'pyav' and 'scipy'."
|
||||
"Supported methods are 'pyav', 'scipy', and 'soxr'."
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -508,7 +508,7 @@ class MultiModalDataParser:
|
||||
*,
|
||||
target_sr: float | None = None,
|
||||
target_channels: int | None = None,
|
||||
audio_resample_method: Literal["pyav", "scipy"] = "pyav",
|
||||
audio_resample_method: Literal["pyav", "scipy", "soxr"] = "pyav",
|
||||
video_needs_metadata: bool = False,
|
||||
expected_hidden_size: int | None = None,
|
||||
) -> None:
|
||||
|
||||
Reference in New Issue
Block a user