[1/N] Harden Transformers modelling backend multi-modal path (#51408)
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
This commit is contained in:
@@ -134,6 +134,17 @@ def get_model_ids_to_test():
|
|||||||
return _get_model_ids_to_test(vllm_only_archs)
|
return _get_model_ids_to_test(vllm_only_archs)
|
||||||
|
|
||||||
|
|
||||||
|
def get_transformers_backend_model_ids_to_test():
|
||||||
|
return sorted(
|
||||||
|
{
|
||||||
|
model_id
|
||||||
|
for arch, info in _TRANSFORMERS_BACKEND_MODELS.items()
|
||||||
|
if "MultiModal" in arch
|
||||||
|
for model_id in (info.default, *info.extras.values())
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def get_text_token_prompts(
|
def get_text_token_prompts(
|
||||||
processor: BaseMultiModalProcessor,
|
processor: BaseMultiModalProcessor,
|
||||||
mm_data: MultiModalDataDict,
|
mm_data: MultiModalDataDict,
|
||||||
@@ -211,6 +222,7 @@ def _test_processing_correctness(
|
|||||||
hit_rate: float,
|
hit_rate: float,
|
||||||
num_batches: int,
|
num_batches: int,
|
||||||
simplify_rate: float,
|
simplify_rate: float,
|
||||||
|
model_impl: str = "auto",
|
||||||
):
|
):
|
||||||
if model_id_or_arch in HF_EXAMPLE_MODELS.get_supported_archs():
|
if model_id_or_arch in HF_EXAMPLE_MODELS.get_supported_archs():
|
||||||
# Use model architecture to get the default model id
|
# Use model architecture to get the default model id
|
||||||
@@ -238,6 +250,7 @@ def _test_processing_correctness(
|
|||||||
enable_mm_embeds=model_info.require_embed_inputs,
|
enable_mm_embeds=model_info.require_embed_inputs,
|
||||||
enforce_eager=model_info.enforce_eager,
|
enforce_eager=model_info.enforce_eager,
|
||||||
dtype=model_info.dtype,
|
dtype=model_info.dtype,
|
||||||
|
model_impl=model_impl,
|
||||||
)
|
)
|
||||||
# Ensure that the cache can fit all of the data
|
# Ensure that the cache can fit all of the data
|
||||||
# (set after because ModelConfig would set it to 0 for encoder-decoder models)
|
# (set after because ModelConfig would set it to 0 for encoder-decoder models)
|
||||||
@@ -480,6 +493,25 @@ def test_processing_correctness(
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("model_id", get_transformers_backend_model_ids_to_test())
|
||||||
|
@pytest.mark.parametrize("hit_rate", [0.3, 0.5, 1.0])
|
||||||
|
@pytest.mark.parametrize("num_batches", [32])
|
||||||
|
@pytest.mark.parametrize("simplify_rate", [1.0])
|
||||||
|
def test_processing_correctness_transformers(
|
||||||
|
model_id: str,
|
||||||
|
hit_rate: float,
|
||||||
|
num_batches: int,
|
||||||
|
simplify_rate: float,
|
||||||
|
):
|
||||||
|
_test_processing_correctness(
|
||||||
|
model_id,
|
||||||
|
hit_rate=hit_rate,
|
||||||
|
num_batches=num_batches,
|
||||||
|
simplify_rate=simplify_rate,
|
||||||
|
model_impl="transformers",
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def _assert_inputs_equal(
|
def _assert_inputs_equal(
|
||||||
a: MultiModalInput,
|
a: MultiModalInput,
|
||||||
b: MultiModalInput,
|
b: MultiModalInput,
|
||||||
|
|||||||
@@ -113,9 +113,7 @@ def test_audio_multimodal_processor(model_id):
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
def test_audio_multiple_inputs():
|
def _process_granite_speech(separator: str):
|
||||||
"""Multiple audios per prompt are each detected as a separate placeholder
|
|
||||||
and multi-modal item by the Transformers backend."""
|
|
||||||
model_id = "ibm-granite/granite-speech-3.3-2b"
|
model_id = "ibm-granite/granite-speech-3.3-2b"
|
||||||
model_config = ModelConfig(model=model_id, model_impl="transformers")
|
model_config = ModelConfig(model=model_id, model_impl="transformers")
|
||||||
mm_processor = MULTIMODAL_REGISTRY.create_processor(model_config)
|
mm_processor = MULTIMODAL_REGISTRY.create_processor(model_config)
|
||||||
@@ -124,15 +122,54 @@ def test_audio_multiple_inputs():
|
|||||||
# One token per audio; the processor expands each to its placeholder run.
|
# One token per audio; the processor expands each to its placeholder run.
|
||||||
prompt = (
|
prompt = (
|
||||||
"<|start_of_role|>user<|end_of_role|>"
|
"<|start_of_role|>user<|end_of_role|>"
|
||||||
f"{audio_token} and {audio_token} transcribe<|end_of_text|>\n"
|
f"{audio_token}{separator}{audio_token} transcribe<|end_of_text|>\n"
|
||||||
)
|
)
|
||||||
audios = [np.zeros(16000, dtype=np.float32), np.zeros(24000, dtype=np.float32)]
|
audios = [np.zeros(16000, dtype=np.float32), np.zeros(24000, dtype=np.float32)]
|
||||||
|
|
||||||
result = mm_processor(
|
return mm_processor(
|
||||||
prompt=prompt,
|
prompt=prompt,
|
||||||
mm_items=mm_processor.info.parse_mm_data({"audio": audios}),
|
mm_items=mm_processor.info.parse_mm_data({"audio": audios}),
|
||||||
hf_processor_mm_kwargs={},
|
hf_processor_mm_kwargs={},
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_audio_multiple_inputs():
|
||||||
|
"""Multiple audios per prompt are each detected as a separate placeholder
|
||||||
|
and multi-modal item by the Transformers modelling backend."""
|
||||||
|
result = _process_granite_speech(separator=" and ")
|
||||||
|
|
||||||
assert len(result["mm_placeholders"]["audio"]) == 2
|
assert len(result["mm_placeholders"]["audio"]) == 2
|
||||||
assert len(result["mm_kwargs"]["audio"]) == 2
|
assert len(result["mm_kwargs"]["audio"]) == 2
|
||||||
|
|
||||||
|
|
||||||
|
def test_audio_fields_not_claimed_by_image():
|
||||||
|
"""Audio fields survive when the image branch is also active."""
|
||||||
|
model_id = "ibm-granite/granite-speech-3.3-2b"
|
||||||
|
model_config = ModelConfig(model=model_id, model_impl="transformers")
|
||||||
|
mm_processor = MULTIMODAL_REGISTRY.create_processor(model_config)
|
||||||
|
|
||||||
|
audio_keys = ["input_features", "input_features_mask"]
|
||||||
|
owned = mm_processor._partition_keys_by_modality(audio_keys, ["audio", "image"])
|
||||||
|
|
||||||
|
assert owned["audio"] == audio_keys
|
||||||
|
assert owned["image"] == []
|
||||||
|
|
||||||
|
|
||||||
|
def test_unclaimed_fields_warn_rather_than_raise():
|
||||||
|
"""Keys no sub-processor declares are dropped with a warning, not an error."""
|
||||||
|
model_id = "ibm-granite/granite-speech-3.3-2b"
|
||||||
|
model_config = ModelConfig(model=model_id, model_impl="transformers")
|
||||||
|
mm_processor = MULTIMODAL_REGISTRY.create_processor(model_config)
|
||||||
|
|
||||||
|
owned = mm_processor._partition_keys_by_modality(
|
||||||
|
["input_features", "surprise_field"], ["audio", "image"]
|
||||||
|
)
|
||||||
|
|
||||||
|
assert owned["audio"] == ["input_features"]
|
||||||
|
assert owned["image"] == []
|
||||||
|
|
||||||
|
|
||||||
|
def test_audio_adjacent_inputs():
|
||||||
|
"""Adjacent audios are rejected rather than silently merged into one placeholder."""
|
||||||
|
with pytest.raises(ValueError, match="told apart"):
|
||||||
|
_process_granite_speech(separator="")
|
||||||
|
|||||||
@@ -56,24 +56,52 @@ def test_multimodal_processor(model_id):
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
def test_image_multiple_inputs():
|
def _process_two_images(separator: str):
|
||||||
"""Multiple images per prompt are each detected as a separate placeholder
|
|
||||||
and multi-modal item by the Transformers backend."""
|
|
||||||
model_id = "llava-hf/llava-onevision-qwen2-0.5b-ov-hf"
|
model_id = "llava-hf/llava-onevision-qwen2-0.5b-ov-hf"
|
||||||
model_config = ModelConfig(model=model_id, model_impl="transformers")
|
model_config = ModelConfig(model=model_id, model_impl="transformers")
|
||||||
mm_processor = MULTIMODAL_REGISTRY.create_processor(model_config)
|
mm_processor = MULTIMODAL_REGISTRY.create_processor(model_config)
|
||||||
|
|
||||||
image = ImageAsset("cherry_blossom").pil_image
|
image = ImageAsset("cherry_blossom").pil_image
|
||||||
prompt = (
|
prompt = (
|
||||||
"<|im_start|>user <image>\n and <image>\n"
|
f"<|im_start|>user <image>{separator}<image>\n"
|
||||||
"What do these images show?<|im_end|><|im_start|>assistant\n"
|
"What do these images show?<|im_end|><|im_start|>assistant\n"
|
||||||
)
|
)
|
||||||
|
|
||||||
result = mm_processor(
|
return mm_processor(
|
||||||
prompt=prompt,
|
prompt=prompt,
|
||||||
mm_items=mm_processor.info.parse_mm_data({"image": [image, image]}),
|
mm_items=mm_processor.info.parse_mm_data({"image": [image, image]}),
|
||||||
hf_processor_mm_kwargs={},
|
hf_processor_mm_kwargs={},
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_image_multiple_inputs():
|
||||||
|
"""Multiple images per prompt are each detected as a separate placeholder
|
||||||
|
and multi-modal item by the Transformers modelling backend."""
|
||||||
|
result = _process_two_images(separator="\n and ")
|
||||||
|
|
||||||
assert len(result["mm_placeholders"]["image"]) == 2
|
assert len(result["mm_placeholders"]["image"]) == 2
|
||||||
assert len(result["mm_kwargs"]["image"]) == 2
|
assert len(result["mm_kwargs"]["image"]) == 2
|
||||||
|
|
||||||
|
|
||||||
|
def test_image_adjacent_inputs():
|
||||||
|
"""Adjacent images stay separate placeholders rather than merging into one."""
|
||||||
|
result = _process_two_images(separator="")
|
||||||
|
|
||||||
|
assert len(result["mm_placeholders"]["image"]) == 2
|
||||||
|
assert len(result["mm_kwargs"]["image"]) == 2
|
||||||
|
|
||||||
|
|
||||||
|
def test_text_only_prompt():
|
||||||
|
"""An image model still accepts a prompt with no images."""
|
||||||
|
model_id = "llava-hf/llava-onevision-qwen2-0.5b-ov-hf"
|
||||||
|
model_config = ModelConfig(model=model_id, model_impl="transformers")
|
||||||
|
mm_processor = MULTIMODAL_REGISTRY.create_processor(model_config)
|
||||||
|
|
||||||
|
result = mm_processor(
|
||||||
|
prompt="<|im_start|>user Hello!<|im_end|><|im_start|>assistant\n",
|
||||||
|
mm_items=mm_processor.info.parse_mm_data({}),
|
||||||
|
hf_processor_mm_kwargs={},
|
||||||
|
)
|
||||||
|
|
||||||
|
assert len(result["prompt_token_ids"]) > 0
|
||||||
|
assert not result["mm_placeholders"]
|
||||||
|
|||||||
@@ -168,7 +168,8 @@ class MultiModalDummyInputsBuilder(BaseDummyInputsBuilder[MultiModalProcessingIn
|
|||||||
if self.info._is_audio_model() and (num_audios := mm_counts.get("audio", 0)):
|
if self.info._is_audio_model() and (num_audios := mm_counts.get("audio", 0)):
|
||||||
processor = self.info.get_hf_processor()
|
processor = self.info.get_hf_processor()
|
||||||
audio_token = getattr(processor, "audio_token", "")
|
audio_token = getattr(processor, "audio_token", "")
|
||||||
text += audio_token * num_audios
|
# Separated so that `_apply_audio` can tell the placeholders apart
|
||||||
|
text += " ".join([audio_token] * num_audios)
|
||||||
if self.info._is_image_model() and (num_images := mm_counts.get("image", 0)):
|
if self.info._is_image_model() and (num_images := mm_counts.get("image", 0)):
|
||||||
processor = self.info.get_hf_processor()
|
processor = self.info.get_hf_processor()
|
||||||
if "gemma3" in processor.__class__.__name__.lower():
|
if "gemma3" in processor.__class__.__name__.lower():
|
||||||
@@ -230,6 +231,54 @@ class MultiModalProcessor(BaseMultiModalProcessor[MultiModalProcessingInfo]):
|
|||||||
"""
|
"""
|
||||||
return None
|
return None
|
||||||
|
|
||||||
|
def _get_modality_field_names(self, modality: str) -> set[str]:
|
||||||
|
"""Field names the sub-processor for `modality` produces."""
|
||||||
|
# TODO: use else branch only once huggingface/transformers#44394 lands.
|
||||||
|
if modality == "audio":
|
||||||
|
sub_processor = self.info._get_audio_processor()
|
||||||
|
else:
|
||||||
|
processor = self.info.get_hf_processor()
|
||||||
|
sub_processor = getattr(processor, f"{modality}_processor", None)
|
||||||
|
|
||||||
|
# Pre-computed embeddings bypass the sub-processor entirely
|
||||||
|
names = {f"{modality}_embeds"}
|
||||||
|
for name in getattr(sub_processor, "model_input_names", None) or ():
|
||||||
|
# Companion masks are emitted but not always declared
|
||||||
|
names.update((name, f"{name}_mask"))
|
||||||
|
return names
|
||||||
|
|
||||||
|
def _partition_keys_by_modality(
|
||||||
|
self,
|
||||||
|
keys: list[str],
|
||||||
|
modalities: list[str],
|
||||||
|
) -> dict[str, list[str]]:
|
||||||
|
"""Attribute each HF processor output key to the modality that produced it."""
|
||||||
|
if len(modalities) == 1:
|
||||||
|
return {modalities[0]: keys}
|
||||||
|
|
||||||
|
claimed = {m: self._get_modality_field_names(m) for m in modalities}
|
||||||
|
|
||||||
|
owned: dict[str, list[str]] = {modality: [] for modality in modalities}
|
||||||
|
unclaimed = []
|
||||||
|
for key in keys:
|
||||||
|
for modality in modalities:
|
||||||
|
if key in claimed[modality]:
|
||||||
|
owned[modality].append(key)
|
||||||
|
break
|
||||||
|
else:
|
||||||
|
unclaimed.append(key)
|
||||||
|
|
||||||
|
if unclaimed:
|
||||||
|
logger.warning_once(
|
||||||
|
"Unable to attribute %s to any of the modalities %s, so they "
|
||||||
|
"will not be passed to the model. Add them to the relevant "
|
||||||
|
"sub-processor's `model_input_names` to fix this.",
|
||||||
|
tuple(unclaimed),
|
||||||
|
tuple(modalities),
|
||||||
|
)
|
||||||
|
|
||||||
|
return owned
|
||||||
|
|
||||||
def _get_mm_fields_config(
|
def _get_mm_fields_config(
|
||||||
self,
|
self,
|
||||||
hf_inputs: "BatchFeature",
|
hf_inputs: "BatchFeature",
|
||||||
@@ -238,34 +287,29 @@ class MultiModalProcessor(BaseMultiModalProcessor[MultiModalProcessingInfo]):
|
|||||||
# HF Processors always return a mask but vLLM doesn't need it
|
# HF Processors always return a mask but vLLM doesn't need it
|
||||||
hf_inputs.pop("attention_mask", None)
|
hf_inputs.pop("attention_mask", None)
|
||||||
|
|
||||||
mm_fields: dict[str, MultiModalFieldConfig] = {}
|
# Written by `_apply_audio`/`_apply_vision`; absent if the modality had no items
|
||||||
if self.info._is_audio_model():
|
sizes = {
|
||||||
num_audio_tokens = hf_inputs.get("num_audio_tokens")
|
"audio": hf_inputs.get("num_audio_tokens"),
|
||||||
mm_fields.update(
|
"image": hf_inputs.get("num_image_patches"),
|
||||||
{
|
}
|
||||||
key: MultiModalFieldConfig.flat_from_sizes(
|
modalities = [m for m, size in sizes.items() if size is not None]
|
||||||
"audio", num_audio_tokens
|
|
||||||
)
|
|
||||||
for key in hf_inputs
|
|
||||||
}
|
|
||||||
)
|
|
||||||
mm_fields["num_audio_tokens"] = MultiModalFieldConfig.batched("audio")
|
|
||||||
if self.info._is_image_model():
|
|
||||||
num_image_patches = hf_inputs.get("num_image_patches")
|
|
||||||
mm_fields.update(
|
|
||||||
{
|
|
||||||
key: MultiModalFieldConfig.flat_from_sizes(
|
|
||||||
"image", num_image_patches
|
|
||||||
)
|
|
||||||
for key in hf_inputs
|
|
||||||
}
|
|
||||||
)
|
|
||||||
mm_fields["image_embeds"] = MultiModalFieldConfig.flat_from_sizes(
|
|
||||||
"image", num_image_patches
|
|
||||||
)
|
|
||||||
|
|
||||||
# Keep these as batched, as they always have batch size as first dim
|
size_keys = {"num_audio_tokens", "num_image_patches"}
|
||||||
|
keys = [key for key in hf_inputs if key not in size_keys]
|
||||||
|
owned = self._partition_keys_by_modality(keys, modalities)
|
||||||
|
|
||||||
|
mm_fields: dict[str, MultiModalFieldConfig] = {
|
||||||
|
key: MultiModalFieldConfig.flat_from_sizes(modality, sizes[modality])
|
||||||
|
for modality in modalities
|
||||||
|
for key in owned[modality]
|
||||||
|
}
|
||||||
|
|
||||||
|
# Keep these as batched, as they always have batch size as first dim
|
||||||
|
if "audio" in modalities:
|
||||||
|
mm_fields["num_audio_tokens"] = MultiModalFieldConfig.batched("audio")
|
||||||
|
if "image" in modalities:
|
||||||
mm_fields["image_grid_thw"] = MultiModalFieldConfig.batched("image")
|
mm_fields["image_grid_thw"] = MultiModalFieldConfig.batched("image")
|
||||||
|
# TODO: route to "video" once the video modality is supported
|
||||||
mm_fields["video_grid_thw"] = MultiModalFieldConfig.batched("image")
|
mm_fields["video_grid_thw"] = MultiModalFieldConfig.batched("image")
|
||||||
mm_fields["num_image_patches"] = MultiModalFieldConfig.batched(
|
mm_fields["num_image_patches"] = MultiModalFieldConfig.batched(
|
||||||
"image", keep_on_cpu=True
|
"image", keep_on_cpu=True
|
||||||
@@ -291,6 +335,7 @@ class MultiModalProcessor(BaseMultiModalProcessor[MultiModalProcessingInfo]):
|
|||||||
self,
|
self,
|
||||||
prompt_ids: list[int],
|
prompt_ids: list[int],
|
||||||
processed_data: "BatchFeature",
|
processed_data: "BatchFeature",
|
||||||
|
num_audios: int,
|
||||||
) -> dict[str, list[PlaceholderRange]]:
|
) -> dict[str, list[PlaceholderRange]]:
|
||||||
audio_token_id = self.info._get_audio_token_id()
|
audio_token_id = self.info._get_audio_token_id()
|
||||||
prompt_tensor = torch.tensor(prompt_ids)
|
prompt_tensor = torch.tensor(prompt_ids)
|
||||||
@@ -301,17 +346,21 @@ class MultiModalProcessor(BaseMultiModalProcessor[MultiModalProcessingInfo]):
|
|||||||
|
|
||||||
padded = torch.cat([torch.tensor([False]), is_audio, torch.tensor([False])])
|
padded = torch.cat([torch.tensor([False]), is_audio, torch.tensor([False])])
|
||||||
transitions = padded.int().diff()
|
transitions = padded.int().diff()
|
||||||
starts = torch.where(transitions == 1)[0]
|
offsets = torch.where(transitions == 1)[0]
|
||||||
ends = torch.where(transitions == -1)[0]
|
lengths = torch.where(transitions == -1)[0] - offsets
|
||||||
lengths = ends - starts
|
|
||||||
|
if len(offsets) != num_audios:
|
||||||
|
raise ValueError(
|
||||||
|
f"Found {len(offsets)} run(s) of the audio token in the prompt but "
|
||||||
|
f"{num_audios} audio item(s) were passed. The Transformers backend "
|
||||||
|
"locates audio placeholders by finding contiguous runs of the audio "
|
||||||
|
"token, so placeholders with no text between them cannot yet be told "
|
||||||
|
"apart. Separate them in the prompt to work around this."
|
||||||
|
)
|
||||||
|
|
||||||
ranges = [
|
ranges = [
|
||||||
PlaceholderRange(
|
PlaceholderRange(offset=offset.item(), length=length.item())
|
||||||
offset=s.item(),
|
for offset, length in zip(offsets, lengths)
|
||||||
length=ln.item(),
|
|
||||||
is_embed=torch.ones(ln.item(), dtype=torch.bool),
|
|
||||||
)
|
|
||||||
for s, ln in zip(starts, lengths)
|
|
||||||
]
|
]
|
||||||
processed_data["num_audio_tokens"] = lengths
|
processed_data["num_audio_tokens"] = lengths
|
||||||
return {"audio": ranges}
|
return {"audio": ranges}
|
||||||
@@ -324,6 +373,7 @@ class MultiModalProcessor(BaseMultiModalProcessor[MultiModalProcessingInfo]):
|
|||||||
hf_processor_mm_kwargs: Mapping[str, object],
|
hf_processor_mm_kwargs: Mapping[str, object],
|
||||||
mm_token_type_ids: torch.Tensor | None,
|
mm_token_type_ids: torch.Tensor | None,
|
||||||
) -> dict[str, list[PlaceholderRange]]:
|
) -> dict[str, list[PlaceholderRange]]:
|
||||||
|
# Placeholders can't be located without them, so give up rather than guess
|
||||||
if mm_token_type_ids is None:
|
if mm_token_type_ids is None:
|
||||||
return {}
|
return {}
|
||||||
|
|
||||||
@@ -383,10 +433,13 @@ class MultiModalProcessor(BaseMultiModalProcessor[MultiModalProcessingInfo]):
|
|||||||
with timing_ctx.record("apply_hf_processor"):
|
with timing_ctx.record("apply_hf_processor"):
|
||||||
hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
|
hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
|
||||||
if not isinstance(prompt, str):
|
if not isinstance(prompt, str):
|
||||||
# the prompt is the tokenized ids which is not supported
|
# HF processors only accept text, and the decoded string already
|
||||||
# by the hf_processor, which is why we would need to decode the ids
|
# contains any special tokens, so don't let them be added again
|
||||||
# into string
|
|
||||||
prompt = hf_processor.decode(prompt)
|
prompt = hf_processor.decode(prompt)
|
||||||
|
tokenization_kwargs = {
|
||||||
|
**tokenization_kwargs,
|
||||||
|
"add_special_tokens": False,
|
||||||
|
}
|
||||||
|
|
||||||
# Bypass cached processor and always apply to the full set of mm inputs
|
# Bypass cached processor and always apply to the full set of mm inputs
|
||||||
# NOTE: we can't just set caching=False because base class method
|
# NOTE: we can't just set caching=False because base class method
|
||||||
@@ -412,9 +465,11 @@ class MultiModalProcessor(BaseMultiModalProcessor[MultiModalProcessingInfo]):
|
|||||||
mm_token_type_ids = processed_data.pop("mm_token_type_ids", mm_token_type_ids)
|
mm_token_type_ids = processed_data.pop("mm_token_type_ids", mm_token_type_ids)
|
||||||
|
|
||||||
mm_placeholders: dict[str, list[PlaceholderRange]] = {}
|
mm_placeholders: dict[str, list[PlaceholderRange]] = {}
|
||||||
if self.info._is_audio_model():
|
if num_audios := mm_items.get_count("audio", strict=False):
|
||||||
mm_placeholders.update(self._apply_audio(prompt_ids, processed_data))
|
mm_placeholders.update(
|
||||||
if self.info._is_image_model():
|
self._apply_audio(prompt_ids, processed_data, num_audios)
|
||||||
|
)
|
||||||
|
if mm_items.get_count("image", strict=False):
|
||||||
mm_placeholders.update(
|
mm_placeholders.update(
|
||||||
self._apply_vision(
|
self._apply_vision(
|
||||||
prompt_ids,
|
prompt_ids,
|
||||||
|
|||||||
Reference in New Issue
Block a user