Add embedding input functionality for disabled modalities [remake] (#32493)
Signed-off-by: Reagan Lee <“reaganjlee@gmail.com”> Signed-off-by: Reagan Lee <reaganjlee@gmail.com> Signed-off-by: Reagan Lee <96998476+reaganjlee@users.noreply.github.com> Co-authored-by: Reagan Lee <“reaganjlee@gmail.com”> Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
Reagan Lee
gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>
parent
785cf28fff
commit
c4df59ad43
@@ -681,16 +681,22 @@ class BaseProcessingInfo:
|
||||
mm_items = self.data_parser.parse_mm_data(mm_data)
|
||||
|
||||
if validate:
|
||||
mm_config = self.ctx.model_config.get_multimodal_config()
|
||||
if not mm_config.enable_mm_embeds:
|
||||
for modality, items in mm_items.items():
|
||||
if isinstance(items, (EmbeddingItems, DictEmbeddingItems)):
|
||||
mm_config = self.ctx.get_mm_config()
|
||||
|
||||
for modality, items in mm_items.items():
|
||||
if isinstance(items, (EmbeddingItems, DictEmbeddingItems)):
|
||||
if not mm_config.enable_mm_embeds:
|
||||
raise ValueError(
|
||||
f"You must set `--enable-mm-embeds` to input "
|
||||
f"`{modality}_embeds`"
|
||||
)
|
||||
|
||||
for modality, items in mm_items.items():
|
||||
if mm_config.get_limit_per_prompt(modality) == 0:
|
||||
logger.debug(
|
||||
"Skipping count validation for modality "
|
||||
"'%s' (embeddings with limit=0)",
|
||||
modality,
|
||||
)
|
||||
continue
|
||||
self.validate_num_items(modality, len(items))
|
||||
|
||||
return mm_items
|
||||
|
||||
@@ -95,7 +95,7 @@ class BaseDummyInputsBuilder(ABC, Generic[_I]):
|
||||
"""
|
||||
dummy_text = self.get_dummy_text(mm_counts)
|
||||
dummy_mm_data = self.get_dummy_mm_data(seq_len, mm_counts, mm_options)
|
||||
dummy_mm_items = self.info.parse_mm_data(dummy_mm_data)
|
||||
dummy_mm_items = self.info.parse_mm_data(dummy_mm_data, validate=False)
|
||||
|
||||
tokenization_kwargs = {"truncation": False}
|
||||
|
||||
|
||||
@@ -1395,7 +1395,7 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
|
||||
missing_modality_data.append(data)
|
||||
mm_missing_data[modality] = missing_modality_data
|
||||
|
||||
mm_missing_items = self.info.parse_mm_data(mm_missing_data)
|
||||
mm_missing_items = self.info.parse_mm_data(mm_missing_data, validate=False)
|
||||
|
||||
return mm_is_cached, mm_missing_items
|
||||
|
||||
|
||||
Reference in New Issue
Block a user