[Model] Use helper function to run MM processors with token inputs (where applicable) (#38018)

Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
This commit is contained in:
Cyrus Leung
2026-03-26 16:44:04 +08:00
committed by GitHub
parent 52069012fe
commit 502c41a8f6
12 changed files with 215 additions and 145 deletions
+3 -2
View File
@@ -196,7 +196,7 @@ class InputProcessingContext:
tokenizer = self.tokenizer
if is_mistral_tokenizer(tokenizer):
tokenizer = tokenizer.transformers_tokenizer
tokenizer = tokenizer.transformers_tokenizer # type: ignore[union-attr]
merged_kwargs = self.get_merged_mm_kwargs(kwargs)
merged_kwargs.pop("tokenizer", None)
@@ -263,9 +263,10 @@ class InputProcessingContext:
requires_kw_only=False,
allow_var_kwargs=True,
)
allowed_kwargs.setdefault("return_tensors", "pt")
try:
output = hf_processor(**data, **allowed_kwargs, return_tensors="pt")
output = hf_processor(**data, **allowed_kwargs)
except Exception as exc:
# See https://github.com/huggingface/tokenizers/issues/537
if (