[Refactor] Simplify dummy data generation (#35025)

Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
This commit is contained in:
Cyrus Leung
2026-02-22 20:55:27 -08:00
committed by GitHub
parent c645e9a214
commit 987506bca6
78 changed files with 282 additions and 367 deletions
+5 -7
View File
@@ -266,11 +266,14 @@ class InputProcessingContext:
if isinstance(tokenizer, MistralTokenizer):
tokenizer = tokenizer.transformers_tokenizer
merged_kwargs = self.get_merged_mm_kwargs(kwargs)
merged_kwargs.pop("tokenizer", None)
return cached_processor_from_config(
self.model_config,
processor_cls=typ,
tokenizer=tokenizer,
**kwargs,
**merged_kwargs,
)
def init_processor(
@@ -283,12 +286,7 @@ class InputProcessingContext:
Initialize a HuggingFace-like processor class, merging the
keyword arguments with those in the model's configuration.
"""
mm_config = self.model_config.get_multimodal_config()
base_kwargs = mm_config.mm_processor_kwargs
if base_kwargs is None:
base_kwargs = {}
merged_kwargs = {**base_kwargs, **kwargs}
merged_kwargs = self.get_merged_mm_kwargs(kwargs)
return typ(**merged_kwargs)