[Bugfix][Multimodal] Avoid caching full prompts in fallback (#54439)
Signed-off-by: waizuichougou <2082431897@qq.com> Co-authored-by: OpenAI Codex <noreply@openai.com>
This commit is contained in:
co-authored by
OpenAI Codex
parent
56058fd572
commit
f6895a5fcb
@@ -1559,14 +1559,11 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
|
|||||||
# let BPE merge tokens across a segment boundary, silently
|
# let BPE merge tokens across a segment boundary, silently
|
||||||
# changing how a non-special-token placeholder is tokenized.
|
# changing how a non-special-token placeholder is tokenized.
|
||||||
new_token_ids = flatten_2d_lists(
|
new_token_ids = flatten_2d_lists(
|
||||||
[
|
[tokenizer.encode(text, add_special_tokens=False) for text in out_texts]
|
||||||
cached_encode(tokenizer, text, add_special_tokens=False)
|
|
||||||
for text in out_texts
|
|
||||||
]
|
|
||||||
)
|
)
|
||||||
else:
|
else:
|
||||||
new_token_ids = cached_encode(
|
new_token_ids = tokenizer.encode(
|
||||||
tokenizer, "".join(out_texts), add_special_tokens=False
|
"".join(out_texts), add_special_tokens=False
|
||||||
)
|
)
|
||||||
|
|
||||||
return new_token_ids, result
|
return new_token_ids, result
|
||||||
|
|||||||
Reference in New Issue
Block a user