[Bugfix][Multimodal] Avoid caching full prompts in fallback (#54439)

Signed-off-by: waizuichougou <2082431897@qq.com> Co-authored-by: OpenAI Codex <noreply@openai.com>
2026-08-30 16:19:38 +00:00
parent 56058fd572
commit f6895a5fcb
1 changed files with 3 additions and 6 deletions
@@ -1559,14 +1559,11 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
            # let BPE merge tokens across a segment boundary, silently
            # changing how a non-special-token placeholder is tokenized.
            new_token_ids = flatten_2d_lists(
-                [
-                    cached_encode(tokenizer, text, add_special_tokens=False)
-                    for text in out_texts
-                ]
+                [tokenizer.encode(text, add_special_tokens=False) for text in out_texts]
            )
        else:
-            new_token_ids = cached_encode(
-                tokenizer, "".join(out_texts), add_special_tokens=False
+            new_token_ids = tokenizer.encode(
+                "".join(out_texts), add_special_tokens=False
            )

        return new_token_ids, result