[Bugfix] Use dedicated MM processor cache in /tokenize to prevent sender-cache pollution (#38545)

Signed-off-by: Sergey Zinchenko <sergey.zinchenko.rnd@gmail.com>
This commit is contained in:
Sergey Zinchenko
2026-04-01 21:14:49 -07:00
committed by GitHub
parent 5f96f9aff1
commit 5a2d420c17
4 changed files with 192 additions and 24 deletions
+9 -1
View File
@@ -451,6 +451,8 @@ class OpenAIServingRender:
request: Any,
prompt_input: str | list[str] | list[int] | list[list[int]] | None,
prompt_embeds: bytes | list[bytes] | None,
*,
skip_mm_cache: bool = False,
) -> list[EngineInput]:
"""Copied from OpenAIServing._preprocess_completion."""
prompts = list[SingletonPrompt | bytes]()
@@ -458,12 +460,14 @@ class OpenAIServingRender:
prompts.extend(prompt_to_seq(prompt_embeds))
if prompt_input is not None:
prompts.extend(prompt_to_seq(prompt_input))
return await self.preprocess_cmpl(request, prompts)
return await self.preprocess_cmpl(request, prompts, skip_mm_cache=skip_mm_cache)
async def preprocess_cmpl(
self,
request: Any,
prompts: Sequence[PromptType | bytes],
*,
skip_mm_cache: bool = False,
) -> list[EngineInput]:
"""Copied from OpenAIServing._preprocess_cmpl."""
renderer = self.renderer
@@ -487,6 +491,7 @@ class OpenAIServingRender:
for k in ("mm_processor_kwargs", "cache_salt")
if (v := getattr(request, k, None)) is not None
},
skip_mm_cache=skip_mm_cache,
)
async def preprocess_chat(
@@ -498,6 +503,8 @@ class OpenAIServingRender:
default_template_kwargs: dict[str, Any] | None,
tool_dicts: list[dict[str, Any]] | None = None,
tool_parser: type[ToolParser] | None = None,
*,
skip_mm_cache: bool = False,
) -> tuple[list[ConversationMessage], list[EngineInput]]:
"""Copied from OpenAIServing._preprocess_chat."""
renderer = self.renderer
@@ -529,6 +536,7 @@ class OpenAIServingRender:
for k in ("mm_processor_kwargs", "cache_salt")
if (v := getattr(request, k, None)) is not None
},
skip_mm_cache=skip_mm_cache,
)
# tool parsing is done only if a tool_parser has been set and if