[Bugfix] Use dedicated MM processor cache in /tokenize to prevent sender-cache pollution (#38545)
Signed-off-by: Sergey Zinchenko <sergey.zinchenko.rnd@gmail.com>
This commit is contained in:
@@ -451,6 +451,8 @@ class OpenAIServingRender:
|
||||
request: Any,
|
||||
prompt_input: str | list[str] | list[int] | list[list[int]] | None,
|
||||
prompt_embeds: bytes | list[bytes] | None,
|
||||
*,
|
||||
skip_mm_cache: bool = False,
|
||||
) -> list[EngineInput]:
|
||||
"""Copied from OpenAIServing._preprocess_completion."""
|
||||
prompts = list[SingletonPrompt | bytes]()
|
||||
@@ -458,12 +460,14 @@ class OpenAIServingRender:
|
||||
prompts.extend(prompt_to_seq(prompt_embeds))
|
||||
if prompt_input is not None:
|
||||
prompts.extend(prompt_to_seq(prompt_input))
|
||||
return await self.preprocess_cmpl(request, prompts)
|
||||
return await self.preprocess_cmpl(request, prompts, skip_mm_cache=skip_mm_cache)
|
||||
|
||||
async def preprocess_cmpl(
|
||||
self,
|
||||
request: Any,
|
||||
prompts: Sequence[PromptType | bytes],
|
||||
*,
|
||||
skip_mm_cache: bool = False,
|
||||
) -> list[EngineInput]:
|
||||
"""Copied from OpenAIServing._preprocess_cmpl."""
|
||||
renderer = self.renderer
|
||||
@@ -487,6 +491,7 @@ class OpenAIServingRender:
|
||||
for k in ("mm_processor_kwargs", "cache_salt")
|
||||
if (v := getattr(request, k, None)) is not None
|
||||
},
|
||||
skip_mm_cache=skip_mm_cache,
|
||||
)
|
||||
|
||||
async def preprocess_chat(
|
||||
@@ -498,6 +503,8 @@ class OpenAIServingRender:
|
||||
default_template_kwargs: dict[str, Any] | None,
|
||||
tool_dicts: list[dict[str, Any]] | None = None,
|
||||
tool_parser: type[ToolParser] | None = None,
|
||||
*,
|
||||
skip_mm_cache: bool = False,
|
||||
) -> tuple[list[ConversationMessage], list[EngineInput]]:
|
||||
"""Copied from OpenAIServing._preprocess_chat."""
|
||||
renderer = self.renderer
|
||||
@@ -529,6 +536,7 @@ class OpenAIServingRender:
|
||||
for k in ("mm_processor_kwargs", "cache_salt")
|
||||
if (v := getattr(request, k, None)) is not None
|
||||
},
|
||||
skip_mm_cache=skip_mm_cache,
|
||||
)
|
||||
|
||||
# tool parsing is done only if a tool_parser has been set and if
|
||||
|
||||
@@ -86,12 +86,14 @@ class OpenAIServingTokenization(OpenAIServing):
|
||||
default_template_content_format=self.chat_template_content_format,
|
||||
default_template_kwargs=self.default_chat_template_kwargs,
|
||||
tool_dicts=tool_dicts,
|
||||
skip_mm_cache=True,
|
||||
)
|
||||
else:
|
||||
engine_inputs = await self.openai_serving_render.preprocess_completion(
|
||||
request,
|
||||
prompt_input=request.prompt,
|
||||
prompt_embeds=None,
|
||||
skip_mm_cache=True,
|
||||
)
|
||||
|
||||
input_ids: list[int] = []
|
||||
|
||||
Reference in New Issue
Block a user