feat(frontend): add --default-chat-template-kwargs CLI argument (#31343)

Signed-off-by: effortprogrammer <yhjhoward7@gmail.com>
This commit is contained in:
Hojin Yang
2025-12-30 03:38:47 +00:00
committed by GitHub
parent e54ee3ea33
commit dc837bc23e
7 changed files with 91 additions and 2 deletions
+5 -1
View File
@@ -6,7 +6,7 @@ import json
import time
from collections.abc import AsyncGenerator, AsyncIterator
from collections.abc import Sequence as GenericSequence
from typing import Final
from typing import Any, Final
import jinja2
import partial_json_parser
@@ -102,6 +102,7 @@ class OpenAIServingChat(OpenAIServing):
enable_force_include_usage: bool = False,
enable_log_outputs: bool = False,
log_error_stack: bool = False,
default_chat_template_kwargs: dict[str, Any] | None = None,
) -> None:
super().__init__(
engine_client=engine_client,
@@ -115,6 +116,7 @@ class OpenAIServingChat(OpenAIServing):
self.chat_template = chat_template
self.chat_template_content_format: Final = chat_template_content_format
self.trust_request_chat_template = trust_request_chat_template
self.default_chat_template_kwargs = default_chat_template_kwargs or {}
self.enable_log_outputs = enable_log_outputs
# set up logits processors
@@ -203,6 +205,7 @@ class OpenAIServingChat(OpenAIServing):
tool_dicts=None,
documents=None,
chat_template_kwargs=None,
default_chat_template_kwargs=self.default_chat_template_kwargs,
tool_parser=None,
add_special_tokens=False,
)
@@ -310,6 +313,7 @@ class OpenAIServingChat(OpenAIServing):
tool_dicts=tool_dicts,
documents=request.documents,
chat_template_kwargs=request.chat_template_kwargs,
default_chat_template_kwargs=self.default_chat_template_kwargs,
tool_parser=tool_parser,
add_special_tokens=request.add_special_tokens,
)