[Frontend] Pythonic tool parser (#9859)

Signed-off-by: Mike Depinet <mike@fixie.ai>
This commit is contained in:
Mike Depinet
2024-11-14 04:14:34 +00:00
committed by GitHub
parent e0853b6508
commit f67ce05d0b
10 changed files with 806 additions and 26 deletions
+5
View File
@@ -74,6 +74,11 @@ class OpenAIServingChat(OpenAIServing):
self.tool_parser: Optional[Callable[[AnyTokenizer], ToolParser]] = None
if self.enable_auto_tools:
try:
if (tool_parser == "pythonic" and
model_config.model.startswith("meta-llama/Llama-3.2")):
logger.warning(
"Llama3.2 models may struggle to emit valid pythonic"
" tool calls")
self.tool_parser = ToolParserManager.get_tool_parser(
tool_parser)
except Exception as e: