[Feature] Add support for Llama 3.1 and 3.2 tool use (#8343)

Signed-off-by: Max de Bayser <mbayser@br.ibm.com>
This commit is contained in:
Maximilien de Bayser
2024-09-26 17:01:42 -07:00
committed by GitHub
parent 1b49148e47
commit 344cd2b6f4
10 changed files with 576 additions and 27 deletions
+3
View File
@@ -30,6 +30,7 @@ from vllm.entrypoints.openai.serving_engine import (BaseModelPath,
PromptAdapterPath,
TextTokensPrompt)
from vllm.entrypoints.openai.tool_parsers import (Hermes2ProToolParser,
Llama3JsonToolParser,
MistralToolParser,
ToolParser)
from vllm.inputs import TokensPrompt
@@ -85,6 +86,8 @@ class OpenAIServingChat(OpenAIServing):
self.tool_parser = MistralToolParser
elif tool_parser == "hermes":
self.tool_parser = Hermes2ProToolParser
elif tool_parser == "llama3_json":
self.tool_parser = Llama3JsonToolParser
else:
raise TypeError("Error: --enable-auto-tool-choice requires "
"--tool-call-parser")