[Feature] Add support for Llama 3.1 and 3.2 tool use (#8343)
Signed-off-by: Max de Bayser <mbayser@br.ibm.com>
This commit is contained in:
@@ -3,18 +3,20 @@ from typing import List
|
||||
import openai
|
||||
import pytest
|
||||
|
||||
from .utils import MESSAGES_WITHOUT_TOOLS, WEATHER_TOOL
|
||||
from .utils import (MESSAGES_WITHOUT_TOOLS, WEATHER_TOOL, ServerConfig,
|
||||
ensure_system_prompt)
|
||||
|
||||
|
||||
# test: make sure chat completions without tools provided work even when tools
|
||||
# are enabled. This makes sure tool call chat templates work, AND that the tool
|
||||
# parser stream processing doesn't change the output of the model.
|
||||
@pytest.mark.asyncio
|
||||
async def test_chat_completion_without_tools(client: openai.AsyncOpenAI):
|
||||
async def test_chat_completion_without_tools(client: openai.AsyncOpenAI,
|
||||
server_config: ServerConfig):
|
||||
models = await client.models.list()
|
||||
model_name: str = models.data[0].id
|
||||
chat_completion = await client.chat.completions.create(
|
||||
messages=MESSAGES_WITHOUT_TOOLS,
|
||||
messages=ensure_system_prompt(MESSAGES_WITHOUT_TOOLS, server_config),
|
||||
temperature=0,
|
||||
max_tokens=150,
|
||||
model=model_name,
|
||||
@@ -34,7 +36,7 @@ async def test_chat_completion_without_tools(client: openai.AsyncOpenAI):
|
||||
|
||||
# make the same request, streaming
|
||||
stream = await client.chat.completions.create(
|
||||
messages=MESSAGES_WITHOUT_TOOLS,
|
||||
messages=ensure_system_prompt(MESSAGES_WITHOUT_TOOLS, server_config),
|
||||
temperature=0,
|
||||
max_tokens=150,
|
||||
model=model_name,
|
||||
@@ -77,11 +79,12 @@ async def test_chat_completion_without_tools(client: openai.AsyncOpenAI):
|
||||
# tools, to make sure we can still get normal chat completion responses
|
||||
# and that they won't be parsed as tools
|
||||
@pytest.mark.asyncio
|
||||
async def test_chat_completion_with_tools(client: openai.AsyncOpenAI):
|
||||
async def test_chat_completion_with_tools(client: openai.AsyncOpenAI,
|
||||
server_config: ServerConfig):
|
||||
models = await client.models.list()
|
||||
model_name: str = models.data[0].id
|
||||
chat_completion = await client.chat.completions.create(
|
||||
messages=MESSAGES_WITHOUT_TOOLS,
|
||||
messages=ensure_system_prompt(MESSAGES_WITHOUT_TOOLS, server_config),
|
||||
temperature=0,
|
||||
max_tokens=150,
|
||||
model=model_name,
|
||||
@@ -102,7 +105,7 @@ async def test_chat_completion_with_tools(client: openai.AsyncOpenAI):
|
||||
|
||||
# make the same request, streaming
|
||||
stream = await client.chat.completions.create(
|
||||
messages=MESSAGES_WITHOUT_TOOLS,
|
||||
messages=ensure_system_prompt(MESSAGES_WITHOUT_TOOLS, server_config),
|
||||
temperature=0,
|
||||
max_tokens=150,
|
||||
model=model_name,
|
||||
|
||||
@@ -6,7 +6,7 @@ import pytest
|
||||
|
||||
from .utils import (MESSAGES_ASKING_FOR_PARALLEL_TOOLS,
|
||||
MESSAGES_WITH_PARALLEL_TOOL_RESPONSE, SEARCH_TOOL,
|
||||
WEATHER_TOOL)
|
||||
WEATHER_TOOL, ServerConfig)
|
||||
|
||||
|
||||
# test: getting the model to generate parallel tool calls (streaming/not)
|
||||
@@ -14,7 +14,13 @@ from .utils import (MESSAGES_ASKING_FOR_PARALLEL_TOOLS,
|
||||
# may be added in the future. e.g. llama 3.1 models are not designed to support
|
||||
# parallel tool calls.
|
||||
@pytest.mark.asyncio
|
||||
async def test_parallel_tool_calls(client: openai.AsyncOpenAI):
|
||||
async def test_parallel_tool_calls(client: openai.AsyncOpenAI,
|
||||
server_config: ServerConfig):
|
||||
|
||||
if not server_config.get("supports_parallel", True):
|
||||
pytest.skip("The {} model doesn't support parallel tool calls".format(
|
||||
server_config["model"]))
|
||||
|
||||
models = await client.models.list()
|
||||
model_name: str = models.data[0].id
|
||||
chat_completion = await client.chat.completions.create(
|
||||
@@ -136,7 +142,13 @@ async def test_parallel_tool_calls(client: openai.AsyncOpenAI):
|
||||
# test: providing parallel tool calls back to the model to get a response
|
||||
# (streaming/not)
|
||||
@pytest.mark.asyncio
|
||||
async def test_parallel_tool_calls_with_results(client: openai.AsyncOpenAI):
|
||||
async def test_parallel_tool_calls_with_results(client: openai.AsyncOpenAI,
|
||||
server_config: ServerConfig):
|
||||
|
||||
if not server_config.get("supports_parallel", True):
|
||||
pytest.skip("The {} model doesn't support parallel tool calls".format(
|
||||
server_config["model"]))
|
||||
|
||||
models = await client.models.list()
|
||||
model_name: str = models.data[0].id
|
||||
chat_completion = await client.chat.completions.create(
|
||||
|
||||
+58
-13
@@ -1,4 +1,5 @@
|
||||
from typing import Dict, List
|
||||
from copy import deepcopy
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
from openai.types.chat import (ChatCompletionMessageParam,
|
||||
ChatCompletionToolParam)
|
||||
@@ -7,9 +8,30 @@ from typing_extensions import TypedDict
|
||||
from tests.utils import VLLM_PATH
|
||||
|
||||
|
||||
class ServerConfig(TypedDict):
|
||||
class ServerConfig(TypedDict, total=False):
|
||||
model: str
|
||||
arguments: List[str]
|
||||
system_prompt: Optional[str]
|
||||
supports_parallel: Optional[bool]
|
||||
|
||||
|
||||
def patch_system_prompt(messages: List[Dict[str, Any]],
|
||||
system_prompt: str) -> List[Dict[str, Any]]:
|
||||
new_messages = deepcopy(messages)
|
||||
if new_messages[0]["role"] == "system":
|
||||
new_messages[0]["content"] = system_prompt
|
||||
else:
|
||||
new_messages.insert(0, {"role": "system", "content": system_prompt})
|
||||
return new_messages
|
||||
|
||||
|
||||
def ensure_system_prompt(messages: List[Dict[str, Any]],
|
||||
config: ServerConfig) -> List[Dict[str, Any]]:
|
||||
prompt = config.get("system_prompt")
|
||||
if prompt:
|
||||
return patch_system_prompt(messages, prompt)
|
||||
else:
|
||||
return messages
|
||||
|
||||
|
||||
# universal args for all models go here. also good if you need to test locally
|
||||
@@ -23,7 +45,33 @@ CONFIGS: Dict[str, ServerConfig] = {
|
||||
"arguments": [
|
||||
"--tool-call-parser", "hermes", "--chat-template",
|
||||
str(VLLM_PATH / "examples/tool_chat_template_hermes.jinja")
|
||||
]
|
||||
],
|
||||
"system_prompt":
|
||||
"You are a helpful assistant with access to tools. If a tool"
|
||||
" that you have would be helpful to answer a user query, "
|
||||
"call the tool. Otherwise, answer the user's query directly "
|
||||
"without calling a tool. DO NOT CALL A TOOL THAT IS IRRELEVANT "
|
||||
"to the user's question - just respond to it normally."
|
||||
},
|
||||
"llama": {
|
||||
"model":
|
||||
"meta-llama/Meta-Llama-3.1-8B-Instruct",
|
||||
"arguments": [
|
||||
"--tool-call-parser", "llama3_json", "--chat-template",
|
||||
str(VLLM_PATH / "examples/tool_chat_template_llama3.1_json.jinja")
|
||||
],
|
||||
"supports_parallel":
|
||||
False,
|
||||
},
|
||||
"llama3.2": {
|
||||
"model":
|
||||
"meta-llama/Llama-3.2-3B-Instruct",
|
||||
"arguments": [
|
||||
"--tool-call-parser", "llama3_json", "--chat-template",
|
||||
str(VLLM_PATH / "examples/tool_chat_template_llama3.2_json.jinja")
|
||||
],
|
||||
"supports_parallel":
|
||||
False,
|
||||
},
|
||||
"mistral": {
|
||||
"model":
|
||||
@@ -32,7 +80,13 @@ CONFIGS: Dict[str, ServerConfig] = {
|
||||
"--tool-call-parser", "mistral", "--chat-template",
|
||||
str(VLLM_PATH / "examples/tool_chat_template_mistral.jinja"),
|
||||
"--ignore-patterns=\"consolidated.safetensors\""
|
||||
]
|
||||
],
|
||||
"system_prompt":
|
||||
"You are a helpful assistant with access to tools. If a tool"
|
||||
" that you have would be helpful to answer a user query, "
|
||||
"call the tool. Otherwise, answer the user's query directly "
|
||||
"without calling a tool. DO NOT CALL A TOOL THAT IS IRRELEVANT "
|
||||
"to the user's question - just respond to it normally."
|
||||
}
|
||||
}
|
||||
|
||||
@@ -97,15 +151,6 @@ SEARCH_TOOL: ChatCompletionToolParam = {
|
||||
}
|
||||
|
||||
MESSAGES_WITHOUT_TOOLS: List[ChatCompletionMessageParam] = [{
|
||||
"role":
|
||||
"system",
|
||||
"content":
|
||||
"You are a helpful assistant with access to tools. If a tool"
|
||||
" that you have would be helpful to answer a user query, "
|
||||
"call the tool. Otherwise, answer the user's query directly "
|
||||
"without calling a tool. DO NOT CALL A TOOL THAT IS IRRELEVANT "
|
||||
"to the user's question - just respond to it normally."
|
||||
}, {
|
||||
"role":
|
||||
"user",
|
||||
"content":
|
||||
|
||||
Reference in New Issue
Block a user