[Frontend] [gpt-oss] Mcp type bug (#27689)
Signed-off-by: Jialin Ouyang <Jialin.Ouyang@gmail.com> Signed-off-by: Alec Solder <alecs@fb.com> Signed-off-by: Ye (Charlotte) Qi <yeq@meta.com> Co-authored-by: Jialin Ouyang <Jialin.Ouyang@gmail.com> Co-authored-by: Alec Solder <alecs@fb.com> Co-authored-by: Ye (Charlotte) Qi <yeq@meta.com>
This commit is contained in:
co-authored by
Jialin Ouyang
Alec Solder
Ye Qi
parent
3c7fefdeba
commit
ab2eb27b74
@@ -26,6 +26,8 @@ def mcp_disabled_server(monkeypatch_module: pytest.MonkeyPatch):
|
||||
with monkeypatch_module.context() as m:
|
||||
m.setenv("VLLM_ENABLE_RESPONSES_API_STORE", "1")
|
||||
m.setenv("PYTHON_EXECUTION_BACKEND", "dangerously_use_uv")
|
||||
# Helps the model follow instructions better
|
||||
m.setenv("VLLM_GPT_OSS_HARMONY_SYSTEM_INSTRUCTIONS", "1")
|
||||
with RemoteOpenAIServer(MODEL_NAME, args) as remote_server:
|
||||
yield remote_server
|
||||
|
||||
@@ -37,7 +39,9 @@ def mcp_enabled_server(monkeypatch_module: pytest.MonkeyPatch):
|
||||
with monkeypatch_module.context() as m:
|
||||
m.setenv("VLLM_ENABLE_RESPONSES_API_STORE", "1")
|
||||
m.setenv("PYTHON_EXECUTION_BACKEND", "dangerously_use_uv")
|
||||
m.setenv("GPT_OSS_SYSTEM_TOOL_MCP_LABELS", "code_interpreter,container")
|
||||
m.setenv("VLLM_GPT_OSS_SYSTEM_TOOL_MCP_LABELS", "code_interpreter,container")
|
||||
# Helps the model follow instructions better
|
||||
m.setenv("VLLM_GPT_OSS_HARMONY_SYSTEM_INSTRUCTIONS", "1")
|
||||
with RemoteOpenAIServer(MODEL_NAME, args) as remote_server:
|
||||
yield remote_server
|
||||
|
||||
@@ -56,18 +60,15 @@ async def mcp_enabled_client(mcp_enabled_server):
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("model_name", [MODEL_NAME])
|
||||
@pytest.mark.skip(reason="Code interpreter tool is not available in CI yet.")
|
||||
async def test_mcp_tool_env_flag_enabled(mcp_enabled_client: OpenAI, model_name: str):
|
||||
response = await mcp_enabled_client.responses.create(
|
||||
model=model_name,
|
||||
# TODO: Ideally should be able to set max tool calls
|
||||
# to prevent multi-turn, but it is not currently supported
|
||||
# would speed up the test
|
||||
input=(
|
||||
"What's the first 4 digits after the decimal point of "
|
||||
"cube root of `19910212 * 20250910`? "
|
||||
"Show only the digits. The python interpreter is not stateful "
|
||||
"and you must print to see the output."
|
||||
"Execute the following code: "
|
||||
"import random; print(random.randint(1, 1000000))"
|
||||
),
|
||||
instructions=(
|
||||
"You must use the Python tool to execute code. Never simulate execution."
|
||||
),
|
||||
tools=[
|
||||
{
|
||||
@@ -77,26 +78,47 @@ async def test_mcp_tool_env_flag_enabled(mcp_enabled_client: OpenAI, model_name:
|
||||
"server_url": "http://localhost:8888",
|
||||
}
|
||||
],
|
||||
extra_body={"enable_response_messages": True},
|
||||
)
|
||||
assert response is not None
|
||||
assert response.status == "completed"
|
||||
assert response.usage.output_tokens_details.tool_output_tokens > 0
|
||||
# Verify output messages: Tool calls and responses on analysis channel
|
||||
tool_call_found = False
|
||||
tool_response_found = False
|
||||
for message in response.output_messages:
|
||||
recipient = message.get("recipient")
|
||||
if recipient and recipient.startswith("python"):
|
||||
tool_call_found = True
|
||||
assert message.get("channel") == "analysis", (
|
||||
"Tool call should be on analysis channel"
|
||||
)
|
||||
author = message.get("author", {})
|
||||
if (
|
||||
author.get("role") == "tool"
|
||||
and author.get("name")
|
||||
and author.get("name").startswith("python")
|
||||
):
|
||||
tool_response_found = True
|
||||
assert message.get("channel") == "analysis", (
|
||||
"Tool response should be on analysis channel"
|
||||
)
|
||||
|
||||
assert tool_call_found, "Should have found at least one Python tool call"
|
||||
assert tool_response_found, "Should have found at least one Python tool response"
|
||||
for message in response.input_messages:
|
||||
assert message.get("author").get("role") != "developer", (
|
||||
"No developer messages should be present with valid mcp tool"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("model_name", [MODEL_NAME])
|
||||
@pytest.mark.skip(reason="Code interpreter tool is not available in CI yet.")
|
||||
async def test_mcp_tool_env_flag_disabled(mcp_disabled_client: OpenAI, model_name: str):
|
||||
response = await mcp_disabled_client.responses.create(
|
||||
model=model_name,
|
||||
# TODO: Ideally should be able to set max tool calls
|
||||
# to prevent multi-turn, but it is not currently supported
|
||||
# would speed up the test
|
||||
input=(
|
||||
"What's the first 4 digits after the decimal point of "
|
||||
"cube root of `19910212 * 20250910`? "
|
||||
"Show only the digits. The python interpreter is not stateful "
|
||||
"and you must print to see the output."
|
||||
"Execute the following code if the tool is present: "
|
||||
"import random; print(random.randint(1, 1000000))"
|
||||
),
|
||||
tools=[
|
||||
{
|
||||
@@ -106,7 +128,34 @@ async def test_mcp_tool_env_flag_disabled(mcp_disabled_client: OpenAI, model_nam
|
||||
"server_url": "http://localhost:8888",
|
||||
}
|
||||
],
|
||||
extra_body={"enable_response_messages": True},
|
||||
)
|
||||
assert response is not None
|
||||
assert response.status == "completed"
|
||||
assert response.usage.output_tokens_details.tool_output_tokens == 0
|
||||
# Verify output messages: No tool calls and responses
|
||||
tool_call_found = False
|
||||
tool_response_found = False
|
||||
for message in response.output_messages:
|
||||
recipient = message.get("recipient")
|
||||
if recipient and recipient.startswith("python"):
|
||||
tool_call_found = True
|
||||
assert message.get("channel") == "analysis", (
|
||||
"Tool call should be on analysis channel"
|
||||
)
|
||||
author = message.get("author", {})
|
||||
if (
|
||||
author.get("role") == "tool"
|
||||
and author.get("name")
|
||||
and author.get("name").startswith("python")
|
||||
):
|
||||
tool_response_found = True
|
||||
assert message.get("channel") == "analysis", (
|
||||
"Tool response should be on analysis channel"
|
||||
)
|
||||
|
||||
assert not tool_call_found, "Should not have a python call"
|
||||
assert not tool_response_found, "Should not have a tool response"
|
||||
for message in response.input_messages:
|
||||
assert message.get("author").get("role") != "developer", (
|
||||
"No developer messages should be present without a valid tool"
|
||||
)
|
||||
|
||||
@@ -6,10 +6,19 @@ from unittest.mock import MagicMock
|
||||
|
||||
import pytest
|
||||
import pytest_asyncio
|
||||
from openai.types.responses.tool import (
|
||||
CodeInterpreterContainerCodeInterpreterToolAuto,
|
||||
LocalShell,
|
||||
Mcp,
|
||||
Tool,
|
||||
)
|
||||
|
||||
from vllm.entrypoints.context import ConversationContext
|
||||
from vllm.entrypoints.openai.protocol import ErrorResponse, ResponsesRequest
|
||||
from vllm.entrypoints.openai.serving_responses import OpenAIServingResponses
|
||||
from vllm.entrypoints.openai.serving_responses import (
|
||||
OpenAIServingResponses,
|
||||
extract_tool_types,
|
||||
)
|
||||
from vllm.entrypoints.tool_server import ToolServer
|
||||
from vllm.inputs.data import TokensPrompt as EngineTokensPrompt
|
||||
|
||||
@@ -62,6 +71,45 @@ def mock_exit_stack():
|
||||
return MagicMock(spec=AsyncExitStack)
|
||||
|
||||
|
||||
def test_extract_tool_types(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
tools: list[Tool] = []
|
||||
assert extract_tool_types(tools) == set()
|
||||
|
||||
tools.append(LocalShell(type="local_shell"))
|
||||
assert extract_tool_types(tools) == {"local_shell"}
|
||||
|
||||
tools.append(CodeInterpreterContainerCodeInterpreterToolAuto(type="auto"))
|
||||
assert extract_tool_types(tools) == {"local_shell", "auto"}
|
||||
|
||||
tools.extend(
|
||||
[
|
||||
Mcp(type="mcp", server_label="random", server_url=""),
|
||||
Mcp(type="mcp", server_label="container", server_url=""),
|
||||
Mcp(type="mcp", server_label="code_interpreter", server_url=""),
|
||||
Mcp(type="mcp", server_label="web_search_preview", server_url=""),
|
||||
]
|
||||
)
|
||||
# When envs.VLLM_GPT_OSS_SYSTEM_TOOL_MCP_LABELS is not set,
|
||||
# mcp tool types are all ignored.
|
||||
assert extract_tool_types(tools) == {"local_shell", "auto"}
|
||||
|
||||
# container is allowed, it would be extracted
|
||||
monkeypatch.setenv("VLLM_GPT_OSS_SYSTEM_TOOL_MCP_LABELS", "container")
|
||||
assert extract_tool_types(tools) == {"local_shell", "auto", "container"}
|
||||
|
||||
# code_interpreter and web_search_preview are allowed,
|
||||
# they would be extracted
|
||||
monkeypatch.setenv(
|
||||
"VLLM_GPT_OSS_SYSTEM_TOOL_MCP_LABELS", "code_interpreter,web_search_preview"
|
||||
)
|
||||
assert extract_tool_types(tools) == {
|
||||
"local_shell",
|
||||
"auto",
|
||||
"code_interpreter",
|
||||
"web_search_preview",
|
||||
}
|
||||
|
||||
|
||||
class TestInitializeToolSessions:
|
||||
"""Test class for _initialize_tool_sessions method"""
|
||||
|
||||
|
||||
@@ -3,7 +3,10 @@
|
||||
|
||||
from openai_harmony import Role
|
||||
|
||||
from vllm.entrypoints.harmony_utils import parse_input_to_harmony_message
|
||||
from vllm.entrypoints.harmony_utils import (
|
||||
has_custom_tools,
|
||||
parse_input_to_harmony_message,
|
||||
)
|
||||
|
||||
|
||||
class TestParseInputToHarmonyMessage:
|
||||
@@ -252,3 +255,12 @@ class TestParseInputToHarmonyMessage:
|
||||
assert len(messages[0].content) == 2
|
||||
assert messages[0].content[0].text == ""
|
||||
assert messages[0].content[1].text == "actual text"
|
||||
|
||||
|
||||
def test_has_custom_tools() -> None:
|
||||
assert not has_custom_tools(set())
|
||||
assert not has_custom_tools({"web_search_preview", "code_interpreter", "container"})
|
||||
assert has_custom_tools({"others"})
|
||||
assert has_custom_tools(
|
||||
{"web_search_preview", "code_interpreter", "container", "others"}
|
||||
)
|
||||
|
||||
@@ -10,6 +10,7 @@ import vllm.envs as envs
|
||||
from vllm.envs import (
|
||||
enable_envs_cache,
|
||||
env_list_with_choices,
|
||||
env_set_with_choices,
|
||||
env_with_choices,
|
||||
environment_variables,
|
||||
)
|
||||
@@ -257,3 +258,110 @@ class TestEnvListWithChoices:
|
||||
with patch.dict(os.environ, {"TEST_ENV": "option1,option1,option2"}):
|
||||
env_func = env_list_with_choices("TEST_ENV", [], ["option1", "option2"])
|
||||
assert env_func() == ["option1", "option1", "option2"]
|
||||
|
||||
|
||||
class TestEnvSetWithChoices:
|
||||
"""Test cases for env_set_with_choices function."""
|
||||
|
||||
def test_default_list_returned_when_env_not_set(self):
|
||||
"""Test that default list is returned when env var is not set."""
|
||||
env_func = env_set_with_choices(
|
||||
"NONEXISTENT_ENV", ["default1", "default2"], ["option1", "option2"]
|
||||
)
|
||||
assert env_func() == {"default1", "default2"}
|
||||
|
||||
def test_empty_default_list_returned_when_env_not_set(self):
|
||||
"""Test that empty default list is returned when env not set."""
|
||||
env_func = env_set_with_choices("NONEXISTENT_ENV", [], ["option1", "option2"])
|
||||
assert env_func() == set()
|
||||
|
||||
def test_single_valid_value_parsed_correctly(self):
|
||||
"""Test that single valid value is parsed correctly."""
|
||||
with patch.dict(os.environ, {"TEST_ENV": "option1"}):
|
||||
env_func = env_set_with_choices("TEST_ENV", [], ["option1", "option2"])
|
||||
assert env_func() == {"option1"}
|
||||
|
||||
def test_multiple_valid_values_parsed_correctly(self):
|
||||
"""Test that multiple valid values are parsed correctly."""
|
||||
with patch.dict(os.environ, {"TEST_ENV": "option1,option2"}):
|
||||
env_func = env_set_with_choices("TEST_ENV", [], ["option1", "option2"])
|
||||
assert env_func() == {"option1", "option2"}
|
||||
|
||||
def test_values_with_whitespace_trimmed(self):
|
||||
"""Test that values with whitespace are trimmed correctly."""
|
||||
with patch.dict(os.environ, {"TEST_ENV": " option1 , option2 "}):
|
||||
env_func = env_set_with_choices("TEST_ENV", [], ["option1", "option2"])
|
||||
assert env_func() == {"option1", "option2"}
|
||||
|
||||
def test_empty_values_filtered_out(self):
|
||||
"""Test that empty values are filtered out."""
|
||||
with patch.dict(os.environ, {"TEST_ENV": "option1,,option2,"}):
|
||||
env_func = env_set_with_choices("TEST_ENV", [], ["option1", "option2"])
|
||||
assert env_func() == {"option1", "option2"}
|
||||
|
||||
def test_empty_string_returns_default(self):
|
||||
"""Test that empty string returns default."""
|
||||
with patch.dict(os.environ, {"TEST_ENV": ""}):
|
||||
env_func = env_set_with_choices(
|
||||
"TEST_ENV", ["default"], ["option1", "option2"]
|
||||
)
|
||||
assert env_func() == {"default"}
|
||||
|
||||
def test_only_commas_returns_default(self):
|
||||
"""Test that string with only commas returns default."""
|
||||
with patch.dict(os.environ, {"TEST_ENV": ",,,"}):
|
||||
env_func = env_set_with_choices(
|
||||
"TEST_ENV", ["default"], ["option1", "option2"]
|
||||
)
|
||||
assert env_func() == {"default"}
|
||||
|
||||
def test_case_sensitive_validation(self):
|
||||
"""Test case sensitive validation."""
|
||||
with patch.dict(os.environ, {"TEST_ENV": "option1,OPTION2"}):
|
||||
env_func = env_set_with_choices(
|
||||
"TEST_ENV", [], ["option1", "option2"], case_sensitive=True
|
||||
)
|
||||
with pytest.raises(ValueError, match="Invalid value 'OPTION2' in TEST_ENV"):
|
||||
env_func()
|
||||
|
||||
def test_case_insensitive_validation(self):
|
||||
"""Test case insensitive validation."""
|
||||
with patch.dict(os.environ, {"TEST_ENV": "OPTION1,option2"}):
|
||||
env_func = env_set_with_choices(
|
||||
"TEST_ENV", [], ["option1", "option2"], case_sensitive=False
|
||||
)
|
||||
assert env_func() == {"OPTION1", "option2"}
|
||||
|
||||
def test_invalid_value_in_list_raises_error(self):
|
||||
"""Test that invalid value in list raises ValueError."""
|
||||
with patch.dict(os.environ, {"TEST_ENV": "option1,invalid,option2"}):
|
||||
env_func = env_set_with_choices("TEST_ENV", [], ["option1", "option2"])
|
||||
with pytest.raises(ValueError, match="Invalid value 'invalid' in TEST_ENV"):
|
||||
env_func()
|
||||
|
||||
def test_callable_choices_resolved_correctly(self):
|
||||
"""Test that callable choices are resolved correctly."""
|
||||
|
||||
def get_choices():
|
||||
return ["dynamic1", "dynamic2"]
|
||||
|
||||
with patch.dict(os.environ, {"TEST_ENV": "dynamic1,dynamic2"}):
|
||||
env_func = env_set_with_choices("TEST_ENV", [], get_choices)
|
||||
assert env_func() == {"dynamic1", "dynamic2"}
|
||||
|
||||
def test_callable_choices_with_invalid_value(self):
|
||||
"""Test that callable choices raise error for invalid values."""
|
||||
|
||||
def get_choices():
|
||||
return ["dynamic1", "dynamic2"]
|
||||
|
||||
with patch.dict(os.environ, {"TEST_ENV": "dynamic1,invalid"}):
|
||||
env_func = env_set_with_choices("TEST_ENV", [], get_choices)
|
||||
with pytest.raises(ValueError, match="Invalid value 'invalid' in TEST_ENV"):
|
||||
env_func()
|
||||
|
||||
def test_duplicate_values_deduped(self):
|
||||
"""Test that duplicate values in the list are deduped."""
|
||||
with patch.dict(os.environ, {"TEST_ENV": "option1,option1,option2"}):
|
||||
env_func = env_set_with_choices("TEST_ENV", [], ["option1", "option2"])
|
||||
assert env_func() == {"option1", "option2"}
|
||||
|
||||
Reference in New Issue
Block a user