[GPT-OSS] Strict tool call and constrained decoding for Harmony (#45560)
Signed-off-by: Yifan Zong <yzong@redhat.com>
This commit is contained in:
@@ -13,7 +13,7 @@ from typing import Any
|
|||||||
import pytest
|
import pytest
|
||||||
import pytest_asyncio
|
import pytest_asyncio
|
||||||
import requests
|
import requests
|
||||||
from openai import InternalServerError, NotFoundError, OpenAI
|
from openai import NotFoundError, OpenAI
|
||||||
from openai_harmony import Message
|
from openai_harmony import Message
|
||||||
|
|
||||||
from tests.utils import RemoteOpenAIServer
|
from tests.utils import RemoteOpenAIServer
|
||||||
@@ -368,8 +368,12 @@ async def test_streaming_types(
|
|||||||
|
|
||||||
@pytest.mark.asyncio
|
@pytest.mark.asyncio
|
||||||
@pytest.mark.parametrize("model_name", [MODEL_NAME])
|
@pytest.mark.parametrize("model_name", [MODEL_NAME])
|
||||||
|
@pytest.mark.parametrize("tool_choice", ["auto", "required"])
|
||||||
async def test_function_calling_with_streaming_types(
|
async def test_function_calling_with_streaming_types(
|
||||||
pairs_of_event_types: dict[str, str], client: OpenAI, model_name: str
|
pairs_of_event_types: dict[str, str],
|
||||||
|
client: OpenAI,
|
||||||
|
model_name: str,
|
||||||
|
tool_choice: str,
|
||||||
):
|
):
|
||||||
"""Streaming event nesting for function-calling responses."""
|
"""Streaming event nesting for function-calling responses."""
|
||||||
|
|
||||||
@@ -382,6 +386,7 @@ async def test_function_calling_with_streaming_types(
|
|||||||
validate_events=_has_function_events,
|
validate_events=_has_function_events,
|
||||||
input=[{"role": "user", "content": "What's the weather like in Paris today?"}],
|
input=[{"role": "user", "content": "What's the weather like in Paris today?"}],
|
||||||
tools=[GET_WEATHER_SCHEMA],
|
tools=[GET_WEATHER_SCHEMA],
|
||||||
|
tool_choice=tool_choice,
|
||||||
temperature=0.0,
|
temperature=0.0,
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -558,7 +563,8 @@ async def test_reasoning_item(client: OpenAI, model_name: str):
|
|||||||
|
|
||||||
@pytest.mark.asyncio
|
@pytest.mark.asyncio
|
||||||
@pytest.mark.parametrize("model_name", [MODEL_NAME])
|
@pytest.mark.parametrize("model_name", [MODEL_NAME])
|
||||||
async def test_function_calling(client: OpenAI, model_name: str):
|
@pytest.mark.parametrize("tool_choice", ["auto", "required"])
|
||||||
|
async def test_function_calling(client: OpenAI, model_name: str, tool_choice: str):
|
||||||
tools = [GET_WEATHER_SCHEMA]
|
tools = [GET_WEATHER_SCHEMA]
|
||||||
|
|
||||||
response = await retry_for_tool_call(
|
response = await retry_for_tool_call(
|
||||||
@@ -567,8 +573,9 @@ async def test_function_calling(client: OpenAI, model_name: str):
|
|||||||
expected_tool_type="function_call",
|
expected_tool_type="function_call",
|
||||||
input="What's the weather like in Paris today?",
|
input="What's the weather like in Paris today?",
|
||||||
tools=tools,
|
tools=tools,
|
||||||
|
tool_choice=tool_choice,
|
||||||
temperature=0.0,
|
temperature=0.0,
|
||||||
extra_body={"request_id": "test_function_calling_non_resp"},
|
extra_body={"request_id": f"test_function_calling_non_resp_{tool_choice}"},
|
||||||
)
|
)
|
||||||
assert response.status == "completed"
|
assert response.status == "completed"
|
||||||
assert has_output_type(response, "function_call"), (
|
assert has_output_type(response, "function_call"), (
|
||||||
@@ -610,7 +617,10 @@ async def test_function_calling(client: OpenAI, model_name: str):
|
|||||||
|
|
||||||
@pytest.mark.asyncio
|
@pytest.mark.asyncio
|
||||||
@pytest.mark.parametrize("model_name", [MODEL_NAME])
|
@pytest.mark.parametrize("model_name", [MODEL_NAME])
|
||||||
async def test_function_calling_multi_turn(client: OpenAI, model_name: str):
|
@pytest.mark.parametrize("tool_choice", ["auto", "required"])
|
||||||
|
async def test_function_calling_multi_turn(
|
||||||
|
client: OpenAI, model_name: str, tool_choice: str
|
||||||
|
):
|
||||||
"""Multi-tool, multi-turn function calling with retry at API level."""
|
"""Multi-tool, multi-turn function calling with retry at API level."""
|
||||||
tools = [
|
tools = [
|
||||||
{
|
{
|
||||||
@@ -635,6 +645,7 @@ async def test_function_calling_multi_turn(client: OpenAI, model_name: str):
|
|||||||
expected_tool_type="function_call",
|
expected_tool_type="function_call",
|
||||||
input="Help me plan a trip to a random place. And tell me the weather there.",
|
input="Help me plan a trip to a random place. And tell me the weather there.",
|
||||||
tools=tools,
|
tools=tools,
|
||||||
|
tool_choice=tool_choice,
|
||||||
temperature=0.0,
|
temperature=0.0,
|
||||||
)
|
)
|
||||||
assert response.status == "completed"
|
assert response.status == "completed"
|
||||||
@@ -659,6 +670,7 @@ async def test_function_calling_multi_turn(client: OpenAI, model_name: str):
|
|||||||
}
|
}
|
||||||
],
|
],
|
||||||
tools=tools,
|
tools=tools,
|
||||||
|
tool_choice=tool_choice,
|
||||||
previous_response_id=response.id,
|
previous_response_id=response.id,
|
||||||
temperature=0.0,
|
temperature=0.0,
|
||||||
)
|
)
|
||||||
@@ -695,20 +707,6 @@ async def test_function_calling_multi_turn(client: OpenAI, model_name: str):
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
|
||||||
@pytest.mark.parametrize("model_name", [MODEL_NAME])
|
|
||||||
async def test_function_calling_required(client: OpenAI, model_name: str):
|
|
||||||
tools = [GET_WEATHER_SCHEMA]
|
|
||||||
|
|
||||||
with pytest.raises(InternalServerError):
|
|
||||||
await client.responses.create(
|
|
||||||
model=model_name,
|
|
||||||
input="What's the weather like in Paris today?",
|
|
||||||
tools=tools,
|
|
||||||
tool_choice="required",
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
@pytest.mark.asyncio
|
||||||
@pytest.mark.parametrize("model_name", [MODEL_NAME])
|
@pytest.mark.parametrize("model_name", [MODEL_NAME])
|
||||||
async def test_system_message_with_tools(client: OpenAI, model_name: str):
|
async def test_system_message_with_tools(client: OpenAI, model_name: str):
|
||||||
@@ -726,7 +724,10 @@ async def test_system_message_with_tools(client: OpenAI, model_name: str):
|
|||||||
|
|
||||||
@pytest.mark.asyncio
|
@pytest.mark.asyncio
|
||||||
@pytest.mark.parametrize("model_name", [MODEL_NAME])
|
@pytest.mark.parametrize("model_name", [MODEL_NAME])
|
||||||
async def test_function_calling_full_history(client: OpenAI, model_name: str):
|
@pytest.mark.parametrize("tool_choice", ["auto", "required"])
|
||||||
|
async def test_function_calling_full_history(
|
||||||
|
client: OpenAI, model_name: str, tool_choice: str
|
||||||
|
):
|
||||||
tools = [GET_WEATHER_SCHEMA]
|
tools = [GET_WEATHER_SCHEMA]
|
||||||
|
|
||||||
input_messages = [
|
input_messages = [
|
||||||
@@ -739,6 +740,7 @@ async def test_function_calling_full_history(client: OpenAI, model_name: str):
|
|||||||
expected_tool_type="function_call",
|
expected_tool_type="function_call",
|
||||||
input=input_messages,
|
input=input_messages,
|
||||||
tools=tools,
|
tools=tools,
|
||||||
|
tool_choice=tool_choice,
|
||||||
temperature=0.0,
|
temperature=0.0,
|
||||||
)
|
)
|
||||||
assert response.status == "completed"
|
assert response.status == "completed"
|
||||||
@@ -772,7 +774,10 @@ async def test_function_calling_full_history(client: OpenAI, model_name: str):
|
|||||||
|
|
||||||
@pytest.mark.asyncio
|
@pytest.mark.asyncio
|
||||||
@pytest.mark.parametrize("model_name", [MODEL_NAME])
|
@pytest.mark.parametrize("model_name", [MODEL_NAME])
|
||||||
async def test_function_calling_with_stream(client: OpenAI, model_name: str):
|
@pytest.mark.parametrize("tool_choice", ["auto", "required"])
|
||||||
|
async def test_function_calling_with_stream(
|
||||||
|
client: OpenAI, model_name: str, tool_choice: str
|
||||||
|
):
|
||||||
"""Function calling via streaming, with retry for non-determinism."""
|
"""Function calling via streaming, with retry for non-determinism."""
|
||||||
tools = [GET_WEATHER_SCHEMA]
|
tools = [GET_WEATHER_SCHEMA]
|
||||||
input_list = [
|
input_list = [
|
||||||
@@ -792,6 +797,7 @@ async def test_function_calling_with_stream(client: OpenAI, model_name: str):
|
|||||||
validate_events=_has_function_call,
|
validate_events=_has_function_call,
|
||||||
input=input_list,
|
input=input_list,
|
||||||
tools=tools,
|
tools=tools,
|
||||||
|
tool_choice=tool_choice,
|
||||||
temperature=0.0,
|
temperature=0.0,
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -853,8 +859,9 @@ async def test_function_calling_with_stream(client: OpenAI, model_name: str):
|
|||||||
|
|
||||||
@pytest.mark.asyncio
|
@pytest.mark.asyncio
|
||||||
@pytest.mark.parametrize("model_name", [MODEL_NAME])
|
@pytest.mark.parametrize("model_name", [MODEL_NAME])
|
||||||
|
@pytest.mark.parametrize("tool_choice", ["auto", "required"])
|
||||||
async def test_function_calling_no_code_interpreter_events(
|
async def test_function_calling_no_code_interpreter_events(
|
||||||
client: OpenAI, model_name: str
|
client: OpenAI, model_name: str, tool_choice: str
|
||||||
):
|
):
|
||||||
"""Verify that function calls don't trigger code_interpreter events.
|
"""Verify that function calls don't trigger code_interpreter events.
|
||||||
|
|
||||||
@@ -880,6 +887,7 @@ async def test_function_calling_no_code_interpreter_events(
|
|||||||
validate_events=_has_function_call,
|
validate_events=_has_function_call,
|
||||||
input=input_list,
|
input=input_list,
|
||||||
tools=tools,
|
tools=tools,
|
||||||
|
tool_choice=tool_choice,
|
||||||
temperature=0.0,
|
temperature=0.0,
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -1048,8 +1056,9 @@ async def test_output_messages_enabled(client: OpenAI, model_name: str, server):
|
|||||||
|
|
||||||
@pytest.mark.asyncio
|
@pytest.mark.asyncio
|
||||||
@pytest.mark.parametrize("model_name", [MODEL_NAME])
|
@pytest.mark.parametrize("model_name", [MODEL_NAME])
|
||||||
|
@pytest.mark.parametrize("tool_choice", ["auto", "required"])
|
||||||
async def test_function_call_with_previous_input_messages(
|
async def test_function_call_with_previous_input_messages(
|
||||||
client: OpenAI, model_name: str
|
client: OpenAI, model_name: str, tool_choice: str
|
||||||
):
|
):
|
||||||
"""Multi-turn function calling using previous_input_messages."""
|
"""Multi-turn function calling using previous_input_messages."""
|
||||||
tools = [
|
tools = [
|
||||||
@@ -1074,6 +1083,7 @@ async def test_function_call_with_previous_input_messages(
|
|||||||
expected_tool_type="function_call",
|
expected_tool_type="function_call",
|
||||||
input="What is the horoscope for Aquarius today?",
|
input="What is the horoscope for Aquarius today?",
|
||||||
tools=tools,
|
tools=tools,
|
||||||
|
tool_choice=tool_choice,
|
||||||
temperature=0.0,
|
temperature=0.0,
|
||||||
extra_body={"enable_response_messages": True},
|
extra_body={"enable_response_messages": True},
|
||||||
max_output_tokens=1000,
|
max_output_tokens=1000,
|
||||||
|
|||||||
@@ -3,6 +3,7 @@
|
|||||||
|
|
||||||
import json
|
import json
|
||||||
from collections.abc import Sequence
|
from collections.abc import Sequence
|
||||||
|
from typing import Any, Literal
|
||||||
|
|
||||||
import pytest
|
import pytest
|
||||||
from openai_harmony import (
|
from openai_harmony import (
|
||||||
@@ -12,14 +13,18 @@ from openai_harmony import (
|
|||||||
Role,
|
Role,
|
||||||
)
|
)
|
||||||
from transformers import AutoTokenizer
|
from transformers import AutoTokenizer
|
||||||
|
from xgrammar import Grammar
|
||||||
|
from xgrammar.testing import _is_grammar_accept_string
|
||||||
|
|
||||||
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
|
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
|
||||||
from vllm.entrypoints.openai.engine.protocol import FunctionCall
|
from vllm.entrypoints.openai.engine.protocol import FunctionCall
|
||||||
from vllm.entrypoints.openai.parser.harmony_utils import (
|
from vllm.entrypoints.openai.parser.harmony_utils import (
|
||||||
get_encoding,
|
get_encoding,
|
||||||
)
|
)
|
||||||
|
from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
|
||||||
from vllm.parser.harmony import HarmonyParser
|
from vllm.parser.harmony import HarmonyParser
|
||||||
from vllm.parser.parser_manager import ParserManager
|
from vllm.parser.parser_manager import ParserManager
|
||||||
|
from vllm.sampling_params import StructuredOutputsParams
|
||||||
|
|
||||||
REASONING_MODEL_NAME = "openai/gpt-oss-20b"
|
REASONING_MODEL_NAME = "openai/gpt-oss-20b"
|
||||||
|
|
||||||
@@ -857,3 +862,335 @@ class TestProcessChunk:
|
|||||||
("analysis", "One"),
|
("analysis", "One"),
|
||||||
("final", "Two"),
|
("final", "Two"),
|
||||||
]
|
]
|
||||||
|
|
||||||
|
|
||||||
|
class TestAdjustRequest:
|
||||||
|
REQUEST_TEXT = "Hello"
|
||||||
|
TOOL_TYPE = "function"
|
||||||
|
TOOL_1_NAME = "get_user_location"
|
||||||
|
TOOL_2_NAME = "get_weather"
|
||||||
|
TOOLS = [
|
||||||
|
{
|
||||||
|
"name": TOOL_1_NAME,
|
||||||
|
"parameters": {
|
||||||
|
"type": "object",
|
||||||
|
"properties": {},
|
||||||
|
"required": [],
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": TOOL_2_NAME,
|
||||||
|
"parameters": {
|
||||||
|
"type": "object",
|
||||||
|
"properties": {"city": {"type": "string"}},
|
||||||
|
"required": ["city"],
|
||||||
|
},
|
||||||
|
},
|
||||||
|
]
|
||||||
|
OUTPUT_SCHEMA = {
|
||||||
|
"type": "object",
|
||||||
|
"properties": {"answer": {"type": "string"}},
|
||||||
|
"required": ["answer"],
|
||||||
|
}
|
||||||
|
|
||||||
|
ANALYSIS = "<|channel|>analysis<|message|>analysis message<|end|><|start|>assistant"
|
||||||
|
COMMENTARY = (
|
||||||
|
"<|channel|>commentary<|message|>commentary message<|end|><|start|>assistant"
|
||||||
|
)
|
||||||
|
TOOL_CALL_1 = (
|
||||||
|
ANALYSIS + f"<|channel|>commentary to=functions.{TOOL_1_NAME} json<|message|>"
|
||||||
|
"{}<|call|>"
|
||||||
|
)
|
||||||
|
TOOL_CALL_2 = (
|
||||||
|
ANALYSIS + f"<|channel|>commentary to=functions.{TOOL_2_NAME} json<|message|>"
|
||||||
|
'{"city": "Tokyo"}<|call|>'
|
||||||
|
)
|
||||||
|
FINAL_JSON_SCHEMA = (
|
||||||
|
ANALYSIS
|
||||||
|
+ '<|channel|>final <|constrain|>json<|message|>{"answer": "Tokyo"}<|end|>'
|
||||||
|
)
|
||||||
|
FINAL_JSON_OBJECT = (
|
||||||
|
ANALYSIS
|
||||||
|
+ '<|channel|>final <|constrain|>json<|message|>{"city": "Tokyo"}<|end|>'
|
||||||
|
)
|
||||||
|
FINAL_TEXT_ONLY = ANALYSIS + "<|channel|>final<|message|>any<|end|>"
|
||||||
|
FINAL_REGEX = ANALYSIS + "<|channel|>final<|message|>regex<|end|>"
|
||||||
|
FINAL_CHOICE = ANALYSIS + "<|channel|>final<|message|>choice1<|end|>"
|
||||||
|
FINAL_GRAMMAR = ANALYSIS + "<|channel|>final<|message|>grammar<|end|>"
|
||||||
|
FINAL_STRUCTURAL_TAG = ANALYSIS + "<|channel|>final<|message|>tag content<|end|>"
|
||||||
|
ADMISSION_SAMPLES = (
|
||||||
|
"COMMENTARY",
|
||||||
|
"TOOL_CALL_1",
|
||||||
|
"TOOL_CALL_2",
|
||||||
|
"FINAL_JSON_SCHEMA",
|
||||||
|
"FINAL_JSON_OBJECT",
|
||||||
|
"FINAL_TEXT_ONLY",
|
||||||
|
"FINAL_REGEX",
|
||||||
|
"FINAL_CHOICE",
|
||||||
|
"FINAL_GRAMMAR",
|
||||||
|
"FINAL_STRUCTURAL_TAG",
|
||||||
|
)
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _build_request(
|
||||||
|
request_kind: Literal["chat", "responses"],
|
||||||
|
tool_choice: str = "none",
|
||||||
|
strict_tools: bool = False,
|
||||||
|
response_format_type: str | None = None,
|
||||||
|
structured_outputs: StructuredOutputsParams | None = None,
|
||||||
|
) -> ChatCompletionRequest | ResponsesRequest:
|
||||||
|
data: dict[str, Any] = {
|
||||||
|
"model": REASONING_MODEL_NAME,
|
||||||
|
}
|
||||||
|
if request_kind == "chat":
|
||||||
|
data["messages"] = [
|
||||||
|
{
|
||||||
|
"role": "user",
|
||||||
|
"content": TestAdjustRequest.REQUEST_TEXT,
|
||||||
|
}
|
||||||
|
]
|
||||||
|
else:
|
||||||
|
data["input"] = TestAdjustRequest.REQUEST_TEXT
|
||||||
|
|
||||||
|
if request_kind == "chat":
|
||||||
|
data["tools"] = [
|
||||||
|
{
|
||||||
|
"type": TestAdjustRequest.TOOL_TYPE,
|
||||||
|
"function": {"strict": strict_tools, **tool_def},
|
||||||
|
}
|
||||||
|
for tool_def in TestAdjustRequest.TOOLS
|
||||||
|
]
|
||||||
|
data["tool_choice"] = (
|
||||||
|
{
|
||||||
|
"type": TestAdjustRequest.TOOL_TYPE,
|
||||||
|
"function": {"name": TestAdjustRequest.TOOL_2_NAME},
|
||||||
|
}
|
||||||
|
if tool_choice == "named"
|
||||||
|
else tool_choice
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
data["tools"] = [
|
||||||
|
{
|
||||||
|
"type": TestAdjustRequest.TOOL_TYPE,
|
||||||
|
"strict": strict_tools,
|
||||||
|
**tool_def,
|
||||||
|
}
|
||||||
|
for tool_def in TestAdjustRequest.TOOLS
|
||||||
|
]
|
||||||
|
data["tool_choice"] = (
|
||||||
|
{
|
||||||
|
"type": TestAdjustRequest.TOOL_TYPE,
|
||||||
|
"name": TestAdjustRequest.TOOL_2_NAME,
|
||||||
|
}
|
||||||
|
if tool_choice == "named"
|
||||||
|
else tool_choice
|
||||||
|
)
|
||||||
|
|
||||||
|
if response_format_type == "json_schema":
|
||||||
|
schema_format = {
|
||||||
|
"name": "answer_format",
|
||||||
|
"schema": TestAdjustRequest.OUTPUT_SCHEMA,
|
||||||
|
"strict": True,
|
||||||
|
}
|
||||||
|
if request_kind == "chat":
|
||||||
|
data["response_format"] = {
|
||||||
|
"type": "json_schema",
|
||||||
|
"json_schema": schema_format,
|
||||||
|
}
|
||||||
|
else:
|
||||||
|
data["text"] = {
|
||||||
|
"format": {
|
||||||
|
"type": "json_schema",
|
||||||
|
**schema_format,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
elif response_format_type == "json_object":
|
||||||
|
if request_kind == "chat":
|
||||||
|
data["response_format"] = {"type": "json_object"}
|
||||||
|
else:
|
||||||
|
data["text"] = {"format": {"type": "json_object"}}
|
||||||
|
|
||||||
|
if structured_outputs is not None:
|
||||||
|
data["structured_outputs"] = structured_outputs
|
||||||
|
|
||||||
|
if request_kind == "chat":
|
||||||
|
return ChatCompletionRequest.model_validate(data)
|
||||||
|
return ResponsesRequest.model_validate(data)
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _assert_format_cleared(
|
||||||
|
adjusted_request: ChatCompletionRequest | ResponsesRequest,
|
||||||
|
) -> None:
|
||||||
|
if isinstance(adjusted_request, ResponsesRequest):
|
||||||
|
assert adjusted_request.text is None or adjusted_request.text.format is None
|
||||||
|
else:
|
||||||
|
assert adjusted_request.response_format is None
|
||||||
|
|
||||||
|
structured_outputs = adjusted_request.structured_outputs
|
||||||
|
assert structured_outputs is not None
|
||||||
|
assert structured_outputs.structural_tag is not None
|
||||||
|
assert structured_outputs.all_non_structural_tag_constraints_none()
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def _assert_structured_outputs_admission(
|
||||||
|
cls,
|
||||||
|
adjusted_request: ChatCompletionRequest | ResponsesRequest,
|
||||||
|
expected_admission: Sequence[str],
|
||||||
|
) -> None:
|
||||||
|
structured_outputs = adjusted_request.structured_outputs
|
||||||
|
assert structured_outputs is not None
|
||||||
|
assert structured_outputs.structural_tag is not None
|
||||||
|
assert structured_outputs.all_non_structural_tag_constraints_none()
|
||||||
|
|
||||||
|
grammar = Grammar.from_structural_tag(structured_outputs.structural_tag)
|
||||||
|
expected_admission_set = set(expected_admission)
|
||||||
|
|
||||||
|
for sample_name in cls.ADMISSION_SAMPLES:
|
||||||
|
admitted = _is_grammar_accept_string(
|
||||||
|
grammar,
|
||||||
|
getattr(cls, sample_name),
|
||||||
|
require_termination=False,
|
||||||
|
)
|
||||||
|
should_admit = sample_name in expected_admission_set
|
||||||
|
assert admitted is should_admit, (
|
||||||
|
f"Expected structured_outputs admission for {sample_name} "
|
||||||
|
f"to be {should_admit}, got {admitted}."
|
||||||
|
)
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("request_kind", ["chat", "responses"])
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
("request_kwargs", "expected_admission"),
|
||||||
|
[
|
||||||
|
(
|
||||||
|
{"tool_choice": "auto", "strict_tools": True},
|
||||||
|
[
|
||||||
|
"COMMENTARY",
|
||||||
|
"TOOL_CALL_1",
|
||||||
|
"TOOL_CALL_2",
|
||||||
|
"FINAL_JSON_SCHEMA",
|
||||||
|
"FINAL_JSON_OBJECT",
|
||||||
|
"FINAL_TEXT_ONLY",
|
||||||
|
"FINAL_REGEX",
|
||||||
|
"FINAL_CHOICE",
|
||||||
|
"FINAL_GRAMMAR",
|
||||||
|
"FINAL_STRUCTURAL_TAG",
|
||||||
|
],
|
||||||
|
),
|
||||||
|
(
|
||||||
|
{"tool_choice": "required"},
|
||||||
|
["COMMENTARY", "TOOL_CALL_1", "TOOL_CALL_2"],
|
||||||
|
),
|
||||||
|
(
|
||||||
|
{"tool_choice": "named"},
|
||||||
|
["COMMENTARY", "TOOL_CALL_2"],
|
||||||
|
),
|
||||||
|
(
|
||||||
|
{"response_format_type": "json_schema"},
|
||||||
|
["FINAL_JSON_SCHEMA"],
|
||||||
|
),
|
||||||
|
(
|
||||||
|
{"response_format_type": "json_object"},
|
||||||
|
["FINAL_JSON_SCHEMA", "FINAL_JSON_OBJECT"],
|
||||||
|
),
|
||||||
|
(
|
||||||
|
{"structured_outputs": StructuredOutputsParams(json=OUTPUT_SCHEMA)},
|
||||||
|
["FINAL_JSON_SCHEMA"],
|
||||||
|
),
|
||||||
|
(
|
||||||
|
{"structured_outputs": StructuredOutputsParams(json_object=True)},
|
||||||
|
["FINAL_JSON_SCHEMA", "FINAL_JSON_OBJECT"],
|
||||||
|
),
|
||||||
|
(
|
||||||
|
{"structured_outputs": StructuredOutputsParams(regex=r"regex")},
|
||||||
|
["FINAL_REGEX"],
|
||||||
|
),
|
||||||
|
(
|
||||||
|
{
|
||||||
|
"structured_outputs": StructuredOutputsParams(
|
||||||
|
choice=["choice1", "choice2"]
|
||||||
|
)
|
||||||
|
},
|
||||||
|
["FINAL_CHOICE"],
|
||||||
|
),
|
||||||
|
(
|
||||||
|
{
|
||||||
|
"structured_outputs": StructuredOutputsParams(
|
||||||
|
grammar='root ::= "grammar"'
|
||||||
|
)
|
||||||
|
},
|
||||||
|
["FINAL_GRAMMAR"],
|
||||||
|
),
|
||||||
|
(
|
||||||
|
{
|
||||||
|
"structured_outputs": StructuredOutputsParams(
|
||||||
|
structural_tag=json.dumps(
|
||||||
|
{
|
||||||
|
"type": "structural_tag",
|
||||||
|
"format": {
|
||||||
|
"type": "json_schema",
|
||||||
|
"json_schema": OUTPUT_SCHEMA,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
)
|
||||||
|
)
|
||||||
|
},
|
||||||
|
["FINAL_JSON_SCHEMA"],
|
||||||
|
),
|
||||||
|
(
|
||||||
|
{
|
||||||
|
"structured_outputs": StructuredOutputsParams(
|
||||||
|
structural_tag=json.dumps(
|
||||||
|
{
|
||||||
|
"type": "structural_tag",
|
||||||
|
"structures": [
|
||||||
|
{
|
||||||
|
"begin": "<tag>",
|
||||||
|
"schema": {"type": "object"},
|
||||||
|
"end": "</tag>",
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"triggers": ["<tag>"],
|
||||||
|
}
|
||||||
|
)
|
||||||
|
)
|
||||||
|
},
|
||||||
|
[
|
||||||
|
# Legacy triggered tags allow free text until a trigger, so
|
||||||
|
# unconstrained final-channel payloads are also admitted.
|
||||||
|
"FINAL_TEXT_ONLY",
|
||||||
|
"FINAL_REGEX",
|
||||||
|
"FINAL_CHOICE",
|
||||||
|
"FINAL_GRAMMAR",
|
||||||
|
"FINAL_STRUCTURAL_TAG",
|
||||||
|
],
|
||||||
|
),
|
||||||
|
],
|
||||||
|
ids=[
|
||||||
|
"tool_auto_strict",
|
||||||
|
"tool_required",
|
||||||
|
"tool_named",
|
||||||
|
"response_format_json_schema",
|
||||||
|
"response_format_json_object",
|
||||||
|
"structured_outputs_json",
|
||||||
|
"structured_outputs_json_object",
|
||||||
|
"structured_outputs_regex",
|
||||||
|
"structured_outputs_choice",
|
||||||
|
"structured_outputs_grammar",
|
||||||
|
"structured_outputs_structural_tag_modern",
|
||||||
|
"structured_outputs_structural_tag_legacy",
|
||||||
|
],
|
||||||
|
)
|
||||||
|
def test_adjust_request(
|
||||||
|
self,
|
||||||
|
harmony_parser,
|
||||||
|
request_kind,
|
||||||
|
request_kwargs,
|
||||||
|
expected_admission,
|
||||||
|
):
|
||||||
|
request = self._build_request(request_kind, **request_kwargs)
|
||||||
|
adjusted_request = harmony_parser.adjust_request(request)
|
||||||
|
self._assert_format_cleared(adjusted_request)
|
||||||
|
self._assert_structured_outputs_admission(
|
||||||
|
adjusted_request,
|
||||||
|
expected_admission,
|
||||||
|
)
|
||||||
|
|||||||
@@ -1,351 +1,12 @@
|
|||||||
# SPDX-License-Identifier: Apache-2.0
|
# SPDX-License-Identifier: Apache-2.0
|
||||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||||
|
|
||||||
import json
|
|
||||||
from unittest.mock import Mock
|
from unittest.mock import Mock
|
||||||
|
|
||||||
import pytest
|
from vllm.reasoning.gptoss_reasoning_parser import GptOssReasoningParser
|
||||||
from transformers import AutoTokenizer
|
|
||||||
|
|
||||||
from vllm.entrypoints.mcp.tool_server import ToolServer
|
|
||||||
from vllm.reasoning import ReasoningParser
|
|
||||||
from vllm.reasoning.gptoss_reasoning_parser import (
|
|
||||||
GptOssReasoningParser,
|
|
||||||
from_builtin_tool_to_tag,
|
|
||||||
no_func_reasoning_tag,
|
|
||||||
)
|
|
||||||
|
|
||||||
REASONING_MODEL_NAME = "openai/gpt-oss-120b"
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.fixture(scope="module")
|
def test_gptoss_reasoning_ended_is_true():
|
||||||
def gpt_oss_tokenizer():
|
parser = GptOssReasoningParser(Mock())
|
||||||
return AutoTokenizer.from_pretrained(REASONING_MODEL_NAME)
|
assert parser.is_reasoning_end([]) is True
|
||||||
|
assert parser.is_reasoning_end_streaming([], []) is True
|
||||||
|
|
||||||
USER_MESSAGE_START = "<|start|>user<|message|>"
|
|
||||||
REASONING_SECTION_START = "<|end|><|start|>assistant<|channel|>analysis<|message|>"
|
|
||||||
END = "<|end|>"
|
|
||||||
ASSISTANT_START = "<|start|>assistant"
|
|
||||||
ASSISTANT_CONTENT_START_PREFIX = END + ASSISTANT_START + "<|channel|>final"
|
|
||||||
ASSISTANT_CONTENT_START_SUFFIX = "<|message|>"
|
|
||||||
ASSISTANT_CONTENT_START = (
|
|
||||||
ASSISTANT_CONTENT_START_PREFIX + ASSISTANT_CONTENT_START_SUFFIX
|
|
||||||
)
|
|
||||||
|
|
||||||
BASIC_CONTENT = {
|
|
||||||
"output": REASONING_SECTION_START
|
|
||||||
+ "This is reasoning"
|
|
||||||
+ ASSISTANT_CONTENT_START
|
|
||||||
+ "This is the rest",
|
|
||||||
"is_reasoning_end": True,
|
|
||||||
}
|
|
||||||
|
|
||||||
BASIC_REASONING_ONLY = {
|
|
||||||
"output": REASONING_SECTION_START + "This is reasoning" + "<|end|>",
|
|
||||||
"is_reasoning_end": False,
|
|
||||||
}
|
|
||||||
BASIC_NO_REASONING_NO_ASSISTANT = {
|
|
||||||
"output": USER_MESSAGE_START + "This is a user message",
|
|
||||||
"is_reasoning_end": False,
|
|
||||||
}
|
|
||||||
|
|
||||||
# Edge-case where the model omits the assistant tag entirely.
|
|
||||||
BASIC_NO_REASONING_ASSISTANT = {
|
|
||||||
"output": USER_MESSAGE_START + "This is a user message<|end|><|channel|>final",
|
|
||||||
"is_reasoning_end": True,
|
|
||||||
}
|
|
||||||
|
|
||||||
COMPLEX_CONTENT_INCOMPLETE_PREFIX_ONLY = {
|
|
||||||
"output": REASONING_SECTION_START
|
|
||||||
+ "This is reasoning"
|
|
||||||
+ ASSISTANT_CONTENT_START_PREFIX,
|
|
||||||
"is_reasoning_end": False,
|
|
||||||
}
|
|
||||||
|
|
||||||
COMPLEX_CONTENT_SUFFIX_ONLY = {
|
|
||||||
"output": REASONING_SECTION_START
|
|
||||||
+ "This is reasoning"
|
|
||||||
+ ASSISTANT_CONTENT_START_SUFFIX,
|
|
||||||
"is_reasoning_end": False,
|
|
||||||
}
|
|
||||||
|
|
||||||
COMPLEX_CONTENT_1_NO_SUFFIX = {
|
|
||||||
"output": REASONING_SECTION_START
|
|
||||||
+ "This is reasoning"
|
|
||||||
+ ASSISTANT_CONTENT_START_PREFIX
|
|
||||||
+ "<|constrain|> JSON ",
|
|
||||||
"is_reasoning_end": False,
|
|
||||||
}
|
|
||||||
|
|
||||||
COMPLEX_CONTENT_1 = {
|
|
||||||
"output": REASONING_SECTION_START
|
|
||||||
+ "This is reasoning"
|
|
||||||
+ ASSISTANT_CONTENT_START_PREFIX
|
|
||||||
+ "<|constrain|> JSON "
|
|
||||||
+ ASSISTANT_CONTENT_START_SUFFIX,
|
|
||||||
"is_reasoning_end": True,
|
|
||||||
}
|
|
||||||
|
|
||||||
COMPLEX_CONTENT_1_WITH_CONTENT = {
|
|
||||||
"output": REASONING_SECTION_START
|
|
||||||
+ "This is reasoning"
|
|
||||||
+ ASSISTANT_CONTENT_START_PREFIX
|
|
||||||
+ "<|constrain|> JSON "
|
|
||||||
+ ASSISTANT_CONTENT_START_SUFFIX
|
|
||||||
+ "This is the rest",
|
|
||||||
"is_reasoning_end": True,
|
|
||||||
}
|
|
||||||
|
|
||||||
COMPLEX_CONTENT_2 = {
|
|
||||||
"output": REASONING_SECTION_START
|
|
||||||
+ "This is reasoning"
|
|
||||||
+ ASSISTANT_CONTENT_START_PREFIX
|
|
||||||
+ "<|constrain|>ReplyAction "
|
|
||||||
+ ASSISTANT_CONTENT_START_SUFFIX
|
|
||||||
+ "This is the rest",
|
|
||||||
"is_reasoning_end": True,
|
|
||||||
}
|
|
||||||
|
|
||||||
MULTI_TURN_CONTENT = {
|
|
||||||
"output": USER_MESSAGE_START
|
|
||||||
+ "1st turn user message"
|
|
||||||
+ REASONING_SECTION_START
|
|
||||||
+ "1st turn reasoning"
|
|
||||||
+ ASSISTANT_CONTENT_START
|
|
||||||
+ "1st turn response"
|
|
||||||
+ END
|
|
||||||
+ USER_MESSAGE_START
|
|
||||||
+ "2nd turn user message"
|
|
||||||
+ END
|
|
||||||
+ ASSISTANT_START,
|
|
||||||
"is_reasoning_end": False,
|
|
||||||
}
|
|
||||||
TEST_CASES = [
|
|
||||||
BASIC_CONTENT,
|
|
||||||
BASIC_REASONING_ONLY,
|
|
||||||
COMPLEX_CONTENT_INCOMPLETE_PREFIX_ONLY,
|
|
||||||
COMPLEX_CONTENT_SUFFIX_ONLY,
|
|
||||||
COMPLEX_CONTENT_1_NO_SUFFIX,
|
|
||||||
COMPLEX_CONTENT_1,
|
|
||||||
COMPLEX_CONTENT_1_WITH_CONTENT,
|
|
||||||
COMPLEX_CONTENT_2,
|
|
||||||
MULTI_TURN_CONTENT,
|
|
||||||
]
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.parametrize(
|
|
||||||
"output, is_reasoning_end",
|
|
||||||
[(t["output"], t["is_reasoning_end"]) for t in TEST_CASES],
|
|
||||||
)
|
|
||||||
def test_gptoss_is_reasoning_end(
|
|
||||||
output,
|
|
||||||
is_reasoning_end,
|
|
||||||
gpt_oss_tokenizer,
|
|
||||||
):
|
|
||||||
output = gpt_oss_tokenizer.tokenize(output)
|
|
||||||
parser: ReasoningParser = GptOssReasoningParser(gpt_oss_tokenizer)
|
|
||||||
|
|
||||||
# Test is_reasoning_end
|
|
||||||
output_ids = gpt_oss_tokenizer.convert_tokens_to_ids(output)
|
|
||||||
actual_is_reasoning_end = parser.is_reasoning_end(output_ids)
|
|
||||||
assert is_reasoning_end == actual_is_reasoning_end
|
|
||||||
|
|
||||||
|
|
||||||
class TestGptOssStructuralTags:
|
|
||||||
"""Test cases for GptOssReasoningParser structural tag functionality."""
|
|
||||||
|
|
||||||
@pytest.fixture
|
|
||||||
def mock_tokenizer(self):
|
|
||||||
"""Create a mock tokenizer for testing."""
|
|
||||||
tokenizer = Mock()
|
|
||||||
tokenizer.encode = Mock(return_value=[1, 2, 3, 4, 5])
|
|
||||||
tokenizer.get_vocab = Mock(return_value={"<|end|>": 6})
|
|
||||||
return tokenizer
|
|
||||||
|
|
||||||
@pytest.fixture
|
|
||||||
def reasoning_parser(self, mock_tokenizer):
|
|
||||||
"""Create a GptOssReasoningParser instance."""
|
|
||||||
return GptOssReasoningParser(mock_tokenizer)
|
|
||||||
|
|
||||||
def test_prepare_structured_tag_no_tool_server(self, reasoning_parser):
|
|
||||||
"""Test prepare_structured_tag with no tool server."""
|
|
||||||
result = reasoning_parser.prepare_structured_tag(None, None)
|
|
||||||
expected = json.dumps(no_func_reasoning_tag)
|
|
||||||
|
|
||||||
assert result == expected
|
|
||||||
|
|
||||||
# Verify the structure is correct
|
|
||||||
parsed = json.loads(result)
|
|
||||||
assert parsed["type"] == "structural_tag"
|
|
||||||
assert parsed["format"]["type"] == "triggered_tags"
|
|
||||||
assert len(parsed["format"]["tags"]) == 1
|
|
||||||
assert parsed["format"]["tags"][0]["begin"] == "<|channel|>analysis<|message|>"
|
|
||||||
assert parsed["format"]["triggers"] == ["<|channel|>analysis"]
|
|
||||||
|
|
||||||
def test_prepare_structured_tag_with_original_tag(self, reasoning_parser):
|
|
||||||
"""Test prepare_structured_tag when original_tag is provided."""
|
|
||||||
original_tag = '{"custom": "tag"}'
|
|
||||||
result = reasoning_parser.prepare_structured_tag(original_tag, None)
|
|
||||||
|
|
||||||
# Should return the original tag unchanged
|
|
||||||
assert result == original_tag
|
|
||||||
|
|
||||||
def test_from_builtin_tool_to_tag(self):
|
|
||||||
"""Test from_builtin_tool_to_tag function."""
|
|
||||||
tags = from_builtin_tool_to_tag("python")
|
|
||||||
|
|
||||||
assert len(tags) == 2
|
|
||||||
assert tags[0]["begin"] == "<|channel|>commentary to=python"
|
|
||||||
assert tags[0]["content"]["type"] == "any_text"
|
|
||||||
assert tags[0]["end"] == "<|end|>"
|
|
||||||
|
|
||||||
assert tags[1]["begin"] == "<|channel|>analysis to=python"
|
|
||||||
assert tags[1]["content"]["type"] == "any_text"
|
|
||||||
assert tags[1]["end"] == "<|end|>"
|
|
||||||
|
|
||||||
@pytest.mark.parametrize(
|
|
||||||
"tools",
|
|
||||||
[
|
|
||||||
[],
|
|
||||||
["browser"],
|
|
||||||
["python"],
|
|
||||||
["container"],
|
|
||||||
["browser", "python"],
|
|
||||||
["browser", "container"],
|
|
||||||
["python", "container"],
|
|
||||||
["browser", "python", "container"],
|
|
||||||
],
|
|
||||||
)
|
|
||||||
def test_json_validity_comprehensive(self, reasoning_parser, tools):
|
|
||||||
"""Test JSON validity across all possible tool combinations."""
|
|
||||||
tool_server = Mock(spec=ToolServer)
|
|
||||||
tool_server.has_tool = Mock(side_effect=lambda tool: tool in tools)
|
|
||||||
|
|
||||||
result = reasoning_parser.prepare_structured_tag(None, tool_server)
|
|
||||||
parsed_result = json.loads(result)
|
|
||||||
|
|
||||||
assert parsed_result["type"] == "structural_tag"
|
|
||||||
assert "format" in parsed_result
|
|
||||||
assert "tags" in parsed_result["format"]
|
|
||||||
assert "triggers" in parsed_result["format"]
|
|
||||||
|
|
||||||
# Tag count should be: 1 (analysis) + 2 * len(tools)
|
|
||||||
expected_tag_count = 1 + (2 * len(tools))
|
|
||||||
assert len(parsed_result["format"]["tags"]) == expected_tag_count
|
|
||||||
|
|
||||||
# Verify triggers are correctly configured
|
|
||||||
expected_triggers = ["<|channel|>analysis"]
|
|
||||||
if tools:
|
|
||||||
expected_triggers.append("<|channel|>commentary to=")
|
|
||||||
assert set(parsed_result["format"]["triggers"]) == set(expected_triggers)
|
|
||||||
|
|
||||||
def test_no_cross_request_state_pollution(self, reasoning_parser):
|
|
||||||
"""Test that sequential calls with different tool servers produce
|
|
||||||
independent results, guarding against shared mutable state
|
|
||||||
(e.g. missing deepcopy in tag_with_builtin_funcs)."""
|
|
||||||
tool_server_1 = Mock(spec=ToolServer)
|
|
||||||
tool_server_1.has_tool = Mock(side_effect=lambda tool: tool == "python")
|
|
||||||
|
|
||||||
tool_server_2 = Mock(spec=ToolServer)
|
|
||||||
tool_server_2.has_tool = Mock(side_effect=lambda tool: tool == "browser")
|
|
||||||
|
|
||||||
result_1 = reasoning_parser.prepare_structured_tag(None, tool_server_1)
|
|
||||||
result_2 = reasoning_parser.prepare_structured_tag(None, tool_server_2)
|
|
||||||
|
|
||||||
tags_1 = [tag["begin"] for tag in json.loads(result_1)["format"]["tags"]]
|
|
||||||
tags_2 = [tag["begin"] for tag in json.loads(result_2)["format"]["tags"]]
|
|
||||||
|
|
||||||
assert "<|channel|>commentary to=python" in tags_1
|
|
||||||
assert "<|channel|>commentary to=browser" not in tags_1
|
|
||||||
|
|
||||||
assert "<|channel|>commentary to=browser" in tags_2
|
|
||||||
assert "<|channel|>commentary to=python" not in tags_2
|
|
||||||
|
|
||||||
def test_tag_format_consistency(self, reasoning_parser):
|
|
||||||
"""Test that all generated tags follow consistent format,
|
|
||||||
catching malformed tags from from_builtin_tool_to_tag."""
|
|
||||||
tool_server = Mock(spec=ToolServer)
|
|
||||||
tool_server.has_tool = Mock(
|
|
||||||
side_effect=lambda tool: tool in ["python", "browser"]
|
|
||||||
)
|
|
||||||
|
|
||||||
result = reasoning_parser.prepare_structured_tag(None, tool_server)
|
|
||||||
parsed_result = json.loads(result)
|
|
||||||
|
|
||||||
for tag in parsed_result["format"]["tags"]:
|
|
||||||
assert "begin" in tag
|
|
||||||
assert "content" in tag
|
|
||||||
assert "end" in tag
|
|
||||||
assert tag["content"]["type"] == "any_text"
|
|
||||||
assert tag["end"] == "<|end|>"
|
|
||||||
assert tag["begin"].startswith("<|channel|>")
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.parametrize(
|
|
||||||
"output, is_reasoning_end",
|
|
||||||
[(t["output"], t["is_reasoning_end"]) for t in TEST_CASES],
|
|
||||||
)
|
|
||||||
def test_gptoss_is_reasoning_end_streaming(
|
|
||||||
output,
|
|
||||||
is_reasoning_end,
|
|
||||||
gpt_oss_tokenizer,
|
|
||||||
):
|
|
||||||
"""Streaming override must agree with is_reasoning_end for all cases."""
|
|
||||||
tokens = gpt_oss_tokenizer.tokenize(output)
|
|
||||||
parser: ReasoningParser = GptOssReasoningParser(gpt_oss_tokenizer)
|
|
||||||
output_ids = gpt_oss_tokenizer.convert_tokens_to_ids(tokens)
|
|
||||||
delta_ids = output_ids[-1:] if output_ids else []
|
|
||||||
actual = parser.is_reasoning_end_streaming(output_ids, delta_ids)
|
|
||||||
assert is_reasoning_end == actual
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.parametrize(
|
|
||||||
"output, is_reasoning_end",
|
|
||||||
[(t["output"], t["is_reasoning_end"]) for t in TEST_CASES],
|
|
||||||
)
|
|
||||||
def test_gptoss_is_reasoning_end_streaming_long_prefix(
|
|
||||||
output,
|
|
||||||
is_reasoning_end,
|
|
||||||
gpt_oss_tokenizer,
|
|
||||||
):
|
|
||||||
"""Windowing must produce correct results even with a long prefix."""
|
|
||||||
tokens = gpt_oss_tokenizer.tokenize(output)
|
|
||||||
parser: ReasoningParser = GptOssReasoningParser(gpt_oss_tokenizer)
|
|
||||||
output_ids = gpt_oss_tokenizer.convert_tokens_to_ids(tokens)
|
|
||||||
# Prepend 10k dummy reasoning tokens to simulate a long generation
|
|
||||||
long_prefix = [1] * 10_000
|
|
||||||
padded_ids = long_prefix + list(output_ids)
|
|
||||||
delta_ids = output_ids[-1:] if output_ids else []
|
|
||||||
actual = parser.is_reasoning_end_streaming(padded_ids, delta_ids)
|
|
||||||
assert is_reasoning_end == actual
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.parametrize(
|
|
||||||
"output, is_reasoning_end",
|
|
||||||
[(t["output"], t["is_reasoning_end"]) for t in TEST_CASES],
|
|
||||||
)
|
|
||||||
def test_gptoss_is_reasoning_end_streaming_large_delta(
|
|
||||||
output,
|
|
||||||
is_reasoning_end,
|
|
||||||
gpt_oss_tokenizer,
|
|
||||||
):
|
|
||||||
"""Simulate speculative decoding where the entire test sequence arrives
|
|
||||||
as a single large delta appended after a long prefix. The window must
|
|
||||||
expand to cover delta_ids so the end pattern is never missed."""
|
|
||||||
tokens = gpt_oss_tokenizer.tokenize(output)
|
|
||||||
parser: ReasoningParser = GptOssReasoningParser(gpt_oss_tokenizer)
|
|
||||||
output_ids = gpt_oss_tokenizer.convert_tokens_to_ids(tokens)
|
|
||||||
long_prefix = [1] * 10_000
|
|
||||||
padded_ids = long_prefix + list(output_ids)
|
|
||||||
# delta_ids = the entire test sequence (as if accepted in one spec step)
|
|
||||||
delta_ids = list(output_ids)
|
|
||||||
actual = parser.is_reasoning_end_streaming(padded_ids, delta_ids)
|
|
||||||
assert is_reasoning_end == actual
|
|
||||||
|
|
||||||
|
|
||||||
def test_gptoss_is_reasoning_end_streaming_signature(gpt_oss_tokenizer):
|
|
||||||
"""Verify the method is callable with the expected signature."""
|
|
||||||
parser = GptOssReasoningParser(gpt_oss_tokenizer)
|
|
||||||
result = parser.is_reasoning_end_streaming([], [])
|
|
||||||
assert result is False
|
|
||||||
|
|||||||
@@ -33,7 +33,7 @@ from vllm.tool_parsers.structural_tag_registry import (
|
|||||||
SUPPORTED_STRUCTURAL_TAG_MODELS,
|
SUPPORTED_STRUCTURAL_TAG_MODELS,
|
||||||
VLLM_BUILTIN_STRUCTURAL_TAG_MODELS,
|
VLLM_BUILTIN_STRUCTURAL_TAG_MODELS,
|
||||||
XGRAMMAR_BUILTIN_STRUCTURAL_TAG_MODELS,
|
XGRAMMAR_BUILTIN_STRUCTURAL_TAG_MODELS,
|
||||||
_get_function_parameters,
|
get_function_parameters,
|
||||||
get_model_structural_tag,
|
get_model_structural_tag,
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -530,7 +530,7 @@ def test_get_function_parameters_relaxes_function_strict_false():
|
|||||||
strict=False,
|
strict=False,
|
||||||
)
|
)
|
||||||
|
|
||||||
assert _get_function_parameters(function) is True
|
assert get_function_parameters(function) is True
|
||||||
|
|
||||||
|
|
||||||
def _k3_tools_with_root_defs() -> list[ChatCompletionToolsParam]:
|
def _k3_tools_with_root_defs() -> list[ChatCompletionToolsParam]:
|
||||||
|
|||||||
@@ -749,11 +749,14 @@ class OpenAIServingResponses(GenerateBaseServing):
|
|||||||
request: ResponsesRequest,
|
request: ResponsesRequest,
|
||||||
prev_response: ResponsesResponse | None,
|
prev_response: ResponsesResponse | None,
|
||||||
):
|
):
|
||||||
if request.tool_choice not in ("auto", "none"):
|
if self.parser is not None:
|
||||||
raise NotImplementedError(
|
# HarmonyParser doesn't need chat_template_kwargs
|
||||||
"Only 'auto' or 'none' tool_choice is supported "
|
# TODO: Unify adjust_request() call with non-harmony branch
|
||||||
"in response API with Harmony"
|
self.parser(
|
||||||
)
|
self.renderer.get_tokenizer(),
|
||||||
|
request.tools,
|
||||||
|
model_config=self.model_config,
|
||||||
|
).adjust_request(request=request)
|
||||||
|
|
||||||
arrival_time = time.time()
|
arrival_time = time.time()
|
||||||
messages = self._construct_input_messages_with_harmony(request, prev_response)
|
messages = self._construct_input_messages_with_harmony(request, prev_response)
|
||||||
|
|||||||
+230
-2
@@ -5,14 +5,31 @@ from __future__ import annotations
|
|||||||
|
|
||||||
import json
|
import json
|
||||||
from collections.abc import Sequence
|
from collections.abc import Sequence
|
||||||
from dataclasses import dataclass
|
from dataclasses import dataclass, replace
|
||||||
from enum import Enum, auto
|
from enum import Enum, auto
|
||||||
from typing import TYPE_CHECKING, NamedTuple
|
from typing import TYPE_CHECKING, NamedTuple
|
||||||
|
|
||||||
from openai_harmony import HarmonyError, Message, Role
|
from openai_harmony import HarmonyError, Message, Role
|
||||||
|
from xgrammar import StructuralTag
|
||||||
|
from xgrammar.openai_tool_call_schema import BuiltinToolParam, FunctionToolParam
|
||||||
|
from xgrammar.structural_tag import (
|
||||||
|
AnyTextFormat,
|
||||||
|
ConstStringFormat,
|
||||||
|
Format,
|
||||||
|
GrammarFormat,
|
||||||
|
JSONSchemaFormat,
|
||||||
|
OptionalFormat,
|
||||||
|
OrFormat,
|
||||||
|
RegexFormat,
|
||||||
|
SequenceFormat,
|
||||||
|
TagFormat,
|
||||||
|
TriggeredTagsFormat,
|
||||||
|
)
|
||||||
|
|
||||||
from vllm.entrypoints.chat_utils import make_tool_call_id
|
from vllm.entrypoints.chat_utils import make_tool_call_id
|
||||||
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
|
from vllm.entrypoints.openai.chat_completion.protocol import (
|
||||||
|
ChatCompletionRequest,
|
||||||
|
)
|
||||||
from vllm.entrypoints.openai.engine.protocol import (
|
from vllm.entrypoints.openai.engine.protocol import (
|
||||||
DeltaFunctionCall,
|
DeltaFunctionCall,
|
||||||
DeltaMessage,
|
DeltaMessage,
|
||||||
@@ -28,7 +45,13 @@ from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
|
|||||||
from vllm.logger import init_logger
|
from vllm.logger import init_logger
|
||||||
from vllm.parser.abstract_parser import DelegatingParser
|
from vllm.parser.abstract_parser import DelegatingParser
|
||||||
from vllm.reasoning.gptoss_reasoning_parser import GptOssReasoningParser
|
from vllm.reasoning.gptoss_reasoning_parser import GptOssReasoningParser
|
||||||
|
from vllm.sampling_params import StructuredOutputsParams
|
||||||
from vllm.tool_parsers.gptoss_tool_parser import GptOssToolParser
|
from vllm.tool_parsers.gptoss_tool_parser import GptOssToolParser
|
||||||
|
from vllm.tool_parsers.structural_tag_registry import (
|
||||||
|
SimplifiedToolChoice,
|
||||||
|
get_function_parameters,
|
||||||
|
register_vllm_structural_tag,
|
||||||
|
)
|
||||||
|
|
||||||
if TYPE_CHECKING:
|
if TYPE_CHECKING:
|
||||||
from openai_harmony import Message, StreamableParser
|
from openai_harmony import Message, StreamableParser
|
||||||
@@ -346,6 +369,12 @@ class HarmonyParser(DelegatingParser):
|
|||||||
reasoning_token_count=reasoning_token_count,
|
reasoning_token_count=reasoning_token_count,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
def adjust_request(
|
||||||
|
self, request: ChatCompletionRequest | ResponsesRequest
|
||||||
|
) -> ChatCompletionRequest | ResponsesRequest:
|
||||||
|
request = _adjust_output_format(request)
|
||||||
|
return super().adjust_request(request)
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def _normalize_recipient(recipient: str | None) -> str | None:
|
def _normalize_recipient(recipient: str | None) -> str | None:
|
||||||
"""Remove constrained formats misparsed into recipients by older Harmony."""
|
"""Remove constrained formats misparsed into recipients by older Harmony."""
|
||||||
@@ -356,3 +385,202 @@ class HarmonyParser(DelegatingParser):
|
|||||||
if constrain_index == -1:
|
if constrain_index == -1:
|
||||||
return recipient
|
return recipient
|
||||||
return recipient[:constrain_index].rstrip() or None
|
return recipient[:constrain_index].rstrip() or None
|
||||||
|
|
||||||
|
|
||||||
|
# Harmomy's stop tokens are <|return|>, <|call|>, <|endoftext|>
|
||||||
|
# <|return|> is represented as "" since it's the default stop token, which xgrammar
|
||||||
|
# disallows under constraints, leading to bad or infinite generation.
|
||||||
|
# StreamableParser doesn't consider <|endoftext|> as a message end, so it's excluded
|
||||||
|
# TODO: Remove <|call|> once #50595 lands.
|
||||||
|
_END_TAG = ["<|end|>", "<|call|>", ""]
|
||||||
|
_FINAL_BEGIN = "<|channel|>final{constrain}<|message|>"
|
||||||
|
_TOOL_CALL_CHANNELS = [
|
||||||
|
"<|channel|>commentary",
|
||||||
|
"<|channel|>analysis",
|
||||||
|
"<|channel|>final",
|
||||||
|
]
|
||||||
|
_FUNCTION_CALL_BEGINS = [
|
||||||
|
"to=functions.{name} {channel} json<|message|>",
|
||||||
|
"to=functions.{name} {channel} <|constrain|>json<|message|>",
|
||||||
|
"{channel} to=functions.{name} json<|message|>",
|
||||||
|
"{channel} to=functions.{name} <|constrain|>json<|message|>",
|
||||||
|
]
|
||||||
|
_JSON_CONTENT = JSONSchemaFormat(json_schema={"type": "object"})
|
||||||
|
_ANY_CONTENT = AnyTextFormat()
|
||||||
|
|
||||||
|
|
||||||
|
def _assemble_tag(
|
||||||
|
allow_analysis: bool, allow_commentary: bool, content: Format
|
||||||
|
) -> StructuralTag:
|
||||||
|
tags = []
|
||||||
|
if allow_analysis:
|
||||||
|
analysis_tag = OptionalFormat(
|
||||||
|
content=SequenceFormat(
|
||||||
|
elements=[
|
||||||
|
TagFormat(
|
||||||
|
begin="<|channel|>analysis<|message|>",
|
||||||
|
content=_ANY_CONTENT,
|
||||||
|
end="<|end|>",
|
||||||
|
),
|
||||||
|
ConstStringFormat(value="<|start|>assistant"),
|
||||||
|
]
|
||||||
|
)
|
||||||
|
)
|
||||||
|
tags.append(analysis_tag)
|
||||||
|
|
||||||
|
if allow_commentary:
|
||||||
|
commentary_tag = OptionalFormat(
|
||||||
|
content=SequenceFormat(
|
||||||
|
elements=[
|
||||||
|
TagFormat(
|
||||||
|
begin="<|channel|>commentary<|message|>",
|
||||||
|
content=_ANY_CONTENT,
|
||||||
|
end="<|end|>",
|
||||||
|
),
|
||||||
|
ConstStringFormat(value="<|start|>assistant"),
|
||||||
|
]
|
||||||
|
)
|
||||||
|
)
|
||||||
|
tags.append(commentary_tag)
|
||||||
|
|
||||||
|
tags.append(content)
|
||||||
|
|
||||||
|
return StructuralTag(format=SequenceFormat(elements=tags))
|
||||||
|
|
||||||
|
|
||||||
|
@register_vllm_structural_tag("harmony")
|
||||||
|
def get_harmony_structural_tag(
|
||||||
|
tools: list[FunctionToolParam],
|
||||||
|
builtin_tools: list[BuiltinToolParam],
|
||||||
|
tool_choice: SimplifiedToolChoice,
|
||||||
|
reasoning: bool,
|
||||||
|
) -> StructuralTag:
|
||||||
|
# reasoning always enabled for Harmony
|
||||||
|
del reasoning
|
||||||
|
|
||||||
|
if builtin_tools:
|
||||||
|
# Fallback for built-in tools
|
||||||
|
tags = [
|
||||||
|
TagFormat(
|
||||||
|
begin="to=",
|
||||||
|
content=AnyTextFormat(excludes=["<|start|>"]),
|
||||||
|
end=_END_TAG,
|
||||||
|
)
|
||||||
|
]
|
||||||
|
tags.extend(
|
||||||
|
TagFormat(
|
||||||
|
begin=channel + " to=",
|
||||||
|
content=AnyTextFormat(excludes=["<|start|>", "<|channel|>"]),
|
||||||
|
end=_END_TAG,
|
||||||
|
)
|
||||||
|
for channel in _TOOL_CALL_CHANNELS
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
tags = [
|
||||||
|
TagFormat(
|
||||||
|
begin=pattern.format(name=tool.function.name, channel=channel),
|
||||||
|
content=JSONSchemaFormat(
|
||||||
|
json_schema=get_function_parameters(tool.function)
|
||||||
|
),
|
||||||
|
end=_END_TAG,
|
||||||
|
)
|
||||||
|
for tool in tools
|
||||||
|
for pattern in _FUNCTION_CALL_BEGINS
|
||||||
|
for channel in _TOOL_CALL_CHANNELS
|
||||||
|
]
|
||||||
|
|
||||||
|
if tool_choice == "auto":
|
||||||
|
tags.append(
|
||||||
|
TagFormat(
|
||||||
|
begin=_FINAL_BEGIN.format(constrain=" <|constrain|>json"),
|
||||||
|
content=_ANY_CONTENT,
|
||||||
|
end=_END_TAG,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
tags.append(
|
||||||
|
TagFormat(
|
||||||
|
begin=_FINAL_BEGIN.format(constrain=""),
|
||||||
|
content=_ANY_CONTENT,
|
||||||
|
end=_END_TAG,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
return _assemble_tag(
|
||||||
|
allow_analysis=True, allow_commentary=True, content=OrFormat(elements=tags)
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _params_to_final_content(params: StructuredOutputsParams) -> Format | None:
|
||||||
|
"""Map StructuredOutputsParams in a XGrammar Format."""
|
||||||
|
if params.json_object:
|
||||||
|
return _JSON_CONTENT
|
||||||
|
if params.json is not None:
|
||||||
|
schema = params.json
|
||||||
|
if isinstance(schema, str):
|
||||||
|
schema = json.loads(schema)
|
||||||
|
return JSONSchemaFormat(json_schema=schema)
|
||||||
|
if params.regex is not None:
|
||||||
|
return RegexFormat(pattern=params.regex)
|
||||||
|
if params.choice is not None:
|
||||||
|
return OrFormat(
|
||||||
|
elements=[ConstStringFormat(value=choice) for choice in params.choice]
|
||||||
|
)
|
||||||
|
if params.grammar is not None:
|
||||||
|
return GrammarFormat(grammar=params.grammar)
|
||||||
|
if params.structural_tag is not None:
|
||||||
|
s_tag = json.loads(params.structural_tag)
|
||||||
|
if "structures" in s_tag:
|
||||||
|
# LegacyStructuralTagResponseFormat
|
||||||
|
return TriggeredTagsFormat(
|
||||||
|
triggers=s_tag["triggers"],
|
||||||
|
tags=[
|
||||||
|
TagFormat(
|
||||||
|
begin=structure["begin"],
|
||||||
|
content=JSONSchemaFormat(json_schema=structure["schema"]),
|
||||||
|
end=structure["end"],
|
||||||
|
)
|
||||||
|
for structure in s_tag["structures"]
|
||||||
|
],
|
||||||
|
)
|
||||||
|
# StructuralTagResponseFormat
|
||||||
|
return StructuralTag.model_validate(s_tag).format
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def _adjust_output_format(
|
||||||
|
request: ChatCompletionRequest | ResponsesRequest,
|
||||||
|
) -> ChatCompletionRequest | ResponsesRequest:
|
||||||
|
"""Canonicalize request constraints into a reasoning-aware StructuralTag."""
|
||||||
|
params = request.extract_structured_outputs()
|
||||||
|
if params is None:
|
||||||
|
return request
|
||||||
|
|
||||||
|
final_content = _params_to_final_content(params)
|
||||||
|
if final_content is None:
|
||||||
|
return request
|
||||||
|
|
||||||
|
if isinstance(final_content, JSONSchemaFormat):
|
||||||
|
begin = _FINAL_BEGIN.format(constrain=" <|constrain|>json")
|
||||||
|
else:
|
||||||
|
begin = _FINAL_BEGIN.format(constrain="")
|
||||||
|
|
||||||
|
structural_tag = _assemble_tag(
|
||||||
|
allow_analysis=True,
|
||||||
|
allow_commentary=False,
|
||||||
|
content=TagFormat(begin=begin, content=final_content, end=_END_TAG),
|
||||||
|
)
|
||||||
|
|
||||||
|
request.structured_outputs = replace(
|
||||||
|
params,
|
||||||
|
json=None,
|
||||||
|
regex=None,
|
||||||
|
choice=None,
|
||||||
|
grammar=None,
|
||||||
|
json_object=None,
|
||||||
|
structural_tag=json.dumps(structural_tag.model_dump()),
|
||||||
|
)
|
||||||
|
if isinstance(request, ResponsesRequest):
|
||||||
|
request.text = None
|
||||||
|
else:
|
||||||
|
request.response_format = None
|
||||||
|
return request
|
||||||
|
|||||||
@@ -1,65 +1,17 @@
|
|||||||
# SPDX-License-Identifier: Apache-2.0
|
# SPDX-License-Identifier: Apache-2.0
|
||||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||||
import json
|
|
||||||
from collections.abc import Iterable, Sequence
|
from collections.abc import Iterable, Sequence
|
||||||
from typing import TYPE_CHECKING
|
from typing import TYPE_CHECKING
|
||||||
|
|
||||||
from transformers import PreTrainedTokenizerBase
|
from transformers import PreTrainedTokenizerBase
|
||||||
|
|
||||||
from vllm.entrypoints.mcp.tool_server import ToolServer
|
|
||||||
from vllm.entrypoints.openai.engine.protocol import DeltaMessage
|
from vllm.entrypoints.openai.engine.protocol import DeltaMessage
|
||||||
from vllm.logger import init_logger
|
|
||||||
from vllm.reasoning import ReasoningParser
|
from vllm.reasoning import ReasoningParser
|
||||||
|
|
||||||
if TYPE_CHECKING:
|
if TYPE_CHECKING:
|
||||||
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
|
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
|
||||||
from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
|
from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
|
||||||
|
|
||||||
logger = init_logger(__name__)
|
|
||||||
|
|
||||||
no_func_reasoning_tag = {
|
|
||||||
"type": "structural_tag",
|
|
||||||
"format": {
|
|
||||||
"type": "triggered_tags",
|
|
||||||
"tags": [
|
|
||||||
{
|
|
||||||
"begin": "<|channel|>analysis<|message|>",
|
|
||||||
"content": {"type": "any_text"},
|
|
||||||
"end": "<|end|>",
|
|
||||||
}
|
|
||||||
],
|
|
||||||
"triggers": ["<|channel|>analysis"],
|
|
||||||
"stop_after_first": False,
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
def from_builtin_tool_to_tag(tool: str) -> list[dict]:
|
|
||||||
tag = [
|
|
||||||
{
|
|
||||||
"begin": f"<|channel|>commentary to={tool}",
|
|
||||||
"content": {"type": "any_text"},
|
|
||||||
"end": "<|end|>",
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"begin": f"<|channel|>analysis to={tool}",
|
|
||||||
"content": {"type": "any_text"},
|
|
||||||
"end": "<|end|>",
|
|
||||||
},
|
|
||||||
]
|
|
||||||
return tag
|
|
||||||
|
|
||||||
|
|
||||||
def tag_with_builtin_funcs(no_func_reasoning_tag, builtin_tool_list: list[str]) -> dict:
|
|
||||||
import copy
|
|
||||||
|
|
||||||
new_tag = copy.deepcopy(no_func_reasoning_tag)
|
|
||||||
new_tag["format"]["triggers"].append("<|channel|>commentary to=")
|
|
||||||
|
|
||||||
for tool in builtin_tool_list:
|
|
||||||
new_tag["format"]["tags"].extend(from_builtin_tool_to_tag(tool))
|
|
||||||
return new_tag
|
|
||||||
|
|
||||||
|
|
||||||
class GptOssReasoningParser(ReasoningParser):
|
class GptOssReasoningParser(ReasoningParser):
|
||||||
"""
|
"""
|
||||||
@@ -71,64 +23,14 @@ class GptOssReasoningParser(ReasoningParser):
|
|||||||
|
|
||||||
def __init__(self, tokenizer: PreTrainedTokenizerBase, *args, **kwargs):
|
def __init__(self, tokenizer: PreTrainedTokenizerBase, *args, **kwargs):
|
||||||
super().__init__(tokenizer, *args, **kwargs)
|
super().__init__(tokenizer, *args, **kwargs)
|
||||||
# The model can output some special tokens between "final" and "<|message|>"
|
|
||||||
# So we need to look for both sequences to determine the end of reasoning.
|
|
||||||
self.reasoning_end_token_ids_prefix = self.model_tokenizer.encode(
|
|
||||||
"<|channel|>final"
|
|
||||||
)
|
|
||||||
self.reasoning_end_token_ids_suffix = self.model_tokenizer.encode("<|message|>")
|
|
||||||
# We also need to check for the <|end|> token to avoid false positives from
|
|
||||||
# previous messages in multi-turn conversations.
|
|
||||||
self.eom_token_id = self.vocab["<|end|>"]
|
|
||||||
self.reasoning_max_num_between_tokens = 20
|
|
||||||
|
|
||||||
def is_reasoning_end(self, input_ids: Sequence[int]) -> bool:
|
def is_reasoning_end(self, input_ids: Sequence[int]) -> bool:
|
||||||
end_token_ids_prefix = self.reasoning_end_token_ids_prefix
|
return True
|
||||||
end_token_ids_suffix = self.reasoning_end_token_ids_suffix
|
|
||||||
assert len(end_token_ids_prefix) > 0, "reasoning_end_token_ids_prefix is empty"
|
|
||||||
assert len(end_token_ids_suffix) > 0, "reasoning_end_token_ids_suffix is empty"
|
|
||||||
# Check if the end sequence is present in the input_ids.
|
|
||||||
# We search from the end of input_ids to find the last match.
|
|
||||||
for i in range(len(input_ids) - len(end_token_ids_prefix), -1, -1):
|
|
||||||
if input_ids[i] == self.eom_token_id:
|
|
||||||
# We looped backwards far enough to find the end of a previous message,
|
|
||||||
# which means we have searched the entirety of the current message
|
|
||||||
# and can exit early without searching further back into prior
|
|
||||||
# messages of the conversation.
|
|
||||||
return False
|
|
||||||
if input_ids[i : i + len(end_token_ids_prefix)] == end_token_ids_prefix:
|
|
||||||
# We have found the prefix, now we look for the suffix after the prefix.
|
|
||||||
suffix_start = i + len(end_token_ids_prefix)
|
|
||||||
for j in range(
|
|
||||||
suffix_start, len(input_ids) - len(end_token_ids_suffix) + 1
|
|
||||||
):
|
|
||||||
if j - suffix_start >= self.reasoning_max_num_between_tokens:
|
|
||||||
break
|
|
||||||
if (
|
|
||||||
input_ids[j : j + len(end_token_ids_suffix)]
|
|
||||||
== end_token_ids_suffix
|
|
||||||
):
|
|
||||||
return True
|
|
||||||
return False
|
|
||||||
|
|
||||||
def is_reasoning_end_streaming(
|
def is_reasoning_end_streaming(
|
||||||
self, input_ids: Sequence[int], delta_ids: Iterable[int]
|
self, input_ids: Sequence[int], delta_ids: Iterable[int]
|
||||||
) -> bool:
|
) -> bool:
|
||||||
# The pattern window covers the end-of-reasoning marker itself.
|
return True
|
||||||
# We add len(delta_ids) so that under speculative decoding (where
|
|
||||||
# a single step can accept many tokens) the entire accepted chunk
|
|
||||||
# is always inside the scan region.
|
|
||||||
delta_ids = tuple(delta_ids)
|
|
||||||
pattern_len = (
|
|
||||||
len(self.reasoning_end_token_ids_prefix)
|
|
||||||
+ self.reasoning_max_num_between_tokens
|
|
||||||
+ len(self.reasoning_end_token_ids_suffix)
|
|
||||||
)
|
|
||||||
window = pattern_len + len(delta_ids)
|
|
||||||
n = len(input_ids)
|
|
||||||
if n <= window:
|
|
||||||
return self.is_reasoning_end(input_ids)
|
|
||||||
return self.is_reasoning_end(input_ids[n - window :])
|
|
||||||
|
|
||||||
def extract_content_ids(self, input_ids: list[int]) -> list[int]:
|
def extract_content_ids(self, input_ids: list[int]) -> list[int]:
|
||||||
raise NotImplementedError(
|
raise NotImplementedError(
|
||||||
@@ -159,33 +61,3 @@ class GptOssReasoningParser(ReasoningParser):
|
|||||||
"GptOssReasoningParser only provides boundary detection. "
|
"GptOssReasoningParser only provides boundary detection. "
|
||||||
"Use HarmonyParser for output parsing."
|
"Use HarmonyParser for output parsing."
|
||||||
)
|
)
|
||||||
|
|
||||||
# This function prepares the structural tag to format reasoning output
|
|
||||||
def prepare_structured_tag(
|
|
||||||
self, original_tag: str | None, tool_server: ToolServer | None
|
|
||||||
) -> str | None:
|
|
||||||
if original_tag is None:
|
|
||||||
if tool_server is None:
|
|
||||||
return json.dumps(no_func_reasoning_tag)
|
|
||||||
else:
|
|
||||||
builtin_tool_list: list[str] = []
|
|
||||||
if tool_server.has_tool("browser"):
|
|
||||||
builtin_tool_list.append("browser")
|
|
||||||
if tool_server.has_tool("python"):
|
|
||||||
builtin_tool_list.append("python")
|
|
||||||
if tool_server.has_tool("container"):
|
|
||||||
builtin_tool_list.append("container")
|
|
||||||
|
|
||||||
if len(builtin_tool_list) > 0:
|
|
||||||
logger.info("Builtin_tool_list: %s", builtin_tool_list)
|
|
||||||
func_tag = json.dumps(
|
|
||||||
tag_with_builtin_funcs(no_func_reasoning_tag, builtin_tool_list)
|
|
||||||
)
|
|
||||||
else:
|
|
||||||
logger.info("Builtin_tool_list is empty")
|
|
||||||
func_tag = json.dumps(no_func_reasoning_tag)
|
|
||||||
|
|
||||||
return func_tag
|
|
||||||
else:
|
|
||||||
# There is potential risk for appending the tag to the original tag
|
|
||||||
return original_tag
|
|
||||||
|
|||||||
@@ -201,6 +201,15 @@ class OnlineRenderer:
|
|||||||
)
|
)
|
||||||
else:
|
else:
|
||||||
# For GPT-OSS.
|
# For GPT-OSS.
|
||||||
|
if self.parser is not None:
|
||||||
|
# HarmonyParser doesn't need chat_template_kwargs
|
||||||
|
# TODO: Unify adjust_request() call with non-harmony branch
|
||||||
|
self.parser(
|
||||||
|
self.renderer.get_tokenizer(),
|
||||||
|
request.tools,
|
||||||
|
model_config=self.model_config,
|
||||||
|
).adjust_request(request=request)
|
||||||
|
|
||||||
should_include_tools = tool_dicts is not None
|
should_include_tools = tool_dicts is not None
|
||||||
conversation, engine_inputs = self._make_request_with_harmony(
|
conversation, engine_inputs = self._make_request_with_harmony(
|
||||||
request, should_include_tools
|
request, should_include_tools
|
||||||
|
|||||||
@@ -22,6 +22,8 @@ class GptOssToolParser(ToolParser):
|
|||||||
capability declaration via HarmonyParser.tool_parser_cls.
|
capability declaration via HarmonyParser.tool_parser_cls.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
|
structural_tag_model = "harmony"
|
||||||
|
|
||||||
def __init__(self, tokenizer: "TokenizerLike", tools: list[Tool] | None = None):
|
def __init__(self, tokenizer: "TokenizerLike", tools: list[Tool] | None = None):
|
||||||
super().__init__(tokenizer, tools)
|
super().__init__(tokenizer, tools)
|
||||||
|
|
||||||
|
|||||||
@@ -65,7 +65,6 @@ XGRAMMAR_BUILTIN_STRUCTURAL_TAG_MODELS = frozenset(
|
|||||||
"qwen_3_5",
|
"qwen_3_5",
|
||||||
"qwen_3_coder",
|
"qwen_3_coder",
|
||||||
"qwen_3",
|
"qwen_3",
|
||||||
"harmony",
|
|
||||||
"deepseek_v3_2",
|
"deepseek_v3_2",
|
||||||
"glm_4_7",
|
"glm_4_7",
|
||||||
"deepseek_v4",
|
"deepseek_v4",
|
||||||
@@ -209,7 +208,7 @@ def _dump_allowed_tool_ref_for_xgrammar(tool_ref: AllowedToolRef) -> AllowedTool
|
|||||||
return tool_ref
|
return tool_ref
|
||||||
|
|
||||||
|
|
||||||
def _get_function_parameters(function) -> dict[str, Any] | bool:
|
def get_function_parameters(function) -> dict[str, Any] | bool:
|
||||||
if getattr(function, "strict", None) is False:
|
if getattr(function, "strict", None) is False:
|
||||||
return True
|
return True
|
||||||
return function.parameters if function.parameters is not None else True
|
return function.parameters if function.parameters is not None else True
|
||||||
@@ -230,7 +229,7 @@ def _hermes_tool_tags(tools: list[FunctionToolParam]) -> list[TagFormat]:
|
|||||||
TagFormat(
|
TagFormat(
|
||||||
begin=begin + tool.function.name + arguments_field_prefix,
|
begin=begin + tool.function.name + arguments_field_prefix,
|
||||||
content=JSONSchemaFormat(
|
content=JSONSchemaFormat(
|
||||||
json_schema=_get_function_parameters(tool.function)
|
json_schema=get_function_parameters(tool.function)
|
||||||
),
|
),
|
||||||
end=end,
|
end=end,
|
||||||
)
|
)
|
||||||
@@ -279,7 +278,7 @@ def _minimax_tool_tags(tools: list[FunctionToolParam]) -> list[TagFormat]:
|
|||||||
TagFormat(
|
TagFormat(
|
||||||
begin=f'<invoke name="{tool.function.name}">\n',
|
begin=f'<invoke name="{tool.function.name}">\n',
|
||||||
content=JSONSchemaFormat(
|
content=JSONSchemaFormat(
|
||||||
json_schema=_get_function_parameters(tool.function),
|
json_schema=get_function_parameters(tool.function),
|
||||||
style="minimax_xml",
|
style="minimax_xml",
|
||||||
),
|
),
|
||||||
end="</invoke>\n",
|
end="</invoke>\n",
|
||||||
@@ -537,7 +536,7 @@ def _k3_arguments_block(parameters: dict[str, Any] | bool) -> Any:
|
|||||||
def _k3_call_tag(tool: FunctionToolParam) -> TagFormat:
|
def _k3_call_tag(tool: FunctionToolParam) -> TagFormat:
|
||||||
"""One ``call`` tag: ``<|open|>call tool="N" index="<digits>"<|sep|> args``."""
|
"""One ``call`` tag: ``<|open|>call tool="N" index="<digits>"<|sep|> args``."""
|
||||||
function = tool.function
|
function = tool.function
|
||||||
parameters = _get_function_parameters(function)
|
parameters = get_function_parameters(function)
|
||||||
begin = f'{_K3_OPEN}call tool="{_k3_escape_attr(function.name)}" index="'
|
begin = f'{_K3_OPEN}call tool="{_k3_escape_attr(function.name)}" index="'
|
||||||
return TagFormat(
|
return TagFormat(
|
||||||
begin=begin,
|
begin=begin,
|
||||||
|
|||||||
Reference in New Issue
Block a user