[GPT-OSS] Strict tool call and constrained decoding for Harmony (#45560)
Signed-off-by: Yifan Zong <yzong@redhat.com>
This commit is contained in:
@@ -749,11 +749,14 @@ class OpenAIServingResponses(GenerateBaseServing):
|
||||
request: ResponsesRequest,
|
||||
prev_response: ResponsesResponse | None,
|
||||
):
|
||||
if request.tool_choice not in ("auto", "none"):
|
||||
raise NotImplementedError(
|
||||
"Only 'auto' or 'none' tool_choice is supported "
|
||||
"in response API with Harmony"
|
||||
)
|
||||
if self.parser is not None:
|
||||
# HarmonyParser doesn't need chat_template_kwargs
|
||||
# TODO: Unify adjust_request() call with non-harmony branch
|
||||
self.parser(
|
||||
self.renderer.get_tokenizer(),
|
||||
request.tools,
|
||||
model_config=self.model_config,
|
||||
).adjust_request(request=request)
|
||||
|
||||
arrival_time = time.time()
|
||||
messages = self._construct_input_messages_with_harmony(request, prev_response)
|
||||
|
||||
+230
-2
@@ -5,14 +5,31 @@ from __future__ import annotations
|
||||
|
||||
import json
|
||||
from collections.abc import Sequence
|
||||
from dataclasses import dataclass
|
||||
from dataclasses import dataclass, replace
|
||||
from enum import Enum, auto
|
||||
from typing import TYPE_CHECKING, NamedTuple
|
||||
|
||||
from openai_harmony import HarmonyError, Message, Role
|
||||
from xgrammar import StructuralTag
|
||||
from xgrammar.openai_tool_call_schema import BuiltinToolParam, FunctionToolParam
|
||||
from xgrammar.structural_tag import (
|
||||
AnyTextFormat,
|
||||
ConstStringFormat,
|
||||
Format,
|
||||
GrammarFormat,
|
||||
JSONSchemaFormat,
|
||||
OptionalFormat,
|
||||
OrFormat,
|
||||
RegexFormat,
|
||||
SequenceFormat,
|
||||
TagFormat,
|
||||
TriggeredTagsFormat,
|
||||
)
|
||||
|
||||
from vllm.entrypoints.chat_utils import make_tool_call_id
|
||||
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
|
||||
from vllm.entrypoints.openai.chat_completion.protocol import (
|
||||
ChatCompletionRequest,
|
||||
)
|
||||
from vllm.entrypoints.openai.engine.protocol import (
|
||||
DeltaFunctionCall,
|
||||
DeltaMessage,
|
||||
@@ -28,7 +45,13 @@ from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
|
||||
from vllm.logger import init_logger
|
||||
from vllm.parser.abstract_parser import DelegatingParser
|
||||
from vllm.reasoning.gptoss_reasoning_parser import GptOssReasoningParser
|
||||
from vllm.sampling_params import StructuredOutputsParams
|
||||
from vllm.tool_parsers.gptoss_tool_parser import GptOssToolParser
|
||||
from vllm.tool_parsers.structural_tag_registry import (
|
||||
SimplifiedToolChoice,
|
||||
get_function_parameters,
|
||||
register_vllm_structural_tag,
|
||||
)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from openai_harmony import Message, StreamableParser
|
||||
@@ -346,6 +369,12 @@ class HarmonyParser(DelegatingParser):
|
||||
reasoning_token_count=reasoning_token_count,
|
||||
)
|
||||
|
||||
def adjust_request(
|
||||
self, request: ChatCompletionRequest | ResponsesRequest
|
||||
) -> ChatCompletionRequest | ResponsesRequest:
|
||||
request = _adjust_output_format(request)
|
||||
return super().adjust_request(request)
|
||||
|
||||
@staticmethod
|
||||
def _normalize_recipient(recipient: str | None) -> str | None:
|
||||
"""Remove constrained formats misparsed into recipients by older Harmony."""
|
||||
@@ -356,3 +385,202 @@ class HarmonyParser(DelegatingParser):
|
||||
if constrain_index == -1:
|
||||
return recipient
|
||||
return recipient[:constrain_index].rstrip() or None
|
||||
|
||||
|
||||
# Harmomy's stop tokens are <|return|>, <|call|>, <|endoftext|>
|
||||
# <|return|> is represented as "" since it's the default stop token, which xgrammar
|
||||
# disallows under constraints, leading to bad or infinite generation.
|
||||
# StreamableParser doesn't consider <|endoftext|> as a message end, so it's excluded
|
||||
# TODO: Remove <|call|> once #50595 lands.
|
||||
_END_TAG = ["<|end|>", "<|call|>", ""]
|
||||
_FINAL_BEGIN = "<|channel|>final{constrain}<|message|>"
|
||||
_TOOL_CALL_CHANNELS = [
|
||||
"<|channel|>commentary",
|
||||
"<|channel|>analysis",
|
||||
"<|channel|>final",
|
||||
]
|
||||
_FUNCTION_CALL_BEGINS = [
|
||||
"to=functions.{name} {channel} json<|message|>",
|
||||
"to=functions.{name} {channel} <|constrain|>json<|message|>",
|
||||
"{channel} to=functions.{name} json<|message|>",
|
||||
"{channel} to=functions.{name} <|constrain|>json<|message|>",
|
||||
]
|
||||
_JSON_CONTENT = JSONSchemaFormat(json_schema={"type": "object"})
|
||||
_ANY_CONTENT = AnyTextFormat()
|
||||
|
||||
|
||||
def _assemble_tag(
|
||||
allow_analysis: bool, allow_commentary: bool, content: Format
|
||||
) -> StructuralTag:
|
||||
tags = []
|
||||
if allow_analysis:
|
||||
analysis_tag = OptionalFormat(
|
||||
content=SequenceFormat(
|
||||
elements=[
|
||||
TagFormat(
|
||||
begin="<|channel|>analysis<|message|>",
|
||||
content=_ANY_CONTENT,
|
||||
end="<|end|>",
|
||||
),
|
||||
ConstStringFormat(value="<|start|>assistant"),
|
||||
]
|
||||
)
|
||||
)
|
||||
tags.append(analysis_tag)
|
||||
|
||||
if allow_commentary:
|
||||
commentary_tag = OptionalFormat(
|
||||
content=SequenceFormat(
|
||||
elements=[
|
||||
TagFormat(
|
||||
begin="<|channel|>commentary<|message|>",
|
||||
content=_ANY_CONTENT,
|
||||
end="<|end|>",
|
||||
),
|
||||
ConstStringFormat(value="<|start|>assistant"),
|
||||
]
|
||||
)
|
||||
)
|
||||
tags.append(commentary_tag)
|
||||
|
||||
tags.append(content)
|
||||
|
||||
return StructuralTag(format=SequenceFormat(elements=tags))
|
||||
|
||||
|
||||
@register_vllm_structural_tag("harmony")
|
||||
def get_harmony_structural_tag(
|
||||
tools: list[FunctionToolParam],
|
||||
builtin_tools: list[BuiltinToolParam],
|
||||
tool_choice: SimplifiedToolChoice,
|
||||
reasoning: bool,
|
||||
) -> StructuralTag:
|
||||
# reasoning always enabled for Harmony
|
||||
del reasoning
|
||||
|
||||
if builtin_tools:
|
||||
# Fallback for built-in tools
|
||||
tags = [
|
||||
TagFormat(
|
||||
begin="to=",
|
||||
content=AnyTextFormat(excludes=["<|start|>"]),
|
||||
end=_END_TAG,
|
||||
)
|
||||
]
|
||||
tags.extend(
|
||||
TagFormat(
|
||||
begin=channel + " to=",
|
||||
content=AnyTextFormat(excludes=["<|start|>", "<|channel|>"]),
|
||||
end=_END_TAG,
|
||||
)
|
||||
for channel in _TOOL_CALL_CHANNELS
|
||||
)
|
||||
else:
|
||||
tags = [
|
||||
TagFormat(
|
||||
begin=pattern.format(name=tool.function.name, channel=channel),
|
||||
content=JSONSchemaFormat(
|
||||
json_schema=get_function_parameters(tool.function)
|
||||
),
|
||||
end=_END_TAG,
|
||||
)
|
||||
for tool in tools
|
||||
for pattern in _FUNCTION_CALL_BEGINS
|
||||
for channel in _TOOL_CALL_CHANNELS
|
||||
]
|
||||
|
||||
if tool_choice == "auto":
|
||||
tags.append(
|
||||
TagFormat(
|
||||
begin=_FINAL_BEGIN.format(constrain=" <|constrain|>json"),
|
||||
content=_ANY_CONTENT,
|
||||
end=_END_TAG,
|
||||
)
|
||||
)
|
||||
tags.append(
|
||||
TagFormat(
|
||||
begin=_FINAL_BEGIN.format(constrain=""),
|
||||
content=_ANY_CONTENT,
|
||||
end=_END_TAG,
|
||||
)
|
||||
)
|
||||
|
||||
return _assemble_tag(
|
||||
allow_analysis=True, allow_commentary=True, content=OrFormat(elements=tags)
|
||||
)
|
||||
|
||||
|
||||
def _params_to_final_content(params: StructuredOutputsParams) -> Format | None:
|
||||
"""Map StructuredOutputsParams in a XGrammar Format."""
|
||||
if params.json_object:
|
||||
return _JSON_CONTENT
|
||||
if params.json is not None:
|
||||
schema = params.json
|
||||
if isinstance(schema, str):
|
||||
schema = json.loads(schema)
|
||||
return JSONSchemaFormat(json_schema=schema)
|
||||
if params.regex is not None:
|
||||
return RegexFormat(pattern=params.regex)
|
||||
if params.choice is not None:
|
||||
return OrFormat(
|
||||
elements=[ConstStringFormat(value=choice) for choice in params.choice]
|
||||
)
|
||||
if params.grammar is not None:
|
||||
return GrammarFormat(grammar=params.grammar)
|
||||
if params.structural_tag is not None:
|
||||
s_tag = json.loads(params.structural_tag)
|
||||
if "structures" in s_tag:
|
||||
# LegacyStructuralTagResponseFormat
|
||||
return TriggeredTagsFormat(
|
||||
triggers=s_tag["triggers"],
|
||||
tags=[
|
||||
TagFormat(
|
||||
begin=structure["begin"],
|
||||
content=JSONSchemaFormat(json_schema=structure["schema"]),
|
||||
end=structure["end"],
|
||||
)
|
||||
for structure in s_tag["structures"]
|
||||
],
|
||||
)
|
||||
# StructuralTagResponseFormat
|
||||
return StructuralTag.model_validate(s_tag).format
|
||||
return None
|
||||
|
||||
|
||||
def _adjust_output_format(
|
||||
request: ChatCompletionRequest | ResponsesRequest,
|
||||
) -> ChatCompletionRequest | ResponsesRequest:
|
||||
"""Canonicalize request constraints into a reasoning-aware StructuralTag."""
|
||||
params = request.extract_structured_outputs()
|
||||
if params is None:
|
||||
return request
|
||||
|
||||
final_content = _params_to_final_content(params)
|
||||
if final_content is None:
|
||||
return request
|
||||
|
||||
if isinstance(final_content, JSONSchemaFormat):
|
||||
begin = _FINAL_BEGIN.format(constrain=" <|constrain|>json")
|
||||
else:
|
||||
begin = _FINAL_BEGIN.format(constrain="")
|
||||
|
||||
structural_tag = _assemble_tag(
|
||||
allow_analysis=True,
|
||||
allow_commentary=False,
|
||||
content=TagFormat(begin=begin, content=final_content, end=_END_TAG),
|
||||
)
|
||||
|
||||
request.structured_outputs = replace(
|
||||
params,
|
||||
json=None,
|
||||
regex=None,
|
||||
choice=None,
|
||||
grammar=None,
|
||||
json_object=None,
|
||||
structural_tag=json.dumps(structural_tag.model_dump()),
|
||||
)
|
||||
if isinstance(request, ResponsesRequest):
|
||||
request.text = None
|
||||
else:
|
||||
request.response_format = None
|
||||
return request
|
||||
|
||||
@@ -1,65 +1,17 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import json
|
||||
from collections.abc import Iterable, Sequence
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from transformers import PreTrainedTokenizerBase
|
||||
|
||||
from vllm.entrypoints.mcp.tool_server import ToolServer
|
||||
from vllm.entrypoints.openai.engine.protocol import DeltaMessage
|
||||
from vllm.logger import init_logger
|
||||
from vllm.reasoning import ReasoningParser
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
|
||||
from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
no_func_reasoning_tag = {
|
||||
"type": "structural_tag",
|
||||
"format": {
|
||||
"type": "triggered_tags",
|
||||
"tags": [
|
||||
{
|
||||
"begin": "<|channel|>analysis<|message|>",
|
||||
"content": {"type": "any_text"},
|
||||
"end": "<|end|>",
|
||||
}
|
||||
],
|
||||
"triggers": ["<|channel|>analysis"],
|
||||
"stop_after_first": False,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def from_builtin_tool_to_tag(tool: str) -> list[dict]:
|
||||
tag = [
|
||||
{
|
||||
"begin": f"<|channel|>commentary to={tool}",
|
||||
"content": {"type": "any_text"},
|
||||
"end": "<|end|>",
|
||||
},
|
||||
{
|
||||
"begin": f"<|channel|>analysis to={tool}",
|
||||
"content": {"type": "any_text"},
|
||||
"end": "<|end|>",
|
||||
},
|
||||
]
|
||||
return tag
|
||||
|
||||
|
||||
def tag_with_builtin_funcs(no_func_reasoning_tag, builtin_tool_list: list[str]) -> dict:
|
||||
import copy
|
||||
|
||||
new_tag = copy.deepcopy(no_func_reasoning_tag)
|
||||
new_tag["format"]["triggers"].append("<|channel|>commentary to=")
|
||||
|
||||
for tool in builtin_tool_list:
|
||||
new_tag["format"]["tags"].extend(from_builtin_tool_to_tag(tool))
|
||||
return new_tag
|
||||
|
||||
|
||||
class GptOssReasoningParser(ReasoningParser):
|
||||
"""
|
||||
@@ -71,64 +23,14 @@ class GptOssReasoningParser(ReasoningParser):
|
||||
|
||||
def __init__(self, tokenizer: PreTrainedTokenizerBase, *args, **kwargs):
|
||||
super().__init__(tokenizer, *args, **kwargs)
|
||||
# The model can output some special tokens between "final" and "<|message|>"
|
||||
# So we need to look for both sequences to determine the end of reasoning.
|
||||
self.reasoning_end_token_ids_prefix = self.model_tokenizer.encode(
|
||||
"<|channel|>final"
|
||||
)
|
||||
self.reasoning_end_token_ids_suffix = self.model_tokenizer.encode("<|message|>")
|
||||
# We also need to check for the <|end|> token to avoid false positives from
|
||||
# previous messages in multi-turn conversations.
|
||||
self.eom_token_id = self.vocab["<|end|>"]
|
||||
self.reasoning_max_num_between_tokens = 20
|
||||
|
||||
def is_reasoning_end(self, input_ids: Sequence[int]) -> bool:
|
||||
end_token_ids_prefix = self.reasoning_end_token_ids_prefix
|
||||
end_token_ids_suffix = self.reasoning_end_token_ids_suffix
|
||||
assert len(end_token_ids_prefix) > 0, "reasoning_end_token_ids_prefix is empty"
|
||||
assert len(end_token_ids_suffix) > 0, "reasoning_end_token_ids_suffix is empty"
|
||||
# Check if the end sequence is present in the input_ids.
|
||||
# We search from the end of input_ids to find the last match.
|
||||
for i in range(len(input_ids) - len(end_token_ids_prefix), -1, -1):
|
||||
if input_ids[i] == self.eom_token_id:
|
||||
# We looped backwards far enough to find the end of a previous message,
|
||||
# which means we have searched the entirety of the current message
|
||||
# and can exit early without searching further back into prior
|
||||
# messages of the conversation.
|
||||
return False
|
||||
if input_ids[i : i + len(end_token_ids_prefix)] == end_token_ids_prefix:
|
||||
# We have found the prefix, now we look for the suffix after the prefix.
|
||||
suffix_start = i + len(end_token_ids_prefix)
|
||||
for j in range(
|
||||
suffix_start, len(input_ids) - len(end_token_ids_suffix) + 1
|
||||
):
|
||||
if j - suffix_start >= self.reasoning_max_num_between_tokens:
|
||||
break
|
||||
if (
|
||||
input_ids[j : j + len(end_token_ids_suffix)]
|
||||
== end_token_ids_suffix
|
||||
):
|
||||
return True
|
||||
return False
|
||||
return True
|
||||
|
||||
def is_reasoning_end_streaming(
|
||||
self, input_ids: Sequence[int], delta_ids: Iterable[int]
|
||||
) -> bool:
|
||||
# The pattern window covers the end-of-reasoning marker itself.
|
||||
# We add len(delta_ids) so that under speculative decoding (where
|
||||
# a single step can accept many tokens) the entire accepted chunk
|
||||
# is always inside the scan region.
|
||||
delta_ids = tuple(delta_ids)
|
||||
pattern_len = (
|
||||
len(self.reasoning_end_token_ids_prefix)
|
||||
+ self.reasoning_max_num_between_tokens
|
||||
+ len(self.reasoning_end_token_ids_suffix)
|
||||
)
|
||||
window = pattern_len + len(delta_ids)
|
||||
n = len(input_ids)
|
||||
if n <= window:
|
||||
return self.is_reasoning_end(input_ids)
|
||||
return self.is_reasoning_end(input_ids[n - window :])
|
||||
return True
|
||||
|
||||
def extract_content_ids(self, input_ids: list[int]) -> list[int]:
|
||||
raise NotImplementedError(
|
||||
@@ -159,33 +61,3 @@ class GptOssReasoningParser(ReasoningParser):
|
||||
"GptOssReasoningParser only provides boundary detection. "
|
||||
"Use HarmonyParser for output parsing."
|
||||
)
|
||||
|
||||
# This function prepares the structural tag to format reasoning output
|
||||
def prepare_structured_tag(
|
||||
self, original_tag: str | None, tool_server: ToolServer | None
|
||||
) -> str | None:
|
||||
if original_tag is None:
|
||||
if tool_server is None:
|
||||
return json.dumps(no_func_reasoning_tag)
|
||||
else:
|
||||
builtin_tool_list: list[str] = []
|
||||
if tool_server.has_tool("browser"):
|
||||
builtin_tool_list.append("browser")
|
||||
if tool_server.has_tool("python"):
|
||||
builtin_tool_list.append("python")
|
||||
if tool_server.has_tool("container"):
|
||||
builtin_tool_list.append("container")
|
||||
|
||||
if len(builtin_tool_list) > 0:
|
||||
logger.info("Builtin_tool_list: %s", builtin_tool_list)
|
||||
func_tag = json.dumps(
|
||||
tag_with_builtin_funcs(no_func_reasoning_tag, builtin_tool_list)
|
||||
)
|
||||
else:
|
||||
logger.info("Builtin_tool_list is empty")
|
||||
func_tag = json.dumps(no_func_reasoning_tag)
|
||||
|
||||
return func_tag
|
||||
else:
|
||||
# There is potential risk for appending the tag to the original tag
|
||||
return original_tag
|
||||
|
||||
@@ -201,6 +201,15 @@ class OnlineRenderer:
|
||||
)
|
||||
else:
|
||||
# For GPT-OSS.
|
||||
if self.parser is not None:
|
||||
# HarmonyParser doesn't need chat_template_kwargs
|
||||
# TODO: Unify adjust_request() call with non-harmony branch
|
||||
self.parser(
|
||||
self.renderer.get_tokenizer(),
|
||||
request.tools,
|
||||
model_config=self.model_config,
|
||||
).adjust_request(request=request)
|
||||
|
||||
should_include_tools = tool_dicts is not None
|
||||
conversation, engine_inputs = self._make_request_with_harmony(
|
||||
request, should_include_tools
|
||||
|
||||
@@ -22,6 +22,8 @@ class GptOssToolParser(ToolParser):
|
||||
capability declaration via HarmonyParser.tool_parser_cls.
|
||||
"""
|
||||
|
||||
structural_tag_model = "harmony"
|
||||
|
||||
def __init__(self, tokenizer: "TokenizerLike", tools: list[Tool] | None = None):
|
||||
super().__init__(tokenizer, tools)
|
||||
|
||||
|
||||
@@ -65,7 +65,6 @@ XGRAMMAR_BUILTIN_STRUCTURAL_TAG_MODELS = frozenset(
|
||||
"qwen_3_5",
|
||||
"qwen_3_coder",
|
||||
"qwen_3",
|
||||
"harmony",
|
||||
"deepseek_v3_2",
|
||||
"glm_4_7",
|
||||
"deepseek_v4",
|
||||
@@ -209,7 +208,7 @@ def _dump_allowed_tool_ref_for_xgrammar(tool_ref: AllowedToolRef) -> AllowedTool
|
||||
return tool_ref
|
||||
|
||||
|
||||
def _get_function_parameters(function) -> dict[str, Any] | bool:
|
||||
def get_function_parameters(function) -> dict[str, Any] | bool:
|
||||
if getattr(function, "strict", None) is False:
|
||||
return True
|
||||
return function.parameters if function.parameters is not None else True
|
||||
@@ -230,7 +229,7 @@ def _hermes_tool_tags(tools: list[FunctionToolParam]) -> list[TagFormat]:
|
||||
TagFormat(
|
||||
begin=begin + tool.function.name + arguments_field_prefix,
|
||||
content=JSONSchemaFormat(
|
||||
json_schema=_get_function_parameters(tool.function)
|
||||
json_schema=get_function_parameters(tool.function)
|
||||
),
|
||||
end=end,
|
||||
)
|
||||
@@ -279,7 +278,7 @@ def _minimax_tool_tags(tools: list[FunctionToolParam]) -> list[TagFormat]:
|
||||
TagFormat(
|
||||
begin=f'<invoke name="{tool.function.name}">\n',
|
||||
content=JSONSchemaFormat(
|
||||
json_schema=_get_function_parameters(tool.function),
|
||||
json_schema=get_function_parameters(tool.function),
|
||||
style="minimax_xml",
|
||||
),
|
||||
end="</invoke>\n",
|
||||
@@ -537,7 +536,7 @@ def _k3_arguments_block(parameters: dict[str, Any] | bool) -> Any:
|
||||
def _k3_call_tag(tool: FunctionToolParam) -> TagFormat:
|
||||
"""One ``call`` tag: ``<|open|>call tool="N" index="<digits>"<|sep|> args``."""
|
||||
function = tool.function
|
||||
parameters = _get_function_parameters(function)
|
||||
parameters = get_function_parameters(function)
|
||||
begin = f'{_K3_OPEN}call tool="{_k3_escape_attr(function.name)}" index="'
|
||||
return TagFormat(
|
||||
begin=begin,
|
||||
|
||||
Reference in New Issue
Block a user