[GPT-OSS] Structure_Tag support for gpt-oss tool-call in cot (#25515)

Signed-off-by: Hanchenli <lihanc2002@gmail.com>
Signed-off-by: Hanchenli <61769611+Hanchenli@users.noreply.github.com>
Signed-off-by: Wei Wei <wwei6@meta.com>
Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>
Co-authored-by: Wei Wei <wwei6@meta.com>
Co-authored-by: Wei Wei <weiweinpu@gmail.com>
Co-authored-by: Cyrus Leung <tlleungac@connect.ust.hk>
This commit is contained in:
Hanchenli
2025-10-17 21:55:54 -07:00
committed by GitHub
co-authored by gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com> Wei Wei Wei Wei Cyrus Leung
parent c312320764
commit 7c572544e4
14 changed files with 911 additions and 32 deletions
+2
View File
@@ -35,6 +35,8 @@ class StructuredOutputsConfig:
reasoning_parser: str = ""
"""Select the reasoning parser depending on the model that you're using.
This is used to parse the reasoning content into OpenAI API format."""
enable_in_reasoning: bool = False
"""Whether to use structured input for reasoning."""
def compute_hash(self) -> str:
"""
+1
View File
@@ -479,6 +479,7 @@ class EngineArgs:
VllmConfig, "structured_outputs_config"
)
reasoning_parser: str = StructuredOutputsConfig.reasoning_parser
# Deprecated guided decoding fields
guided_decoding_backend: str | None = None
guided_decoding_disable_fallback: bool | None = None
+21 -5
View File
@@ -200,7 +200,7 @@ class JsonSchemaResponseFormat(OpenAIBaseModel):
strict: bool | None = None
class StructuralTag(OpenAIBaseModel):
class LegacyStructuralTag(OpenAIBaseModel):
begin: str
# schema is the field, but that causes conflicts with pydantic so
# instead use structural_tag_schema with an alias
@@ -208,10 +208,20 @@ class StructuralTag(OpenAIBaseModel):
end: str
class LegacyStructuralTagResponseFormat(OpenAIBaseModel):
type: Literal["structural_tag"]
structures: list[LegacyStructuralTag]
triggers: list[str]
class StructuralTagResponseFormat(OpenAIBaseModel):
type: Literal["structural_tag"]
structures: list[StructuralTag]
triggers: list[str]
format: Any
AnyStructuralTagResponseFormat: TypeAlias = (
LegacyStructuralTagResponseFormat | StructuralTagResponseFormat
)
class ResponseFormat(OpenAIBaseModel):
@@ -220,7 +230,9 @@ class ResponseFormat(OpenAIBaseModel):
json_schema: JsonSchemaResponseFormat | None = None
AnyResponseFormat: TypeAlias = ResponseFormat | StructuralTagResponseFormat
AnyResponseFormat: TypeAlias = (
ResponseFormat | StructuralTagResponseFormat | LegacyStructuralTagResponseFormat
)
class StreamOptions(OpenAIBaseModel):
@@ -823,7 +835,11 @@ class ChatCompletionRequest(OpenAIBaseModel):
elif response_format.type == "structural_tag":
structural_tag = response_format
assert structural_tag is not None and isinstance(
structural_tag, StructuralTagResponseFormat
structural_tag,
(
LegacyStructuralTagResponseFormat,
StructuralTagResponseFormat,
),
)
s_tag_obj = structural_tag.model_dump(by_alias=True)
self.structured_outputs.structural_tag = json.dumps(s_tag_obj)
+15 -1
View File
@@ -98,7 +98,7 @@ from vllm.logger import init_logger
from vllm.logprobs import Logprob as SampleLogprob
from vllm.logprobs import SampleLogprobs
from vllm.outputs import CompletionOutput
from vllm.sampling_params import SamplingParams
from vllm.sampling_params import SamplingParams, StructuredOutputsParams
from vllm.transformers_utils.tokenizer import AnyTokenizer
from vllm.utils import random_uuid
@@ -365,6 +365,19 @@ class OpenAIServingResponses(OpenAIServing):
context = HarmonyContext(messages, available_tools)
else:
context = SimpleContext()
if self.reasoning_parser is not None:
reasoning_parser = self.reasoning_parser(tokenizer)
if sampling_params.structured_outputs is None:
sampling_params.structured_outputs = StructuredOutputsParams()
struct_out = sampling_params.structured_outputs
if struct_out.all_non_structural_tag_constraints_none():
sampling_params.structured_outputs.structural_tag = (
reasoning_parser.prepare_structured_tag(
sampling_params.structured_outputs.structural_tag,
self.tool_server,
)
)
generator = self._generate_with_builtin_tools(
request_id=request.request_id,
request_prompt=request_prompts[i],
@@ -1919,6 +1932,7 @@ class OpenAIServingResponses(OpenAIServing):
processer = self._process_harmony_streaming_events
else:
processer = self._process_simple_streaming_events
# TODO Hanchen make sampling params to include the structural tag
initial_response = ResponsesResponse.from_request(
request,
+12
View File
@@ -7,6 +7,7 @@ from collections.abc import Callable, Sequence
from functools import cached_property
from typing import TYPE_CHECKING, Any
from vllm.entrypoints.tool_server import ToolServer
from vllm.logger import init_logger
from vllm.utils.collections import is_list_of
from vllm.utils.import_utils import import_from_path
@@ -115,6 +116,17 @@ class ReasoningParser:
previously been parsed and extracted (see constructor)
"""
def prepare_structured_tag(
self,
original_tag: str | None,
tool_server: ToolServer | None,
) -> str:
"""
Instance method that is implemented for preparing the structured tag
Otherwise, None is returned
"""
return None
class ReasoningParserManager:
reasoning_parsers: dict[str, type] = {}
+75 -1
View File
@@ -1,17 +1,61 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import json
from collections.abc import Sequence
from transformers import PreTrainedTokenizerBase
from vllm.entrypoints.harmony_utils import parse_chat_output
from vllm.entrypoints.openai.protocol import ChatCompletionRequest, DeltaMessage
from vllm.entrypoints.tool_server import ToolServer
from vllm.logger import init_logger
from vllm.reasoning import ReasoningParser, ReasoningParserManager
logger = init_logger(__name__)
no_func_reaonsing_tag = {
"type": "structural_tag",
"format": {
"type": "triggered_tags",
"tags": [
{
"begin": "<|channel|>analysis<|message|>",
"content": {"type": "any_text"},
"end": "<|end|>",
}
],
"triggers": ["<|channel|>analysis"],
"stop_after_first": False,
},
}
def from_builtin_tool_to_tag(tool: str) -> list[dict]:
tag = [
{
"begin": f"<|channel|>commentary to={tool}",
"content": {"type": "any_text"},
"end": "<|end|>",
},
{
"begin": f"<|channel|>analysis to={tool}",
"content": {"type": "any_text"},
"end": "<|end|>",
},
]
return tag
def tag_with_builtin_funcs(no_func_reaonsing_tag, builtin_tool_list: list[str]) -> dict:
import copy
new_tag = copy.deepcopy(no_func_reaonsing_tag)
new_tag["format"]["triggers"].append("<|channel|>commentary to=")
for tool in builtin_tool_list:
new_tag["format"]["tags"].extend(from_builtin_tool_to_tag(tool))
return new_tag
@ReasoningParserManager.register_module("openai_gptoss")
class GptOssReasoningParser(ReasoningParser):
@@ -81,3 +125,33 @@ class GptOssReasoningParser(ReasoningParser):
raise NotImplementedError(
"gpt-oss has a special branch for parsing reasoning in non-streaming mode. This method shouldn't be used." # noqa: E501
)
# This function prepares the structural tag to format reasoning output
def prepare_structured_tag(
self, original_tag: str | None, tool_server: ToolServer | None
) -> str:
if original_tag is None:
if tool_server is None:
return json.dumps(no_func_reaonsing_tag)
else:
builtin_tool_list: list[str] = []
if tool_server.has_tool("browser"):
builtin_tool_list.append("browser")
if tool_server.has_tool("python"):
builtin_tool_list.append("python")
if tool_server.has_tool("container"):
builtin_tool_list.append("container")
if len(builtin_tool_list) > 0:
logger.info("Builtin_tool_list: %s", builtin_tool_list)
func_tag = json.dumps(
tag_with_builtin_funcs(no_func_reaonsing_tag, builtin_tool_list)
)
else:
logger.info("Builtin_tool_list is empty")
func_tag = json.dumps(no_func_reaonsing_tag)
return func_tag
else:
# There is potential risk for appending the tag to the original tag
return original_tag
+32
View File
@@ -58,6 +58,7 @@ class StructuredOutputsParams:
self.choice is not None,
self.grammar is not None,
self.json_object is not None,
self.structural_tag is not None,
]
)
if count > 1:
@@ -66,6 +67,37 @@ class StructuredOutputsParams:
f"but multiple are specified: {self.__dict__}"
)
def all_constraints_none(self) -> bool:
"""
Returns True if all structured-output constraint fields are None.
"""
return all(
getattr(self, field) is None
for field in (
"json",
"regex",
"choice",
"grammar",
"json_object",
"structural_tag",
)
)
def all_non_structural_tag_constraints_none(self) -> bool:
"""
Returns True if all structured-output constraint fields are None.
"""
return all(
getattr(self, field) is None
for field in (
"json",
"regex",
"choice",
"grammar",
"json_object",
)
)
@dataclass
class GuidedDecodingParams(StructuredOutputsParams):
+14
View File
@@ -73,6 +73,10 @@ class StructuredOutputManager:
)
self.reasoner = reasoner_cls(tokenizer=self.tokenizer)
self.enable_in_reasoning = (
self.vllm_config.structured_outputs_config.enable_in_reasoning
)
def grammar_init(self, request: Request) -> None:
if request.structured_output_request is None:
return
@@ -274,7 +278,13 @@ class StructuredOutputManager:
return bitmask_tensor.numpy()
def should_fill_bitmask(self, request: Request) -> bool:
# NOTE (Hanchen) if enable_in_reasoning is True, it means that
# the model needs to be constrained in reasoning. So we should always
# enable the bitmask filling.
if self.reasoner is not None:
if self.enable_in_reasoning:
return True
assert request.structured_output_request is not None
if request.structured_output_request.reasoning_ended is None:
request.structured_output_request.reasoning_ended = (
@@ -297,6 +307,10 @@ class StructuredOutputManager:
if self.reasoner is None:
return True
# if the model needs structured in reasoning, we should advance
if self.enable_in_reasoning:
return True
structured_req = request.structured_output_request
if structured_req.reasoning_ended:
return True
+28 -24
View File
@@ -91,18 +91,20 @@ class XgrammarBackend(StructuredOutputBackend):
ctx = self.compiler.compile_regex(grammar_spec)
elif request_type == StructuredOutputOptions.STRUCTURAL_TAG:
s_tag = json.loads(grammar_spec)
tags = [
xgr.StructuralTagItem(
begin=s["begin"],
schema=json.dumps(s["schema"]),
end=s["end"],
)
for s in s_tag["structures"]
]
structural_tag = xgr.StructuralTag.from_legacy_structural_tag(
tags, s_tag["triggers"]
)
ctx = self.compiler.compile_structural_tag(structural_tag)
if "structures" in s_tag:
# Falling back to deprecated method of compiling structural tag
tags = [
xgr.StructuralTagItem(
begin=s["begin"],
schema=json.dumps(s["schema"]),
end=s["end"],
)
for s in s_tag["structures"]
]
ctx = self.compiler.compile_structural_tag(tags, s_tag["triggers"])
else:
logger.info("Compiling structural tag grammar_spec: %s", grammar_spec)
ctx = self.compiler.compile_structural_tag(grammar_spec)
else:
logger.error(
"Validation should have already occurred. Please file an issue."
@@ -320,17 +322,19 @@ def validate_xgrammar_grammar(sampling_params: SamplingParams) -> None:
if so_params.structural_tag:
try:
s_tag = json.loads(so_params.structural_tag)
tags = [
xgr.StructuralTagItem(
begin=s["begin"],
schema=json.dumps(s["schema"]),
end=s["end"],
)
for s in s_tag["structures"]
]
structural_tag = xgr.StructuralTag.from_legacy_structural_tag(
tags, s_tag["triggers"]
)
xgr.Grammar.from_structural_tag(structural_tag)
# Using the deprecated method of compiling structural tag
if "structures" in s_tag:
tags = [
xgr.StructuralTagItem(
begin=s["begin"],
schema=json.dumps(s["schema"]),
end=s["end"],
)
for s in s_tag["structures"]
]
xgr.Grammar.from_structural_tag(tags, s_tag["triggers"])
else:
xgr.Grammar.from_structural_tag(so_params.structural_tag)
except Exception as e:
raise ValueError("Invalid structural tag specification.") from e
+6 -1
View File
@@ -28,7 +28,12 @@ class StructuredOutputRequest:
if sampling_params is None:
return None
params = sampling_params.structured_outputs
return StructuredOutputRequest(params=params) if params else None
if params:
if params.all_constraints_none():
return None
else:
return StructuredOutputRequest(params=params)
return None
def _check_grammar_completion(self) -> bool:
# NOTE: We have to lazy import to gate circular imports