[GPT-OSS] Structure_Tag support for gpt-oss tool-call in cot (#25515)
Signed-off-by: Hanchenli <lihanc2002@gmail.com> Signed-off-by: Hanchenli <61769611+Hanchenli@users.noreply.github.com> Signed-off-by: Wei Wei <wwei6@meta.com> Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com> Co-authored-by: Wei Wei <wwei6@meta.com> Co-authored-by: Wei Wei <weiweinpu@gmail.com> Co-authored-by: Cyrus Leung <tlleungac@connect.ust.hk>
This commit is contained in:
co-authored by
gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>
Wei Wei
Wei Wei
Cyrus Leung
parent
c312320764
commit
7c572544e4
@@ -35,6 +35,8 @@ class StructuredOutputsConfig:
|
||||
reasoning_parser: str = ""
|
||||
"""Select the reasoning parser depending on the model that you're using.
|
||||
This is used to parse the reasoning content into OpenAI API format."""
|
||||
enable_in_reasoning: bool = False
|
||||
"""Whether to use structured input for reasoning."""
|
||||
|
||||
def compute_hash(self) -> str:
|
||||
"""
|
||||
|
||||
@@ -479,6 +479,7 @@ class EngineArgs:
|
||||
VllmConfig, "structured_outputs_config"
|
||||
)
|
||||
reasoning_parser: str = StructuredOutputsConfig.reasoning_parser
|
||||
|
||||
# Deprecated guided decoding fields
|
||||
guided_decoding_backend: str | None = None
|
||||
guided_decoding_disable_fallback: bool | None = None
|
||||
|
||||
@@ -200,7 +200,7 @@ class JsonSchemaResponseFormat(OpenAIBaseModel):
|
||||
strict: bool | None = None
|
||||
|
||||
|
||||
class StructuralTag(OpenAIBaseModel):
|
||||
class LegacyStructuralTag(OpenAIBaseModel):
|
||||
begin: str
|
||||
# schema is the field, but that causes conflicts with pydantic so
|
||||
# instead use structural_tag_schema with an alias
|
||||
@@ -208,10 +208,20 @@ class StructuralTag(OpenAIBaseModel):
|
||||
end: str
|
||||
|
||||
|
||||
class LegacyStructuralTagResponseFormat(OpenAIBaseModel):
|
||||
type: Literal["structural_tag"]
|
||||
structures: list[LegacyStructuralTag]
|
||||
triggers: list[str]
|
||||
|
||||
|
||||
class StructuralTagResponseFormat(OpenAIBaseModel):
|
||||
type: Literal["structural_tag"]
|
||||
structures: list[StructuralTag]
|
||||
triggers: list[str]
|
||||
format: Any
|
||||
|
||||
|
||||
AnyStructuralTagResponseFormat: TypeAlias = (
|
||||
LegacyStructuralTagResponseFormat | StructuralTagResponseFormat
|
||||
)
|
||||
|
||||
|
||||
class ResponseFormat(OpenAIBaseModel):
|
||||
@@ -220,7 +230,9 @@ class ResponseFormat(OpenAIBaseModel):
|
||||
json_schema: JsonSchemaResponseFormat | None = None
|
||||
|
||||
|
||||
AnyResponseFormat: TypeAlias = ResponseFormat | StructuralTagResponseFormat
|
||||
AnyResponseFormat: TypeAlias = (
|
||||
ResponseFormat | StructuralTagResponseFormat | LegacyStructuralTagResponseFormat
|
||||
)
|
||||
|
||||
|
||||
class StreamOptions(OpenAIBaseModel):
|
||||
@@ -823,7 +835,11 @@ class ChatCompletionRequest(OpenAIBaseModel):
|
||||
elif response_format.type == "structural_tag":
|
||||
structural_tag = response_format
|
||||
assert structural_tag is not None and isinstance(
|
||||
structural_tag, StructuralTagResponseFormat
|
||||
structural_tag,
|
||||
(
|
||||
LegacyStructuralTagResponseFormat,
|
||||
StructuralTagResponseFormat,
|
||||
),
|
||||
)
|
||||
s_tag_obj = structural_tag.model_dump(by_alias=True)
|
||||
self.structured_outputs.structural_tag = json.dumps(s_tag_obj)
|
||||
|
||||
@@ -98,7 +98,7 @@ from vllm.logger import init_logger
|
||||
from vllm.logprobs import Logprob as SampleLogprob
|
||||
from vllm.logprobs import SampleLogprobs
|
||||
from vllm.outputs import CompletionOutput
|
||||
from vllm.sampling_params import SamplingParams
|
||||
from vllm.sampling_params import SamplingParams, StructuredOutputsParams
|
||||
from vllm.transformers_utils.tokenizer import AnyTokenizer
|
||||
from vllm.utils import random_uuid
|
||||
|
||||
@@ -365,6 +365,19 @@ class OpenAIServingResponses(OpenAIServing):
|
||||
context = HarmonyContext(messages, available_tools)
|
||||
else:
|
||||
context = SimpleContext()
|
||||
|
||||
if self.reasoning_parser is not None:
|
||||
reasoning_parser = self.reasoning_parser(tokenizer)
|
||||
if sampling_params.structured_outputs is None:
|
||||
sampling_params.structured_outputs = StructuredOutputsParams()
|
||||
struct_out = sampling_params.structured_outputs
|
||||
if struct_out.all_non_structural_tag_constraints_none():
|
||||
sampling_params.structured_outputs.structural_tag = (
|
||||
reasoning_parser.prepare_structured_tag(
|
||||
sampling_params.structured_outputs.structural_tag,
|
||||
self.tool_server,
|
||||
)
|
||||
)
|
||||
generator = self._generate_with_builtin_tools(
|
||||
request_id=request.request_id,
|
||||
request_prompt=request_prompts[i],
|
||||
@@ -1919,6 +1932,7 @@ class OpenAIServingResponses(OpenAIServing):
|
||||
processer = self._process_harmony_streaming_events
|
||||
else:
|
||||
processer = self._process_simple_streaming_events
|
||||
# TODO Hanchen make sampling params to include the structural tag
|
||||
|
||||
initial_response = ResponsesResponse.from_request(
|
||||
request,
|
||||
|
||||
@@ -7,6 +7,7 @@ from collections.abc import Callable, Sequence
|
||||
from functools import cached_property
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
from vllm.entrypoints.tool_server import ToolServer
|
||||
from vllm.logger import init_logger
|
||||
from vllm.utils.collections import is_list_of
|
||||
from vllm.utils.import_utils import import_from_path
|
||||
@@ -115,6 +116,17 @@ class ReasoningParser:
|
||||
previously been parsed and extracted (see constructor)
|
||||
"""
|
||||
|
||||
def prepare_structured_tag(
|
||||
self,
|
||||
original_tag: str | None,
|
||||
tool_server: ToolServer | None,
|
||||
) -> str:
|
||||
"""
|
||||
Instance method that is implemented for preparing the structured tag
|
||||
Otherwise, None is returned
|
||||
"""
|
||||
return None
|
||||
|
||||
|
||||
class ReasoningParserManager:
|
||||
reasoning_parsers: dict[str, type] = {}
|
||||
|
||||
@@ -1,17 +1,61 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import json
|
||||
from collections.abc import Sequence
|
||||
|
||||
from transformers import PreTrainedTokenizerBase
|
||||
|
||||
from vllm.entrypoints.harmony_utils import parse_chat_output
|
||||
from vllm.entrypoints.openai.protocol import ChatCompletionRequest, DeltaMessage
|
||||
from vllm.entrypoints.tool_server import ToolServer
|
||||
from vllm.logger import init_logger
|
||||
from vllm.reasoning import ReasoningParser, ReasoningParserManager
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
no_func_reaonsing_tag = {
|
||||
"type": "structural_tag",
|
||||
"format": {
|
||||
"type": "triggered_tags",
|
||||
"tags": [
|
||||
{
|
||||
"begin": "<|channel|>analysis<|message|>",
|
||||
"content": {"type": "any_text"},
|
||||
"end": "<|end|>",
|
||||
}
|
||||
],
|
||||
"triggers": ["<|channel|>analysis"],
|
||||
"stop_after_first": False,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def from_builtin_tool_to_tag(tool: str) -> list[dict]:
|
||||
tag = [
|
||||
{
|
||||
"begin": f"<|channel|>commentary to={tool}",
|
||||
"content": {"type": "any_text"},
|
||||
"end": "<|end|>",
|
||||
},
|
||||
{
|
||||
"begin": f"<|channel|>analysis to={tool}",
|
||||
"content": {"type": "any_text"},
|
||||
"end": "<|end|>",
|
||||
},
|
||||
]
|
||||
return tag
|
||||
|
||||
|
||||
def tag_with_builtin_funcs(no_func_reaonsing_tag, builtin_tool_list: list[str]) -> dict:
|
||||
import copy
|
||||
|
||||
new_tag = copy.deepcopy(no_func_reaonsing_tag)
|
||||
new_tag["format"]["triggers"].append("<|channel|>commentary to=")
|
||||
|
||||
for tool in builtin_tool_list:
|
||||
new_tag["format"]["tags"].extend(from_builtin_tool_to_tag(tool))
|
||||
return new_tag
|
||||
|
||||
|
||||
@ReasoningParserManager.register_module("openai_gptoss")
|
||||
class GptOssReasoningParser(ReasoningParser):
|
||||
@@ -81,3 +125,33 @@ class GptOssReasoningParser(ReasoningParser):
|
||||
raise NotImplementedError(
|
||||
"gpt-oss has a special branch for parsing reasoning in non-streaming mode. This method shouldn't be used." # noqa: E501
|
||||
)
|
||||
|
||||
# This function prepares the structural tag to format reasoning output
|
||||
def prepare_structured_tag(
|
||||
self, original_tag: str | None, tool_server: ToolServer | None
|
||||
) -> str:
|
||||
if original_tag is None:
|
||||
if tool_server is None:
|
||||
return json.dumps(no_func_reaonsing_tag)
|
||||
else:
|
||||
builtin_tool_list: list[str] = []
|
||||
if tool_server.has_tool("browser"):
|
||||
builtin_tool_list.append("browser")
|
||||
if tool_server.has_tool("python"):
|
||||
builtin_tool_list.append("python")
|
||||
if tool_server.has_tool("container"):
|
||||
builtin_tool_list.append("container")
|
||||
|
||||
if len(builtin_tool_list) > 0:
|
||||
logger.info("Builtin_tool_list: %s", builtin_tool_list)
|
||||
func_tag = json.dumps(
|
||||
tag_with_builtin_funcs(no_func_reaonsing_tag, builtin_tool_list)
|
||||
)
|
||||
else:
|
||||
logger.info("Builtin_tool_list is empty")
|
||||
func_tag = json.dumps(no_func_reaonsing_tag)
|
||||
|
||||
return func_tag
|
||||
else:
|
||||
# There is potential risk for appending the tag to the original tag
|
||||
return original_tag
|
||||
|
||||
@@ -58,6 +58,7 @@ class StructuredOutputsParams:
|
||||
self.choice is not None,
|
||||
self.grammar is not None,
|
||||
self.json_object is not None,
|
||||
self.structural_tag is not None,
|
||||
]
|
||||
)
|
||||
if count > 1:
|
||||
@@ -66,6 +67,37 @@ class StructuredOutputsParams:
|
||||
f"but multiple are specified: {self.__dict__}"
|
||||
)
|
||||
|
||||
def all_constraints_none(self) -> bool:
|
||||
"""
|
||||
Returns True if all structured-output constraint fields are None.
|
||||
"""
|
||||
return all(
|
||||
getattr(self, field) is None
|
||||
for field in (
|
||||
"json",
|
||||
"regex",
|
||||
"choice",
|
||||
"grammar",
|
||||
"json_object",
|
||||
"structural_tag",
|
||||
)
|
||||
)
|
||||
|
||||
def all_non_structural_tag_constraints_none(self) -> bool:
|
||||
"""
|
||||
Returns True if all structured-output constraint fields are None.
|
||||
"""
|
||||
return all(
|
||||
getattr(self, field) is None
|
||||
for field in (
|
||||
"json",
|
||||
"regex",
|
||||
"choice",
|
||||
"grammar",
|
||||
"json_object",
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
@dataclass
|
||||
class GuidedDecodingParams(StructuredOutputsParams):
|
||||
|
||||
@@ -73,6 +73,10 @@ class StructuredOutputManager:
|
||||
)
|
||||
self.reasoner = reasoner_cls(tokenizer=self.tokenizer)
|
||||
|
||||
self.enable_in_reasoning = (
|
||||
self.vllm_config.structured_outputs_config.enable_in_reasoning
|
||||
)
|
||||
|
||||
def grammar_init(self, request: Request) -> None:
|
||||
if request.structured_output_request is None:
|
||||
return
|
||||
@@ -274,7 +278,13 @@ class StructuredOutputManager:
|
||||
return bitmask_tensor.numpy()
|
||||
|
||||
def should_fill_bitmask(self, request: Request) -> bool:
|
||||
# NOTE (Hanchen) if enable_in_reasoning is True, it means that
|
||||
# the model needs to be constrained in reasoning. So we should always
|
||||
# enable the bitmask filling.
|
||||
|
||||
if self.reasoner is not None:
|
||||
if self.enable_in_reasoning:
|
||||
return True
|
||||
assert request.structured_output_request is not None
|
||||
if request.structured_output_request.reasoning_ended is None:
|
||||
request.structured_output_request.reasoning_ended = (
|
||||
@@ -297,6 +307,10 @@ class StructuredOutputManager:
|
||||
if self.reasoner is None:
|
||||
return True
|
||||
|
||||
# if the model needs structured in reasoning, we should advance
|
||||
if self.enable_in_reasoning:
|
||||
return True
|
||||
|
||||
structured_req = request.structured_output_request
|
||||
if structured_req.reasoning_ended:
|
||||
return True
|
||||
|
||||
@@ -91,18 +91,20 @@ class XgrammarBackend(StructuredOutputBackend):
|
||||
ctx = self.compiler.compile_regex(grammar_spec)
|
||||
elif request_type == StructuredOutputOptions.STRUCTURAL_TAG:
|
||||
s_tag = json.loads(grammar_spec)
|
||||
tags = [
|
||||
xgr.StructuralTagItem(
|
||||
begin=s["begin"],
|
||||
schema=json.dumps(s["schema"]),
|
||||
end=s["end"],
|
||||
)
|
||||
for s in s_tag["structures"]
|
||||
]
|
||||
structural_tag = xgr.StructuralTag.from_legacy_structural_tag(
|
||||
tags, s_tag["triggers"]
|
||||
)
|
||||
ctx = self.compiler.compile_structural_tag(structural_tag)
|
||||
if "structures" in s_tag:
|
||||
# Falling back to deprecated method of compiling structural tag
|
||||
tags = [
|
||||
xgr.StructuralTagItem(
|
||||
begin=s["begin"],
|
||||
schema=json.dumps(s["schema"]),
|
||||
end=s["end"],
|
||||
)
|
||||
for s in s_tag["structures"]
|
||||
]
|
||||
ctx = self.compiler.compile_structural_tag(tags, s_tag["triggers"])
|
||||
else:
|
||||
logger.info("Compiling structural tag grammar_spec: %s", grammar_spec)
|
||||
ctx = self.compiler.compile_structural_tag(grammar_spec)
|
||||
else:
|
||||
logger.error(
|
||||
"Validation should have already occurred. Please file an issue."
|
||||
@@ -320,17 +322,19 @@ def validate_xgrammar_grammar(sampling_params: SamplingParams) -> None:
|
||||
if so_params.structural_tag:
|
||||
try:
|
||||
s_tag = json.loads(so_params.structural_tag)
|
||||
tags = [
|
||||
xgr.StructuralTagItem(
|
||||
begin=s["begin"],
|
||||
schema=json.dumps(s["schema"]),
|
||||
end=s["end"],
|
||||
)
|
||||
for s in s_tag["structures"]
|
||||
]
|
||||
structural_tag = xgr.StructuralTag.from_legacy_structural_tag(
|
||||
tags, s_tag["triggers"]
|
||||
)
|
||||
xgr.Grammar.from_structural_tag(structural_tag)
|
||||
|
||||
# Using the deprecated method of compiling structural tag
|
||||
if "structures" in s_tag:
|
||||
tags = [
|
||||
xgr.StructuralTagItem(
|
||||
begin=s["begin"],
|
||||
schema=json.dumps(s["schema"]),
|
||||
end=s["end"],
|
||||
)
|
||||
for s in s_tag["structures"]
|
||||
]
|
||||
xgr.Grammar.from_structural_tag(tags, s_tag["triggers"])
|
||||
else:
|
||||
xgr.Grammar.from_structural_tag(so_params.structural_tag)
|
||||
except Exception as e:
|
||||
raise ValueError("Invalid structural tag specification.") from e
|
||||
|
||||
@@ -28,7 +28,12 @@ class StructuredOutputRequest:
|
||||
if sampling_params is None:
|
||||
return None
|
||||
params = sampling_params.structured_outputs
|
||||
return StructuredOutputRequest(params=params) if params else None
|
||||
if params:
|
||||
if params.all_constraints_none():
|
||||
return None
|
||||
else:
|
||||
return StructuredOutputRequest(params=params)
|
||||
return None
|
||||
|
||||
def _check_grammar_completion(self) -> bool:
|
||||
# NOTE: We have to lazy import to gate circular imports
|
||||
|
||||
Reference in New Issue
Block a user