[Refactor] Chat Completions Harmony Refactor, non-streaming path. (#45171)
Signed-off-by: Yifan Zong <yzong@redhat.com>
This commit is contained in:
@@ -38,12 +38,12 @@ from vllm.entrypoints.serve.render.serving import OpenAIServingRender
|
|||||||
from vllm.exceptions import VLLMValidationError
|
from vllm.exceptions import VLLMValidationError
|
||||||
from vllm.inputs import TokensPrompt
|
from vllm.inputs import TokensPrompt
|
||||||
from vllm.outputs import CompletionOutput, RequestOutput
|
from vllm.outputs import CompletionOutput, RequestOutput
|
||||||
|
from vllm.parser import HarmonyParser
|
||||||
from vllm.renderers.hf import HfRenderer
|
from vllm.renderers.hf import HfRenderer
|
||||||
from vllm.renderers.mistral import MistralRenderer
|
from vllm.renderers.mistral import MistralRenderer
|
||||||
from vllm.tokenizers import get_tokenizer
|
from vllm.tokenizers import get_tokenizer
|
||||||
from vllm.tokenizers.mistral import MistralTokenizer
|
from vllm.tokenizers.mistral import MistralTokenizer
|
||||||
from vllm.tokenizers.registry import cached_tokenizer_from_config
|
from vllm.tokenizers.registry import cached_tokenizer_from_config
|
||||||
from vllm.tool_parsers import ToolParserManager
|
|
||||||
from vllm.v1.engine.async_llm import AsyncLLM
|
from vllm.v1.engine.async_llm import AsyncLLM
|
||||||
|
|
||||||
GPT_OSS_MODEL_NAME = "openai/gpt-oss-20b"
|
GPT_OSS_MODEL_NAME = "openai/gpt-oss-20b"
|
||||||
@@ -575,7 +575,13 @@ def _build_serving_render(
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
def _build_serving_chat(engine: AsyncLLM) -> OpenAIServingChat:
|
def _build_serving_chat(
|
||||||
|
engine: AsyncLLM,
|
||||||
|
*,
|
||||||
|
reasoning_parser: str = "",
|
||||||
|
tool_parser: str | None = None,
|
||||||
|
enable_auto_tools: bool = False,
|
||||||
|
) -> OpenAIServingChat:
|
||||||
models = OpenAIServingModels(
|
models = OpenAIServingModels(
|
||||||
engine_client=engine,
|
engine_client=engine,
|
||||||
base_model_paths=BASE_MODEL_PATHS,
|
base_model_paths=BASE_MODEL_PATHS,
|
||||||
@@ -590,6 +596,9 @@ def _build_serving_chat(engine: AsyncLLM) -> OpenAIServingChat:
|
|||||||
chat_template=CHAT_TEMPLATE,
|
chat_template=CHAT_TEMPLATE,
|
||||||
chat_template_content_format="auto",
|
chat_template_content_format="auto",
|
||||||
request_logger=None,
|
request_logger=None,
|
||||||
|
reasoning_parser=reasoning_parser,
|
||||||
|
tool_parser=tool_parser,
|
||||||
|
enable_auto_tools=enable_auto_tools,
|
||||||
)
|
)
|
||||||
|
|
||||||
return serving_chat
|
return serving_chat
|
||||||
@@ -637,7 +646,7 @@ async def test_serving_chat_returns_correct_model_name():
|
|||||||
serving_chat = _build_serving_chat(mock_engine)
|
serving_chat = _build_serving_chat(mock_engine)
|
||||||
messages = [{"role": "user", "content": "what is 1+1?"}]
|
messages = [{"role": "user", "content": "what is 1+1?"}]
|
||||||
|
|
||||||
async def return_model_name(*args):
|
async def return_model_name(*args, **kwargs):
|
||||||
return args[3]
|
return args[3]
|
||||||
|
|
||||||
serving_chat.chat_completion_full_generator = return_model_name
|
serving_chat.chat_completion_full_generator = return_model_name
|
||||||
@@ -1210,15 +1219,21 @@ class TestServingChatWithHarmony:
|
|||||||
mock_engine = MagicMock(spec=AsyncLLM)
|
mock_engine = MagicMock(spec=AsyncLLM)
|
||||||
mock_engine.errored = False
|
mock_engine.errored = False
|
||||||
mock_engine.model_config = MockModelConfig()
|
mock_engine.model_config = MockModelConfig()
|
||||||
|
mock_engine.model_config.hf_config = MockHFConfig(model_type="gpt_oss")
|
||||||
|
mock_engine.model_config.hf_text_config = MockHFConfig(model_type="gpt_oss")
|
||||||
mock_engine.input_processor = MagicMock()
|
mock_engine.input_processor = MagicMock()
|
||||||
mock_engine.renderer = _build_renderer(mock_engine.model_config)
|
mock_engine.renderer = _build_renderer(mock_engine.model_config)
|
||||||
return mock_engine
|
return mock_engine
|
||||||
|
|
||||||
@pytest.fixture()
|
@pytest.fixture()
|
||||||
def serving_chat(self, mock_engine) -> OpenAIServingChat:
|
def serving_chat(self, mock_engine) -> OpenAIServingChat:
|
||||||
chat = _build_serving_chat(mock_engine)
|
chat = _build_serving_chat(
|
||||||
chat.use_harmony = True
|
mock_engine,
|
||||||
chat.tool_parser = ToolParserManager.get_tool_parser("openai")
|
reasoning_parser="openai_gptoss",
|
||||||
|
tool_parser="openai",
|
||||||
|
enable_auto_tools=True,
|
||||||
|
)
|
||||||
|
assert chat.parser_cls is HarmonyParser
|
||||||
return chat
|
return chat
|
||||||
|
|
||||||
def mock_request_output_from_req_and_token_ids(
|
def mock_request_output_from_req_and_token_ids(
|
||||||
@@ -1277,6 +1292,7 @@ class TestServingChatWithHarmony:
|
|||||||
stream: bool = False,
|
stream: bool = False,
|
||||||
) -> ChatCompletionResponse:
|
) -> ChatCompletionResponse:
|
||||||
harmony_token_ids = get_encoding().encode(harmony_str, allowed_special="all")
|
harmony_token_ids = get_encoding().encode(harmony_str, allowed_special="all")
|
||||||
|
tokenizer = get_tokenizer(GPT_OSS_MODEL_NAME)
|
||||||
|
|
||||||
async def result_generator():
|
async def result_generator():
|
||||||
if stream:
|
if stream:
|
||||||
@@ -1304,11 +1320,12 @@ class TestServingChatWithHarmony:
|
|||||||
request_id=req.request_id,
|
request_id=req.request_id,
|
||||||
model_name=req.model,
|
model_name=req.model,
|
||||||
conversation=[],
|
conversation=[],
|
||||||
tokenizer=get_tokenizer(req.model),
|
tokenizer=tokenizer,
|
||||||
request_metadata=RequestResponseMetadata(
|
request_metadata=RequestResponseMetadata(
|
||||||
request_id=req.request_id,
|
request_id=req.request_id,
|
||||||
model_name=req.model,
|
model_name=req.model,
|
||||||
),
|
),
|
||||||
|
chat_template_kwargs=serving_chat._effective_chat_template_kwargs(req),
|
||||||
)
|
)
|
||||||
|
|
||||||
if stream:
|
if stream:
|
||||||
@@ -1316,11 +1333,18 @@ class TestServingChatWithHarmony:
|
|||||||
return await result
|
return await result
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
@pytest.mark.asyncio
|
||||||
async def test_simple_chat(self, serving_chat, stream):
|
@pytest.mark.parametrize(
|
||||||
|
"include_reasoning", [True, False], ids=["with_reasoning", "no_reasoning"]
|
||||||
|
)
|
||||||
|
async def test_simple_chat(self, serving_chat, stream, include_reasoning):
|
||||||
messages = [{"role": "user", "content": "what is 1+1?"}]
|
messages = [{"role": "user", "content": "what is 1+1?"}]
|
||||||
|
|
||||||
# Test the Harmony messages for the first turn's input
|
# Test the Harmony messages for the first turn's input
|
||||||
req = ChatCompletionRequest(model=MODEL_NAME, messages=messages)
|
req = ChatCompletionRequest(
|
||||||
|
model=MODEL_NAME,
|
||||||
|
messages=messages,
|
||||||
|
include_reasoning=include_reasoning,
|
||||||
|
)
|
||||||
input_messages, _ = (
|
input_messages, _ = (
|
||||||
serving_chat.openai_serving_render._make_request_with_harmony(req)
|
serving_chat.openai_serving_render._make_request_with_harmony(req)
|
||||||
)
|
)
|
||||||
@@ -1342,7 +1366,11 @@ class TestServingChatWithHarmony:
|
|||||||
response = await self.generate_response_from_harmony_str(
|
response = await self.generate_response_from_harmony_str(
|
||||||
serving_chat, req, response_str, stream=stream
|
serving_chat, req, response_str, stream=stream
|
||||||
)
|
)
|
||||||
verify_chat_response(response, content=final_str, reasoning=reasoning_str)
|
verify_chat_response(
|
||||||
|
response,
|
||||||
|
content=final_str,
|
||||||
|
reasoning=reasoning_str if include_reasoning else None,
|
||||||
|
)
|
||||||
|
|
||||||
# Add the output messages from the first turn as input to the second turn
|
# Add the output messages from the first turn as input to the second turn
|
||||||
for choice in response.choices:
|
for choice in response.choices:
|
||||||
|
|||||||
@@ -11,12 +11,10 @@ from vllm.entrypoints.openai.parser.harmony_utils import (
|
|||||||
auto_drop_analysis_messages,
|
auto_drop_analysis_messages,
|
||||||
create_tool_definition,
|
create_tool_definition,
|
||||||
extract_function_from_recipient,
|
extract_function_from_recipient,
|
||||||
get_encoding,
|
|
||||||
get_system_message,
|
get_system_message,
|
||||||
has_custom_tools,
|
has_custom_tools,
|
||||||
is_function_recipient,
|
is_function_recipient,
|
||||||
parse_chat_input_to_harmony_message,
|
parse_chat_input_to_harmony_message,
|
||||||
parse_chat_output,
|
|
||||||
)
|
)
|
||||||
from vllm.entrypoints.openai.responses.harmony import (
|
from vllm.entrypoints.openai.responses.harmony import (
|
||||||
response_input_to_harmony,
|
response_input_to_harmony,
|
||||||
@@ -941,110 +939,6 @@ class TestAutoDropAnalysisMessages:
|
|||||||
assert cleaned_messages == messages[1:]
|
assert cleaned_messages == messages[1:]
|
||||||
|
|
||||||
|
|
||||||
class TestParseChatOutput:
|
|
||||||
def test_parse_chat_output_interrupted_first_message(self) -> None:
|
|
||||||
harmony_str = "<|channel|>final<|message|>I'm in the middle of answering"
|
|
||||||
token_ids = get_encoding().encode(harmony_str, allowed_special="all")
|
|
||||||
reasoning, final_content, _ = parse_chat_output(token_ids)
|
|
||||||
assert reasoning is None
|
|
||||||
assert final_content == "I'm in the middle of answering"
|
|
||||||
|
|
||||||
def test_parse_chat_output_interrupted_reasoning_first_message(self) -> None:
|
|
||||||
harmony_str = "<|channel|>analysis<|message|>I'm in the middle of thinking"
|
|
||||||
token_ids = get_encoding().encode(harmony_str, allowed_special="all")
|
|
||||||
reasoning, final_content, _ = parse_chat_output(token_ids)
|
|
||||||
assert reasoning == "I'm in the middle of thinking"
|
|
||||||
assert final_content is None
|
|
||||||
|
|
||||||
def test_parse_chat_output_complete_reasoning_interrupted_content(self) -> None:
|
|
||||||
harmony_str = (
|
|
||||||
"<|channel|>analysis<|message|>I'm thinking.<|end|>"
|
|
||||||
"<|start|>assistant<|channel|>final"
|
|
||||||
"<|message|>I'm in the middle of answering"
|
|
||||||
)
|
|
||||||
token_ids = get_encoding().encode(harmony_str, allowed_special="all")
|
|
||||||
reasoning, final_content, _ = parse_chat_output(token_ids)
|
|
||||||
assert reasoning == "I'm thinking."
|
|
||||||
assert final_content == "I'm in the middle of answering"
|
|
||||||
|
|
||||||
def test_parse_chat_output_complete_content(self) -> None:
|
|
||||||
harmony_str = "<|channel|>final<|message|>The answer is 4.<|end|>"
|
|
||||||
token_ids = get_encoding().encode(harmony_str, allowed_special="all")
|
|
||||||
reasoning, final_content, _ = parse_chat_output(token_ids)
|
|
||||||
assert reasoning is None
|
|
||||||
assert final_content == "The answer is 4."
|
|
||||||
|
|
||||||
def test_parse_chat_output_complete_commentary(self) -> None:
|
|
||||||
harmony_str = (
|
|
||||||
"<|channel|>commentary<|message|>I need to call some tools.<|end|>"
|
|
||||||
)
|
|
||||||
token_ids = get_encoding().encode(harmony_str, allowed_special="all")
|
|
||||||
reasoning, final_content, _ = parse_chat_output(token_ids)
|
|
||||||
assert reasoning is None
|
|
||||||
assert final_content == "I need to call some tools."
|
|
||||||
|
|
||||||
def test_parse_chat_output_complete_reasoning(self) -> None:
|
|
||||||
harmony_str = (
|
|
||||||
"<|channel|>analysis<|message|>I've thought hard about this.<|end|>"
|
|
||||||
)
|
|
||||||
token_ids = get_encoding().encode(harmony_str, allowed_special="all")
|
|
||||||
reasoning, final_content, _ = parse_chat_output(token_ids)
|
|
||||||
assert reasoning == "I've thought hard about this."
|
|
||||||
assert final_content is None
|
|
||||||
|
|
||||||
def test_parse_chat_output_complete_reasoning_and_content(self) -> None:
|
|
||||||
harmony_str = (
|
|
||||||
"<|channel|>analysis<|message|>I've thought hard about this.<|end|>"
|
|
||||||
"<|start|>assistant<|channel|>final<|message|>The answer is 4.<|end|>"
|
|
||||||
)
|
|
||||||
token_ids = get_encoding().encode(harmony_str, allowed_special="all")
|
|
||||||
reasoning, final_content, _ = parse_chat_output(token_ids)
|
|
||||||
assert reasoning == "I've thought hard about this."
|
|
||||||
assert final_content == "The answer is 4."
|
|
||||||
|
|
||||||
def test_parse_chat_output_commentary_with_recipient_excluded(self) -> None:
|
|
||||||
"""Commentary with a recipient (tool call) should not appear in
|
|
||||||
final_content — those are handled separately by the tool parser.
|
|
||||||
|
|
||||||
The first message is a preamble (visible), the second is a tool
|
|
||||||
call (excluded). Only the preamble should appear in final_content.
|
|
||||||
"""
|
|
||||||
harmony_str = (
|
|
||||||
"<|channel|>commentary"
|
|
||||||
"<|message|>Let me check the weather.<|end|>"
|
|
||||||
"<|start|>assistant to=functions.get_weather"
|
|
||||||
"<|channel|>commentary"
|
|
||||||
'<|message|>{"location": "SF"}<|end|>'
|
|
||||||
)
|
|
||||||
token_ids = get_encoding().encode(harmony_str, allowed_special="all")
|
|
||||||
reasoning, final_content, _ = parse_chat_output(token_ids)
|
|
||||||
assert reasoning is None
|
|
||||||
assert final_content == "Let me check the weather."
|
|
||||||
|
|
||||||
def test_parse_chat_output_interrupted_preamble(self) -> None:
|
|
||||||
"""Partial/interrupted preamble (commentary without recipient) should
|
|
||||||
appear in final_content, not reasoning."""
|
|
||||||
harmony_str = "<|channel|>commentary<|message|>I'll search for that"
|
|
||||||
token_ids = get_encoding().encode(harmony_str, allowed_special="all")
|
|
||||||
reasoning, final_content, _ = parse_chat_output(token_ids)
|
|
||||||
assert reasoning is None
|
|
||||||
assert final_content == "I'll search for that"
|
|
||||||
|
|
||||||
def test_parse_chat_output_preamble_then_final(self) -> None:
|
|
||||||
"""Preamble followed by a final message should both appear in
|
|
||||||
final_content, joined by newline."""
|
|
||||||
harmony_str = (
|
|
||||||
"<|channel|>commentary"
|
|
||||||
"<|message|>Let me look that up.<|end|>"
|
|
||||||
"<|start|>assistant<|channel|>final"
|
|
||||||
"<|message|>The answer is 42.<|end|>"
|
|
||||||
)
|
|
||||||
token_ids = get_encoding().encode(harmony_str, allowed_special="all")
|
|
||||||
reasoning, final_content, _ = parse_chat_output(token_ids)
|
|
||||||
assert reasoning is None
|
|
||||||
assert final_content == "Let me look that up.\nThe answer is 42."
|
|
||||||
|
|
||||||
|
|
||||||
def test_has_custom_tools() -> None:
|
def test_has_custom_tools() -> None:
|
||||||
assert not has_custom_tools(set())
|
assert not has_custom_tools(set())
|
||||||
assert not has_custom_tools({"web_search_preview", "code_interpreter", "container"})
|
assert not has_custom_tools({"web_search_preview", "code_interpreter", "container"})
|
||||||
|
|||||||
@@ -0,0 +1,452 @@
|
|||||||
|
# SPDX-License-Identifier: Apache-2.0
|
||||||
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||||
|
|
||||||
|
import json
|
||||||
|
from collections.abc import Sequence
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
from openai_harmony import (
|
||||||
|
Conversation,
|
||||||
|
Message,
|
||||||
|
RenderConversationConfig,
|
||||||
|
Role,
|
||||||
|
)
|
||||||
|
from transformers import AutoTokenizer
|
||||||
|
|
||||||
|
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
|
||||||
|
from vllm.entrypoints.openai.engine.protocol import FunctionCall
|
||||||
|
from vllm.entrypoints.openai.parser.harmony_utils import (
|
||||||
|
get_encoding,
|
||||||
|
)
|
||||||
|
from vllm.parser.harmony import HarmonyParser
|
||||||
|
from vllm.parser.parser_manager import ParserManager
|
||||||
|
|
||||||
|
REASONING_MODEL_NAME = "openai/gpt-oss-20b"
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture(scope="module")
|
||||||
|
def gpt_oss_tokenizer():
|
||||||
|
return AutoTokenizer.from_pretrained(REASONING_MODEL_NAME)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture
|
||||||
|
def harmony_parser(gpt_oss_tokenizer):
|
||||||
|
parser_cls = ParserManager.get_parser(
|
||||||
|
tool_parser_name="openai",
|
||||||
|
reasoning_parser_name="openai_gptoss",
|
||||||
|
enable_auto_tools=True,
|
||||||
|
model_name=REASONING_MODEL_NAME,
|
||||||
|
is_harmony=True,
|
||||||
|
)
|
||||||
|
assert parser_cls is HarmonyParser
|
||||||
|
return parser_cls(gpt_oss_tokenizer)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture
|
||||||
|
def chat_request():
|
||||||
|
return ChatCompletionRequest(
|
||||||
|
model="openai/gpt-oss-20b",
|
||||||
|
messages=[{"role": "user", "content": "Hello"}],
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def encode_output(harmony_str: str) -> list[int]:
|
||||||
|
return get_encoding().encode(harmony_str, allowed_special="all")
|
||||||
|
|
||||||
|
|
||||||
|
def assistant(content: str, channel: str) -> Message:
|
||||||
|
return Message.from_role_and_content(Role.ASSISTANT, content).with_channel(channel)
|
||||||
|
|
||||||
|
|
||||||
|
def tool_call(
|
||||||
|
recipient: str,
|
||||||
|
content: str,
|
||||||
|
channel: str = "commentary",
|
||||||
|
content_type: str | None = "json",
|
||||||
|
) -> Message:
|
||||||
|
message = assistant(content, channel).with_recipient(recipient)
|
||||||
|
return message if content_type is None else message.with_content_type(content_type)
|
||||||
|
|
||||||
|
|
||||||
|
def get_model_output_tokens(
|
||||||
|
prompt_messages: Sequence[Message],
|
||||||
|
response_messages: Sequence[Message],
|
||||||
|
) -> list[int]:
|
||||||
|
enc = get_encoding()
|
||||||
|
# Keep analysis messages when synthesizing model-output-only token sequences
|
||||||
|
# for parser tests; the default render path drops them after a later final turn.
|
||||||
|
config = RenderConversationConfig(auto_drop_analysis=False)
|
||||||
|
prompt_ids = enc.render_conversation_for_completion(
|
||||||
|
Conversation.from_messages(list(prompt_messages)),
|
||||||
|
Role.ASSISTANT,
|
||||||
|
config=config,
|
||||||
|
)
|
||||||
|
full_ids = enc.render_conversation_for_completion(
|
||||||
|
Conversation.from_messages([*prompt_messages, *response_messages]),
|
||||||
|
Role.ASSISTANT,
|
||||||
|
config=config,
|
||||||
|
)
|
||||||
|
assert full_ids[: len(prompt_ids)] == prompt_ids
|
||||||
|
return full_ids[len(prompt_ids) :]
|
||||||
|
|
||||||
|
|
||||||
|
def get_text(msg: Message) -> str:
|
||||||
|
return msg.content[0].text if msg.content else ""
|
||||||
|
|
||||||
|
|
||||||
|
def visible_segments(result) -> list[tuple[str | None, str | None, str]]:
|
||||||
|
return [
|
||||||
|
(segment.channel, segment.recipient, segment.delta)
|
||||||
|
for segment in result.segments
|
||||||
|
if not segment.is_boundary and segment.delta
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
def tool_call_tuples(tool_calls: list[FunctionCall] | None) -> list[tuple[str, str]]:
|
||||||
|
return [] if tool_calls is None else [(tc.name, tc.arguments) for tc in tool_calls]
|
||||||
|
|
||||||
|
|
||||||
|
class TestParse:
|
||||||
|
# Rendered conversation outputs.
|
||||||
|
|
||||||
|
def test_reasoning_only(self, harmony_parser, chat_request):
|
||||||
|
prompt = [Message.from_role_and_content(Role.USER, "Why?")]
|
||||||
|
response = [assistant("This is reasoning", "analysis")]
|
||||||
|
|
||||||
|
reasoning, content, tool_calls = harmony_parser.parse(
|
||||||
|
"",
|
||||||
|
chat_request,
|
||||||
|
model_output_token_ids=get_model_output_tokens(prompt, response),
|
||||||
|
)
|
||||||
|
|
||||||
|
assert reasoning == "This is reasoning"
|
||||||
|
assert content is None
|
||||||
|
assert tool_calls is None
|
||||||
|
|
||||||
|
def test_content_only(self, harmony_parser, chat_request):
|
||||||
|
prompt = [Message.from_role_and_content(Role.USER, "Hello")]
|
||||||
|
response = [assistant("This is a test", "final")]
|
||||||
|
|
||||||
|
reasoning, content, tool_calls = harmony_parser.parse(
|
||||||
|
"",
|
||||||
|
chat_request,
|
||||||
|
model_output_token_ids=get_model_output_tokens(prompt, response),
|
||||||
|
)
|
||||||
|
|
||||||
|
assert reasoning is None
|
||||||
|
assert content == "This is a test"
|
||||||
|
assert tool_calls is None
|
||||||
|
|
||||||
|
def test_reasoning_and_content(self, harmony_parser, chat_request):
|
||||||
|
prompt = [Message.from_role_and_content(Role.USER, "What is 2+2?")]
|
||||||
|
response = [
|
||||||
|
assistant("I should think first.", "analysis"),
|
||||||
|
assistant("The answer is 4.", "final"),
|
||||||
|
]
|
||||||
|
|
||||||
|
reasoning, content, tool_calls = harmony_parser.parse(
|
||||||
|
"",
|
||||||
|
chat_request,
|
||||||
|
model_output_token_ids=get_model_output_tokens(prompt, response),
|
||||||
|
)
|
||||||
|
|
||||||
|
assert reasoning == "I should think first."
|
||||||
|
assert content == "The answer is 4."
|
||||||
|
assert tool_calls is None
|
||||||
|
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
"tool_args",
|
||||||
|
[
|
||||||
|
'{"location": "Tokyo"}',
|
||||||
|
'{\n"location": "Tokyo"\n}',
|
||||||
|
],
|
||||||
|
)
|
||||||
|
@pytest.mark.parametrize("tool_channel", ["commentary", "analysis"])
|
||||||
|
def test_single_tool_call(
|
||||||
|
self, harmony_parser, chat_request, tool_args, tool_channel
|
||||||
|
):
|
||||||
|
prompt = [
|
||||||
|
Message.from_role_and_content(Role.USER, "What is the weather in Tokyo?")
|
||||||
|
]
|
||||||
|
response = [tool_call("functions.get_current_weather", tool_args, tool_channel)]
|
||||||
|
|
||||||
|
reasoning, content, tool_calls = harmony_parser.parse(
|
||||||
|
"",
|
||||||
|
chat_request,
|
||||||
|
model_output_token_ids=get_model_output_tokens(prompt, response),
|
||||||
|
)
|
||||||
|
|
||||||
|
assert reasoning is None
|
||||||
|
assert content is None
|
||||||
|
assert tool_call_tuples(tool_calls) == [
|
||||||
|
("get_current_weather", json.dumps({"location": "Tokyo"}))
|
||||||
|
]
|
||||||
|
|
||||||
|
def test_multiple_tool_calls_varied_formats(self, harmony_parser, chat_request):
|
||||||
|
prompt = [
|
||||||
|
Message.from_role_and_content(
|
||||||
|
Role.USER, "What is the weather in Tokyo based on where I'm at?"
|
||||||
|
)
|
||||||
|
]
|
||||||
|
response = [
|
||||||
|
tool_call("functions.get_current_weather", '{"location": "Tokyo"}'),
|
||||||
|
tool_call("functions.get_user_location", '{"location": "Tokyo"}'),
|
||||||
|
tool_call(
|
||||||
|
"functions.no_content_type",
|
||||||
|
'{"location": "Tokyo"}',
|
||||||
|
content_type=None,
|
||||||
|
),
|
||||||
|
tool_call("functions.not_json_no_content_type", "foo", content_type=None),
|
||||||
|
tool_call("functions.empty_args", "{}"),
|
||||||
|
tool_call("functions.no_args", ""),
|
||||||
|
]
|
||||||
|
|
||||||
|
_, content, tool_calls = harmony_parser.parse(
|
||||||
|
"",
|
||||||
|
chat_request,
|
||||||
|
model_output_token_ids=get_model_output_tokens(prompt, response),
|
||||||
|
)
|
||||||
|
|
||||||
|
assert content is None
|
||||||
|
assert tool_call_tuples(tool_calls) == [
|
||||||
|
("get_current_weather", json.dumps({"location": "Tokyo"})),
|
||||||
|
("get_user_location", json.dumps({"location": "Tokyo"})),
|
||||||
|
("no_content_type", json.dumps({"location": "Tokyo"})),
|
||||||
|
("not_json_no_content_type", "foo"),
|
||||||
|
("empty_args", json.dumps({})),
|
||||||
|
("no_args", ""),
|
||||||
|
]
|
||||||
|
|
||||||
|
def test_tool_call_bare_recipient(self, harmony_parser, chat_request):
|
||||||
|
prompt = [Message.from_role_and_content(Role.USER, "Weather?")]
|
||||||
|
response = [tool_call("get_current_weather", '{"location": "Tokyo"}')]
|
||||||
|
|
||||||
|
_, _, tool_calls = harmony_parser.parse(
|
||||||
|
"",
|
||||||
|
chat_request,
|
||||||
|
model_output_token_ids=get_model_output_tokens(prompt, response),
|
||||||
|
)
|
||||||
|
|
||||||
|
assert tool_call_tuples(tool_calls) == [
|
||||||
|
("get_current_weather", json.dumps({"location": "Tokyo"}))
|
||||||
|
]
|
||||||
|
|
||||||
|
def test_multiple_tool_calls_bare_recipients(self, harmony_parser, chat_request):
|
||||||
|
prompt = [Message.from_role_and_content(Role.USER, "Use both tools.")]
|
||||||
|
response = [
|
||||||
|
tool_call("get_current_weather", '{"location": "Tokyo"}'),
|
||||||
|
tool_call("get_user_location", "{}"),
|
||||||
|
]
|
||||||
|
|
||||||
|
_, _, tool_calls = harmony_parser.parse(
|
||||||
|
"",
|
||||||
|
chat_request,
|
||||||
|
model_output_token_ids=get_model_output_tokens(prompt, response),
|
||||||
|
)
|
||||||
|
|
||||||
|
assert tool_call_tuples(tool_calls) == [
|
||||||
|
("get_current_weather", json.dumps({"location": "Tokyo"})),
|
||||||
|
("get_user_location", json.dumps({})),
|
||||||
|
]
|
||||||
|
|
||||||
|
def test_assistant_recipient_not_tool(self, harmony_parser, chat_request):
|
||||||
|
prompt = [Message.from_role_and_content(Role.USER, "Hello")]
|
||||||
|
response = [
|
||||||
|
tool_call("assistant", "Some tool response", content_type=None),
|
||||||
|
assistant("Here is the answer", "final"),
|
||||||
|
]
|
||||||
|
|
||||||
|
reasoning, content, tool_calls = harmony_parser.parse(
|
||||||
|
"",
|
||||||
|
chat_request,
|
||||||
|
model_output_token_ids=get_model_output_tokens(prompt, response),
|
||||||
|
)
|
||||||
|
|
||||||
|
assert reasoning is None
|
||||||
|
assert content == "Here is the answer"
|
||||||
|
assert tool_calls is None
|
||||||
|
|
||||||
|
def test_tool_call_dotted_name(self, harmony_parser, chat_request):
|
||||||
|
prompt = [Message.from_role_and_content(Role.USER, "Compute 2+3")]
|
||||||
|
response = [tool_call("math.sum", '{"a": 2, "b": 3}')]
|
||||||
|
|
||||||
|
_, _, tool_calls = harmony_parser.parse(
|
||||||
|
"",
|
||||||
|
chat_request,
|
||||||
|
model_output_token_ids=get_model_output_tokens(prompt, response),
|
||||||
|
)
|
||||||
|
|
||||||
|
assert tool_call_tuples(tool_calls) == [
|
||||||
|
("math.sum", json.dumps({"a": 2, "b": 3}))
|
||||||
|
]
|
||||||
|
|
||||||
|
def test_tool_calls_with_final_content(self, harmony_parser, chat_request):
|
||||||
|
prompt = [Message.from_role_and_content(Role.USER, "What is the weather?")]
|
||||||
|
response = [
|
||||||
|
assistant("User asked about the weather.", "analysis"),
|
||||||
|
tool_call("functions.get_current_weather", '{"location": "Tokyo"}'),
|
||||||
|
assistant("This tool call will get the weather.", "final"),
|
||||||
|
]
|
||||||
|
|
||||||
|
reasoning, content, tool_calls = harmony_parser.parse(
|
||||||
|
"",
|
||||||
|
chat_request,
|
||||||
|
model_output_token_ids=get_model_output_tokens(prompt, response),
|
||||||
|
)
|
||||||
|
|
||||||
|
assert reasoning == "User asked about the weather."
|
||||||
|
assert content == "This tool call will get the weather."
|
||||||
|
assert tool_call_tuples(tool_calls) == [
|
||||||
|
("get_current_weather", json.dumps({"location": "Tokyo"}))
|
||||||
|
]
|
||||||
|
|
||||||
|
# Raw/truncated Harmony output streams.
|
||||||
|
|
||||||
|
def test_interrupted_first_message(self, harmony_parser, chat_request):
|
||||||
|
reasoning, content, tool_calls = harmony_parser.parse(
|
||||||
|
"",
|
||||||
|
chat_request,
|
||||||
|
model_output_token_ids=encode_output(
|
||||||
|
"<|channel|>final<|message|>I'm in the middle of answering"
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
assert reasoning is None
|
||||||
|
assert content == "I'm in the middle of answering"
|
||||||
|
assert tool_calls is None
|
||||||
|
|
||||||
|
def test_interrupted_reasoning_first_message(self, harmony_parser, chat_request):
|
||||||
|
reasoning, content, tool_calls = harmony_parser.parse(
|
||||||
|
"",
|
||||||
|
chat_request,
|
||||||
|
model_output_token_ids=encode_output(
|
||||||
|
"<|channel|>analysis<|message|>I'm in the middle of thinking"
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
assert reasoning == "I'm in the middle of thinking"
|
||||||
|
assert content is None
|
||||||
|
assert tool_calls is None
|
||||||
|
|
||||||
|
def test_truncated_output(self, harmony_parser, chat_request):
|
||||||
|
reasoning, content, tool_calls = harmony_parser.parse(
|
||||||
|
"",
|
||||||
|
chat_request,
|
||||||
|
model_output_token_ids=encode_output(
|
||||||
|
"<|channel|>analysis<|message|>I'm thinking.<|end|>"
|
||||||
|
"<|start|>assistant<|channel|>final<|message|>"
|
||||||
|
"I'm in the middle of answering"
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
assert reasoning == "I'm thinking."
|
||||||
|
assert content == "I'm in the middle of answering"
|
||||||
|
assert tool_calls is None
|
||||||
|
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
("harmony_str", "expected_content"),
|
||||||
|
[
|
||||||
|
(
|
||||||
|
"<|channel|>commentary<|message|>I'll search for that",
|
||||||
|
"I'll search for that",
|
||||||
|
),
|
||||||
|
(
|
||||||
|
"<|channel|>commentary<|message|>Let me look that up.<|end|>"
|
||||||
|
"<|start|>assistant<|channel|>final<|message|>The answer is 42.<|end|>",
|
||||||
|
"Let me look that up.\nThe answer is 42.",
|
||||||
|
),
|
||||||
|
],
|
||||||
|
)
|
||||||
|
def test_commentary_preambles(
|
||||||
|
self,
|
||||||
|
harmony_parser,
|
||||||
|
chat_request,
|
||||||
|
harmony_str,
|
||||||
|
expected_content,
|
||||||
|
):
|
||||||
|
reasoning, content, tool_calls = harmony_parser.parse(
|
||||||
|
"",
|
||||||
|
chat_request,
|
||||||
|
model_output_token_ids=encode_output(harmony_str),
|
||||||
|
)
|
||||||
|
|
||||||
|
assert reasoning is None
|
||||||
|
assert content == expected_content
|
||||||
|
assert tool_calls is None
|
||||||
|
|
||||||
|
def test_commentary_with_recipient_excluded(self, harmony_parser, chat_request):
|
||||||
|
reasoning, content, tool_calls = harmony_parser.parse(
|
||||||
|
"",
|
||||||
|
chat_request,
|
||||||
|
model_output_token_ids=encode_output(
|
||||||
|
"<|channel|>commentary"
|
||||||
|
"<|message|>Let me check the weather.<|end|>"
|
||||||
|
"<|start|>assistant to=functions.get_weather"
|
||||||
|
"<|channel|>commentary"
|
||||||
|
'<|message|>{"location": "SF"}<|end|>'
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
assert reasoning is None
|
||||||
|
assert content == "Let me check the weather."
|
||||||
|
assert tool_call_tuples(tool_calls) == [
|
||||||
|
("get_weather", json.dumps({"location": "SF"}))
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
class TestProcessChunk:
|
||||||
|
def test_empty(self, harmony_parser):
|
||||||
|
result = harmony_parser.process_chunk([])
|
||||||
|
assert result.segments == []
|
||||||
|
assert result.reasoning_token_count == 0
|
||||||
|
|
||||||
|
def test_single_channel(self, harmony_parser):
|
||||||
|
result = harmony_parser.process_chunk(
|
||||||
|
encode_output("<|channel|>final<|message|>Hello")
|
||||||
|
)
|
||||||
|
|
||||||
|
assert visible_segments(result) == [("final", None, "Hello")]
|
||||||
|
|
||||||
|
def test_cross_channel(self, harmony_parser):
|
||||||
|
result = harmony_parser.process_chunk(
|
||||||
|
encode_output(
|
||||||
|
"<|channel|>analysis<|message|>Think<|end|>"
|
||||||
|
"<|start|>assistant<|channel|>final<|message|>Answer"
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
assert visible_segments(result) == [
|
||||||
|
("analysis", None, "Think"),
|
||||||
|
("final", None, "Answer"),
|
||||||
|
]
|
||||||
|
|
||||||
|
def test_boundary_detection(self, harmony_parser):
|
||||||
|
result = harmony_parser.process_chunk(
|
||||||
|
encode_output("<|channel|>final<|message|>Done<|end|>")
|
||||||
|
)
|
||||||
|
|
||||||
|
boundary_segments = [
|
||||||
|
segment for segment in result.segments if segment.is_boundary
|
||||||
|
]
|
||||||
|
assert len(boundary_segments) == 1
|
||||||
|
assert boundary_segments[0].completed_message is not None
|
||||||
|
assert boundary_segments[0].completed_message.channel == "final"
|
||||||
|
assert get_text(boundary_segments[0].completed_message) == "Done"
|
||||||
|
|
||||||
|
def test_multi_boundary(self, harmony_parser):
|
||||||
|
result = harmony_parser.process_chunk(
|
||||||
|
encode_output(
|
||||||
|
"<|channel|>analysis<|message|>One<|end|>"
|
||||||
|
"<|start|>assistant<|channel|>final<|message|>Two<|end|>"
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
boundary_segments = [
|
||||||
|
segment for segment in result.segments if segment.is_boundary
|
||||||
|
]
|
||||||
|
assert [
|
||||||
|
get_text(segment.completed_message) for segment in boundary_segments
|
||||||
|
] == [
|
||||||
|
"One",
|
||||||
|
"Two",
|
||||||
|
]
|
||||||
@@ -1,415 +0,0 @@
|
|||||||
# SPDX-License-Identifier: Apache-2.0
|
|
||||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
|
||||||
|
|
||||||
import json
|
|
||||||
|
|
||||||
import pytest
|
|
||||||
from openai_harmony import (
|
|
||||||
Conversation,
|
|
||||||
DeveloperContent,
|
|
||||||
HarmonyEncodingName,
|
|
||||||
Message,
|
|
||||||
Role,
|
|
||||||
SystemContent,
|
|
||||||
load_harmony_encoding,
|
|
||||||
)
|
|
||||||
|
|
||||||
from vllm.entrypoints.openai.engine.protocol import FunctionCall, ToolCall
|
|
||||||
from vllm.tokenizers import get_tokenizer
|
|
||||||
from vllm.tool_parsers.openai_tool_parser import OpenAIToolParser
|
|
||||||
|
|
||||||
MODEL = "gpt2"
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.fixture(scope="module")
|
|
||||||
def openai_tokenizer():
|
|
||||||
# The parser does not use the tokenizer, but the constructor requires it.
|
|
||||||
return get_tokenizer(MODEL)
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.fixture
|
|
||||||
def openai_tool_parser(openai_tokenizer):
|
|
||||||
return OpenAIToolParser(openai_tokenizer)
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.fixture(scope="module")
|
|
||||||
def harmony_encoding():
|
|
||||||
return load_harmony_encoding(HarmonyEncodingName.HARMONY_GPT_OSS)
|
|
||||||
|
|
||||||
|
|
||||||
def assert_tool_calls(
|
|
||||||
actual_tool_calls: list[ToolCall],
|
|
||||||
expected_tool_calls: list[ToolCall],
|
|
||||||
):
|
|
||||||
assert len(actual_tool_calls) == len(expected_tool_calls)
|
|
||||||
|
|
||||||
for actual_tool_call, expected_tool_call in zip(
|
|
||||||
actual_tool_calls, expected_tool_calls
|
|
||||||
):
|
|
||||||
assert isinstance(actual_tool_call.id, str)
|
|
||||||
assert len(actual_tool_call.id) > 16 # Default from protocol.py
|
|
||||||
assert actual_tool_call.type == "function"
|
|
||||||
assert actual_tool_call.function == expected_tool_call.function
|
|
||||||
|
|
||||||
|
|
||||||
def test_extract_tool_calls_no_tools(openai_tool_parser, harmony_encoding):
|
|
||||||
convo = Conversation.from_messages(
|
|
||||||
[
|
|
||||||
Message.from_role_and_content(
|
|
||||||
Role.SYSTEM,
|
|
||||||
SystemContent.new(),
|
|
||||||
),
|
|
||||||
Message.from_role_and_content(
|
|
||||||
Role.DEVELOPER,
|
|
||||||
DeveloperContent.new().with_instructions("Talk like a pirate!"),
|
|
||||||
),
|
|
||||||
Message.from_role_and_content(Role.USER, "Arrr, how be you?"),
|
|
||||||
Message.from_role_and_content(
|
|
||||||
Role.ASSISTANT, "This is a test"
|
|
||||||
).with_channel("final"),
|
|
||||||
]
|
|
||||||
)
|
|
||||||
token_ids = harmony_encoding.render_conversation_for_completion(
|
|
||||||
convo, Role.ASSISTANT
|
|
||||||
)
|
|
||||||
extracted_info = openai_tool_parser.extract_tool_calls(
|
|
||||||
"",
|
|
||||||
request=None,
|
|
||||||
token_ids=token_ids,
|
|
||||||
)
|
|
||||||
assert not extracted_info.tools_called
|
|
||||||
assert extracted_info.tool_calls == []
|
|
||||||
assert extracted_info.content == "This is a test"
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.parametrize(
|
|
||||||
"tool_args",
|
|
||||||
[
|
|
||||||
'{"location": "Tokyo"}',
|
|
||||||
'{\n"location": "Tokyo"\n}',
|
|
||||||
],
|
|
||||||
)
|
|
||||||
def test_extract_tool_calls_single_tool(
|
|
||||||
openai_tool_parser, harmony_encoding, tool_args
|
|
||||||
):
|
|
||||||
convo = Conversation.from_messages(
|
|
||||||
[
|
|
||||||
Message.from_role_and_content(Role.USER, "What is the weather in Tokyo?"),
|
|
||||||
Message.from_role_and_content(
|
|
||||||
Role.ASSISTANT,
|
|
||||||
'User asks: "What is the weather in Tokyo?" We need to use get_current_weather tool.', # noqa: E501
|
|
||||||
).with_channel("analysis"),
|
|
||||||
Message.from_role_and_content(Role.ASSISTANT, tool_args)
|
|
||||||
.with_channel("commentary")
|
|
||||||
.with_recipient("functions.get_current_weather")
|
|
||||||
.with_content_type("json"),
|
|
||||||
]
|
|
||||||
)
|
|
||||||
token_ids = harmony_encoding.render_conversation_for_completion(
|
|
||||||
convo, Role.ASSISTANT
|
|
||||||
)
|
|
||||||
|
|
||||||
extracted_info = openai_tool_parser.extract_tool_calls(
|
|
||||||
"",
|
|
||||||
request=None,
|
|
||||||
token_ids=token_ids,
|
|
||||||
)
|
|
||||||
assert extracted_info.tools_called
|
|
||||||
expected_tool_calls = [
|
|
||||||
ToolCall(
|
|
||||||
function=FunctionCall(
|
|
||||||
name="get_current_weather",
|
|
||||||
arguments=json.dumps({"location": "Tokyo"}),
|
|
||||||
)
|
|
||||||
)
|
|
||||||
]
|
|
||||||
assert_tool_calls(extracted_info.tool_calls, expected_tool_calls)
|
|
||||||
assert extracted_info.content is None
|
|
||||||
|
|
||||||
|
|
||||||
def test_extract_tool_calls_multiple_tools(
|
|
||||||
openai_tool_parser,
|
|
||||||
harmony_encoding,
|
|
||||||
):
|
|
||||||
convo = Conversation.from_messages(
|
|
||||||
[
|
|
||||||
Message.from_role_and_content(
|
|
||||||
Role.USER, "What is the weather in Tokyo based on where I'm at?"
|
|
||||||
),
|
|
||||||
Message.from_role_and_content(
|
|
||||||
Role.ASSISTANT,
|
|
||||||
'User asks: "What is the weather in Tokyo?" based on their location. We need to use get_current_weather tool and get_user_location tool.', # noqa: E501
|
|
||||||
).with_channel("analysis"),
|
|
||||||
Message.from_role_and_content(Role.ASSISTANT, '{"location": "Tokyo"}')
|
|
||||||
.with_channel("commentary")
|
|
||||||
.with_recipient("functions.get_current_weather")
|
|
||||||
.with_content_type("json"),
|
|
||||||
Message.from_role_and_content(Role.ASSISTANT, '{"location": "Tokyo"}')
|
|
||||||
.with_channel("commentary")
|
|
||||||
.with_recipient("functions.get_user_location")
|
|
||||||
.with_content_type("json"),
|
|
||||||
Message.from_role_and_content(Role.ASSISTANT, '{"location": "Tokyo"}')
|
|
||||||
.with_channel("commentary")
|
|
||||||
.with_recipient("functions.no_content_type"),
|
|
||||||
Message.from_role_and_content(Role.ASSISTANT, "foo")
|
|
||||||
.with_channel("commentary")
|
|
||||||
.with_recipient("functions.not_json_no_content_type"),
|
|
||||||
Message.from_role_and_content(Role.ASSISTANT, "{}")
|
|
||||||
.with_channel("commentary")
|
|
||||||
.with_recipient("functions.empty_args")
|
|
||||||
.with_content_type("json"),
|
|
||||||
Message.from_role_and_content(Role.ASSISTANT, "")
|
|
||||||
.with_channel("commentary")
|
|
||||||
.with_recipient("functions.no_args")
|
|
||||||
.with_content_type("json"),
|
|
||||||
]
|
|
||||||
)
|
|
||||||
token_ids = harmony_encoding.render_conversation_for_completion(
|
|
||||||
convo,
|
|
||||||
Role.ASSISTANT,
|
|
||||||
)
|
|
||||||
|
|
||||||
extracted_info = openai_tool_parser.extract_tool_calls(
|
|
||||||
"",
|
|
||||||
request=None,
|
|
||||||
token_ids=token_ids,
|
|
||||||
)
|
|
||||||
assert extracted_info.tools_called
|
|
||||||
expected_tool_calls = [
|
|
||||||
ToolCall(
|
|
||||||
function=FunctionCall(
|
|
||||||
name="get_current_weather",
|
|
||||||
arguments=json.dumps({"location": "Tokyo"}),
|
|
||||||
)
|
|
||||||
),
|
|
||||||
ToolCall(
|
|
||||||
function=FunctionCall(
|
|
||||||
name="get_user_location",
|
|
||||||
arguments=json.dumps({"location": "Tokyo"}),
|
|
||||||
)
|
|
||||||
),
|
|
||||||
ToolCall(
|
|
||||||
function=FunctionCall(
|
|
||||||
name="no_content_type",
|
|
||||||
arguments=json.dumps({"location": "Tokyo"}),
|
|
||||||
)
|
|
||||||
),
|
|
||||||
ToolCall(
|
|
||||||
function=FunctionCall(
|
|
||||||
name="not_json_no_content_type",
|
|
||||||
arguments="foo",
|
|
||||||
)
|
|
||||||
),
|
|
||||||
ToolCall(
|
|
||||||
function=FunctionCall(
|
|
||||||
name="empty_args",
|
|
||||||
arguments=json.dumps({}),
|
|
||||||
)
|
|
||||||
),
|
|
||||||
ToolCall(
|
|
||||||
function=FunctionCall(
|
|
||||||
name="no_args",
|
|
||||||
arguments="",
|
|
||||||
)
|
|
||||||
),
|
|
||||||
]
|
|
||||||
assert_tool_calls(extracted_info.tool_calls, expected_tool_calls)
|
|
||||||
assert extracted_info.content is None
|
|
||||||
|
|
||||||
|
|
||||||
def test_extract_tool_calls_bare_function_name(
|
|
||||||
openai_tool_parser,
|
|
||||||
harmony_encoding,
|
|
||||||
):
|
|
||||||
convo = Conversation.from_messages(
|
|
||||||
[
|
|
||||||
Message.from_role_and_content(Role.USER, "What is the weather in Tokyo?"),
|
|
||||||
Message.from_role_and_content(
|
|
||||||
Role.ASSISTANT,
|
|
||||||
"We need to use get_current_weather tool.",
|
|
||||||
).with_channel("analysis"),
|
|
||||||
Message.from_role_and_content(Role.ASSISTANT, '{"location": "Tokyo"}')
|
|
||||||
.with_channel("commentary")
|
|
||||||
.with_recipient("get_current_weather")
|
|
||||||
.with_content_type("json"),
|
|
||||||
]
|
|
||||||
)
|
|
||||||
token_ids = harmony_encoding.render_conversation_for_completion(
|
|
||||||
convo, Role.ASSISTANT
|
|
||||||
)
|
|
||||||
|
|
||||||
extracted_info = openai_tool_parser.extract_tool_calls(
|
|
||||||
"",
|
|
||||||
request=None,
|
|
||||||
token_ids=token_ids,
|
|
||||||
)
|
|
||||||
assert extracted_info.tools_called
|
|
||||||
expected_tool_calls = [
|
|
||||||
ToolCall(
|
|
||||||
function=FunctionCall(
|
|
||||||
name="get_current_weather",
|
|
||||||
arguments=json.dumps({"location": "Tokyo"}),
|
|
||||||
)
|
|
||||||
)
|
|
||||||
]
|
|
||||||
assert_tool_calls(extracted_info.tool_calls, expected_tool_calls)
|
|
||||||
assert extracted_info.content is None
|
|
||||||
|
|
||||||
|
|
||||||
def test_extract_tool_calls_bare_function_name_multiple(
|
|
||||||
openai_tool_parser,
|
|
||||||
harmony_encoding,
|
|
||||||
):
|
|
||||||
convo = Conversation.from_messages(
|
|
||||||
[
|
|
||||||
Message.from_role_and_content(
|
|
||||||
Role.USER, "What is the weather in Tokyo based on where I'm at?"
|
|
||||||
),
|
|
||||||
Message.from_role_and_content(
|
|
||||||
Role.ASSISTANT,
|
|
||||||
"We need to use both tools.",
|
|
||||||
).with_channel("analysis"),
|
|
||||||
Message.from_role_and_content(Role.ASSISTANT, '{"location": "Tokyo"}')
|
|
||||||
.with_channel("commentary")
|
|
||||||
.with_recipient("get_current_weather")
|
|
||||||
.with_content_type("json"),
|
|
||||||
Message.from_role_and_content(Role.ASSISTANT, "{}")
|
|
||||||
.with_channel("commentary")
|
|
||||||
.with_recipient("get_user_location")
|
|
||||||
.with_content_type("json"),
|
|
||||||
]
|
|
||||||
)
|
|
||||||
token_ids = harmony_encoding.render_conversation_for_completion(
|
|
||||||
convo, Role.ASSISTANT
|
|
||||||
)
|
|
||||||
|
|
||||||
extracted_info = openai_tool_parser.extract_tool_calls(
|
|
||||||
"",
|
|
||||||
request=None,
|
|
||||||
token_ids=token_ids,
|
|
||||||
)
|
|
||||||
assert extracted_info.tools_called
|
|
||||||
expected_tool_calls = [
|
|
||||||
ToolCall(
|
|
||||||
function=FunctionCall(
|
|
||||||
name="get_current_weather",
|
|
||||||
arguments=json.dumps({"location": "Tokyo"}),
|
|
||||||
)
|
|
||||||
),
|
|
||||||
ToolCall(
|
|
||||||
function=FunctionCall(
|
|
||||||
name="get_user_location",
|
|
||||||
arguments=json.dumps({}),
|
|
||||||
)
|
|
||||||
),
|
|
||||||
]
|
|
||||||
assert_tool_calls(extracted_info.tool_calls, expected_tool_calls)
|
|
||||||
|
|
||||||
|
|
||||||
def test_extract_tool_calls_assistant_recipient_ignored(
|
|
||||||
openai_tool_parser,
|
|
||||||
harmony_encoding,
|
|
||||||
):
|
|
||||||
convo = Conversation.from_messages(
|
|
||||||
[
|
|
||||||
Message.from_role_and_content(Role.USER, "Hello"),
|
|
||||||
Message.from_role_and_content(Role.ASSISTANT, "Some tool response")
|
|
||||||
.with_channel("commentary")
|
|
||||||
.with_recipient("assistant"),
|
|
||||||
Message.from_role_and_content(
|
|
||||||
Role.ASSISTANT, "Here is the answer"
|
|
||||||
).with_channel("final"),
|
|
||||||
]
|
|
||||||
)
|
|
||||||
token_ids = harmony_encoding.render_conversation_for_completion(
|
|
||||||
convo, Role.ASSISTANT
|
|
||||||
)
|
|
||||||
|
|
||||||
extracted_info = openai_tool_parser.extract_tool_calls(
|
|
||||||
"",
|
|
||||||
request=None,
|
|
||||||
token_ids=token_ids,
|
|
||||||
)
|
|
||||||
assert not extracted_info.tools_called
|
|
||||||
assert extracted_info.tool_calls == []
|
|
||||||
assert extracted_info.content == "Here is the answer"
|
|
||||||
|
|
||||||
|
|
||||||
def test_extract_tool_calls_dotted_function_name(
|
|
||||||
openai_tool_parser,
|
|
||||||
harmony_encoding,
|
|
||||||
):
|
|
||||||
convo = Conversation.from_messages(
|
|
||||||
[
|
|
||||||
Message.from_role_and_content(Role.USER, "Compute 2+3"),
|
|
||||||
Message.from_role_and_content(Role.ASSISTANT, '{"a": 2, "b": 3}')
|
|
||||||
.with_channel("commentary")
|
|
||||||
.with_recipient("math.sum")
|
|
||||||
.with_content_type("json"),
|
|
||||||
]
|
|
||||||
)
|
|
||||||
token_ids = harmony_encoding.render_conversation_for_completion(
|
|
||||||
convo, Role.ASSISTANT
|
|
||||||
)
|
|
||||||
|
|
||||||
extracted_info = openai_tool_parser.extract_tool_calls(
|
|
||||||
"",
|
|
||||||
request=None,
|
|
||||||
token_ids=token_ids,
|
|
||||||
)
|
|
||||||
assert extracted_info.tools_called
|
|
||||||
expected_tool_calls = [
|
|
||||||
ToolCall(
|
|
||||||
function=FunctionCall(
|
|
||||||
name="math.sum",
|
|
||||||
arguments=json.dumps({"a": 2, "b": 3}),
|
|
||||||
)
|
|
||||||
)
|
|
||||||
]
|
|
||||||
assert_tool_calls(extracted_info.tool_calls, expected_tool_calls)
|
|
||||||
|
|
||||||
|
|
||||||
def test_extract_tool_calls_with_content(
|
|
||||||
openai_tool_parser,
|
|
||||||
harmony_encoding,
|
|
||||||
):
|
|
||||||
final_content = "This tool call will get the weather."
|
|
||||||
convo = Conversation.from_messages(
|
|
||||||
[
|
|
||||||
Message.from_role_and_content(
|
|
||||||
Role.USER, "What is the weather in Tokyo based on where I'm at?"
|
|
||||||
),
|
|
||||||
Message.from_role_and_content(
|
|
||||||
Role.ASSISTANT,
|
|
||||||
'User asks: "What is the weather in Tokyo?" based on their location. We need to use get_current_weather tool and get_user_location tool.', # noqa: E501
|
|
||||||
).with_channel("analysis"),
|
|
||||||
Message.from_role_and_content(Role.ASSISTANT, '{"location": "Tokyo"}')
|
|
||||||
.with_channel("commentary")
|
|
||||||
.with_recipient("functions.get_current_weather")
|
|
||||||
.with_content_type("json"),
|
|
||||||
Message.from_role_and_content(Role.ASSISTANT, final_content).with_channel(
|
|
||||||
"final"
|
|
||||||
),
|
|
||||||
]
|
|
||||||
)
|
|
||||||
token_ids = harmony_encoding.render_conversation_for_completion(
|
|
||||||
convo,
|
|
||||||
Role.ASSISTANT,
|
|
||||||
)
|
|
||||||
|
|
||||||
extracted_info = openai_tool_parser.extract_tool_calls(
|
|
||||||
"",
|
|
||||||
request=None,
|
|
||||||
token_ids=token_ids,
|
|
||||||
)
|
|
||||||
assert extracted_info.tools_called
|
|
||||||
expected_tool_calls = [
|
|
||||||
ToolCall(
|
|
||||||
function=FunctionCall(
|
|
||||||
name="get_current_weather",
|
|
||||||
arguments=json.dumps({"location": "Tokyo"}),
|
|
||||||
)
|
|
||||||
),
|
|
||||||
]
|
|
||||||
assert_tool_calls(extracted_info.tool_calls, expected_tool_calls)
|
|
||||||
assert extracted_info.content == final_content
|
|
||||||
@@ -54,7 +54,6 @@ from vllm.entrypoints.openai.engine.serving import (
|
|||||||
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
|
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
|
||||||
from vllm.entrypoints.openai.parser.harmony_utils import (
|
from vllm.entrypoints.openai.parser.harmony_utils import (
|
||||||
get_streamable_parser_for_assistant,
|
get_streamable_parser_for_assistant,
|
||||||
parse_chat_output,
|
|
||||||
)
|
)
|
||||||
from vllm.entrypoints.serve.utils.api_utils import get_max_tokens, should_include_usage
|
from vllm.entrypoints.serve.utils.api_utils import get_max_tokens, should_include_usage
|
||||||
from vllm.entrypoints.serve.utils.request_logger import RequestLogger
|
from vllm.entrypoints.serve.utils.request_logger import RequestLogger
|
||||||
@@ -135,6 +134,7 @@ class OpenAIServingChat(OpenAIServing):
|
|||||||
reasoning_parser_name=reasoning_parser,
|
reasoning_parser_name=reasoning_parser,
|
||||||
enable_auto_tools=enable_auto_tools,
|
enable_auto_tools=enable_auto_tools,
|
||||||
model_name=self.model_config.model,
|
model_name=self.model_config.model,
|
||||||
|
is_harmony=self.model_config.hf_config.model_type == "gpt_oss",
|
||||||
)
|
)
|
||||||
if (
|
if (
|
||||||
is_mistral_tool_parser(self.tool_parser)
|
is_mistral_tool_parser(self.tool_parser)
|
||||||
@@ -359,14 +359,6 @@ class OpenAIServingChat(OpenAIServing):
|
|||||||
assert len(generators) == 1
|
assert len(generators) == 1
|
||||||
(result_generator,) = generators
|
(result_generator,) = generators
|
||||||
|
|
||||||
parser: Parser | None = None
|
|
||||||
if self.parser_cls is not None:
|
|
||||||
parser = self.parser_cls(
|
|
||||||
tokenizer,
|
|
||||||
request.tools,
|
|
||||||
chat_template_kwargs=chat_template_kwargs,
|
|
||||||
)
|
|
||||||
|
|
||||||
if request.stream:
|
if request.stream:
|
||||||
return self.chat_completion_stream_generator(
|
return self.chat_completion_stream_generator(
|
||||||
request,
|
request,
|
||||||
@@ -387,7 +379,7 @@ class OpenAIServingChat(OpenAIServing):
|
|||||||
conversation,
|
conversation,
|
||||||
tokenizer,
|
tokenizer,
|
||||||
request_metadata,
|
request_metadata,
|
||||||
parser,
|
chat_template_kwargs=chat_template_kwargs,
|
||||||
)
|
)
|
||||||
|
|
||||||
def get_chat_request_role(self, request: ChatCompletionRequest) -> str:
|
def get_chat_request_role(self, request: ChatCompletionRequest) -> str:
|
||||||
@@ -840,7 +832,7 @@ class OpenAIServingChat(OpenAIServing):
|
|||||||
conversation: list[ConversationMessage],
|
conversation: list[ConversationMessage],
|
||||||
tokenizer: TokenizerLike,
|
tokenizer: TokenizerLike,
|
||||||
request_metadata: RequestResponseMetadata,
|
request_metadata: RequestResponseMetadata,
|
||||||
parser: Parser | None = None,
|
chat_template_kwargs: dict[str, Any] | None = None,
|
||||||
) -> ErrorResponse | ChatCompletionResponse:
|
) -> ErrorResponse | ChatCompletionResponse:
|
||||||
created_time = int(time.time())
|
created_time = int(time.time())
|
||||||
final_res: RequestOutput | None = None
|
final_res: RequestOutput | None = None
|
||||||
@@ -871,7 +863,6 @@ class OpenAIServingChat(OpenAIServing):
|
|||||||
self._raise_if_error(output.finish_reason, request_id)
|
self._raise_if_error(output.finish_reason, request_id)
|
||||||
token_ids = output.token_ids
|
token_ids = output.token_ids
|
||||||
out_logprobs = output.logprobs
|
out_logprobs = output.logprobs
|
||||||
tool_call_info = None
|
|
||||||
|
|
||||||
if request.logprobs and request.top_logprobs is not None:
|
if request.logprobs and request.top_logprobs is not None:
|
||||||
assert out_logprobs is not None, "Did not output logprobs"
|
assert out_logprobs is not None, "Did not output logprobs"
|
||||||
@@ -885,75 +876,20 @@ class OpenAIServingChat(OpenAIServing):
|
|||||||
else:
|
else:
|
||||||
logprobs = None
|
logprobs = None
|
||||||
|
|
||||||
if self.use_harmony:
|
parser: Parser | None = None
|
||||||
reasoning, content, _ = parse_chat_output(token_ids)
|
if self.parser_cls is not None:
|
||||||
if not request.include_reasoning:
|
parser = self.parser_cls(
|
||||||
reasoning = None
|
tokenizer,
|
||||||
|
request.tools,
|
||||||
if self.tool_parser is not None:
|
chat_template_kwargs=chat_template_kwargs,
|
||||||
if tokenizer is None:
|
|
||||||
raise ValueError(
|
|
||||||
"Tokenizer not available when `skip_tokenizer_init=True`"
|
|
||||||
)
|
|
||||||
|
|
||||||
tool_parser = self.tool_parser(tokenizer, request.tools)
|
|
||||||
# NOTE: We use token_ids for openai tool parser
|
|
||||||
tool_call_info = tool_parser.extract_tool_calls(
|
|
||||||
"",
|
|
||||||
request=request,
|
|
||||||
token_ids=token_ids, # type: ignore
|
|
||||||
)
|
|
||||||
content = tool_call_info.content
|
|
||||||
message = ChatMessage(
|
|
||||||
role=role,
|
|
||||||
reasoning=reasoning,
|
|
||||||
content=content,
|
|
||||||
tool_calls=tool_call_info.tool_calls,
|
|
||||||
)
|
|
||||||
else:
|
|
||||||
message = ChatMessage(
|
|
||||||
role=role,
|
|
||||||
reasoning=reasoning,
|
|
||||||
content=content,
|
|
||||||
)
|
|
||||||
|
|
||||||
# Encode routed_experts for transport. JSON can't carry raw
|
|
||||||
# bytes, so we write the ndarray as a ``.npy`` byte stream
|
|
||||||
# and base64-encode it. ``pybase64`` is ~3x faster than the
|
|
||||||
# stdlib ``base64`` on large payloads thanks to SIMD.
|
|
||||||
routed_experts_b64 = None
|
|
||||||
if output.routed_experts is not None:
|
|
||||||
buf = io.BytesIO()
|
|
||||||
np.save(buf, output.routed_experts)
|
|
||||||
routed_experts_b64 = base64.b64encode(buf.getvalue()).decode(
|
|
||||||
"ascii"
|
|
||||||
)
|
|
||||||
|
|
||||||
choice_data = ChatCompletionResponseChoice(
|
|
||||||
index=output.index,
|
|
||||||
message=message,
|
|
||||||
logprobs=logprobs,
|
|
||||||
finish_reason=(
|
|
||||||
"tool_calls"
|
|
||||||
if (tool_call_info is not None and tool_call_info.tools_called)
|
|
||||||
else output.finish_reason
|
|
||||||
if output.finish_reason
|
|
||||||
else "stop"
|
|
||||||
),
|
|
||||||
stop_reason=output.stop_reason,
|
|
||||||
token_ids=(
|
|
||||||
as_list(output.token_ids) if request.return_token_ids else None
|
|
||||||
),
|
|
||||||
routed_experts=routed_experts_b64,
|
|
||||||
)
|
)
|
||||||
choices.append(choice_data)
|
|
||||||
continue
|
|
||||||
|
|
||||||
if parser is not None:
|
if parser is not None:
|
||||||
reasoning, content, tool_calls = parser.parse(
|
reasoning, content, tool_calls = parser.parse(
|
||||||
output.text,
|
output.text,
|
||||||
request,
|
request,
|
||||||
enable_auto_tools=self.enable_auto_tools,
|
enable_auto_tools=self.enable_auto_tools,
|
||||||
|
model_output_token_ids=token_ids,
|
||||||
)
|
)
|
||||||
if not request.include_reasoning:
|
if not request.include_reasoning:
|
||||||
reasoning = None
|
reasoning = None
|
||||||
|
|||||||
@@ -2,7 +2,7 @@
|
|||||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||||
|
|
||||||
import datetime
|
import datetime
|
||||||
from collections.abc import Iterable, Sequence
|
from collections.abc import Sequence
|
||||||
from typing import Any
|
from typing import Any
|
||||||
|
|
||||||
from openai.types.responses.tool import Tool
|
from openai.types.responses.tool import Tool
|
||||||
@@ -456,65 +456,3 @@ def render_for_completion(messages: list[Message]) -> list[int]:
|
|||||||
|
|
||||||
def get_streamable_parser_for_assistant() -> StreamableParser:
|
def get_streamable_parser_for_assistant() -> StreamableParser:
|
||||||
return StreamableParser(get_encoding(), role=Role.ASSISTANT)
|
return StreamableParser(get_encoding(), role=Role.ASSISTANT)
|
||||||
|
|
||||||
|
|
||||||
def parse_output_into_messages(token_ids: Iterable[int]) -> StreamableParser:
|
|
||||||
parser = get_streamable_parser_for_assistant()
|
|
||||||
for token_id in token_ids:
|
|
||||||
parser.process(token_id)
|
|
||||||
return parser
|
|
||||||
|
|
||||||
|
|
||||||
def parse_chat_output(
|
|
||||||
token_ids: Sequence[int],
|
|
||||||
) -> tuple[str | None, str | None, bool]:
|
|
||||||
"""
|
|
||||||
Parse the output of a Harmony chat completion into reasoning and final content.
|
|
||||||
Note that when the `openai` tool parser is used, serving_chat only uses this
|
|
||||||
for the reasoning content and gets the final content from the tool call parser.
|
|
||||||
|
|
||||||
When the `openai` tool parser is not enabled, or when `GptOssReasoningParser` is
|
|
||||||
in use,this needs to return the final content without any tool calls parsed.
|
|
||||||
|
|
||||||
Empty reasoning or final content is returned as None instead of an empty string.
|
|
||||||
"""
|
|
||||||
parser = parse_output_into_messages(token_ids)
|
|
||||||
output_msgs = parser.messages
|
|
||||||
is_tool_call = False # TODO: update this when tool call is supported
|
|
||||||
|
|
||||||
# Get completed messages from the parser
|
|
||||||
# - analysis channel: hidden reasoning
|
|
||||||
# - commentary channel without recipient (preambles): visible to user
|
|
||||||
# - final channel: visible to user
|
|
||||||
# - commentary with recipient (tool calls): handled separately by tool parser
|
|
||||||
reasoning_texts = [
|
|
||||||
msg.content[0].text for msg in output_msgs if msg.channel == "analysis"
|
|
||||||
]
|
|
||||||
final_texts = [
|
|
||||||
msg.content[0].text
|
|
||||||
for msg in output_msgs
|
|
||||||
if msg.channel == "final" or (msg.channel == "commentary" and not msg.recipient)
|
|
||||||
]
|
|
||||||
|
|
||||||
# Extract partial messages from the parser
|
|
||||||
if parser.current_channel == "analysis" and parser.current_content:
|
|
||||||
reasoning_texts.append(parser.current_content)
|
|
||||||
elif parser.current_channel == "final" and parser.current_content:
|
|
||||||
final_texts.append(parser.current_content)
|
|
||||||
elif (
|
|
||||||
parser.current_channel == "commentary"
|
|
||||||
and not parser.current_recipient
|
|
||||||
and parser.current_content
|
|
||||||
):
|
|
||||||
# Preambles (commentary without recipient) are visible to user
|
|
||||||
final_texts.append(parser.current_content)
|
|
||||||
|
|
||||||
# Flatten multiple messages into a single string
|
|
||||||
reasoning: str | None = "\n".join(reasoning_texts)
|
|
||||||
final_content: str | None = "\n".join(final_texts)
|
|
||||||
|
|
||||||
# Return None instead of empty string since existing callers check for None
|
|
||||||
reasoning = reasoning or None
|
|
||||||
final_content = final_content or None
|
|
||||||
|
|
||||||
return reasoning, final_content, is_tool_call
|
|
||||||
|
|||||||
@@ -191,6 +191,7 @@ class OpenAIServingResponses(OpenAIServing):
|
|||||||
reasoning_parser_name=reasoning_parser,
|
reasoning_parser_name=reasoning_parser,
|
||||||
enable_auto_tools=enable_auto_tools,
|
enable_auto_tools=enable_auto_tools,
|
||||||
model_name=self.model_config.model,
|
model_name=self.model_config.model,
|
||||||
|
is_harmony=self.model_config.hf_config.model_type == "gpt_oss",
|
||||||
)
|
)
|
||||||
self.enable_prompt_tokens_details = enable_prompt_tokens_details
|
self.enable_prompt_tokens_details = enable_prompt_tokens_details
|
||||||
self.enable_force_include_usage = enable_force_include_usage
|
self.enable_force_include_usage = enable_force_include_usage
|
||||||
|
|||||||
@@ -5,10 +5,12 @@ from vllm.parser.abstract_parser import (
|
|||||||
DelegatingParser,
|
DelegatingParser,
|
||||||
Parser,
|
Parser,
|
||||||
)
|
)
|
||||||
|
from vllm.parser.harmony import HarmonyParser
|
||||||
from vllm.parser.parser_manager import ParserManager
|
from vllm.parser.parser_manager import ParserManager
|
||||||
|
|
||||||
__all__ = [
|
__all__ = [
|
||||||
"Parser",
|
"Parser",
|
||||||
"DelegatingParser",
|
"DelegatingParser",
|
||||||
|
"HarmonyParser",
|
||||||
"ParserManager",
|
"ParserManager",
|
||||||
]
|
]
|
||||||
|
|||||||
@@ -282,6 +282,7 @@ class Parser:
|
|||||||
model_output: str,
|
model_output: str,
|
||||||
request: ChatCompletionRequest | ResponsesRequest,
|
request: ChatCompletionRequest | ResponsesRequest,
|
||||||
enable_auto_tools: bool = False,
|
enable_auto_tools: bool = False,
|
||||||
|
model_output_token_ids: Sequence[int] = (),
|
||||||
) -> tuple[str | None, str | None, list[FunctionCall] | None]:
|
) -> tuple[str | None, str | None, list[FunctionCall] | None]:
|
||||||
"""Parse a complete model output, extracting reasoning and tool calls.
|
"""Parse a complete model output, extracting reasoning and tool calls.
|
||||||
|
|
||||||
@@ -289,6 +290,7 @@ class Parser:
|
|||||||
model_output: The complete model-generated string.
|
model_output: The complete model-generated string.
|
||||||
request: The request object used to generate the output.
|
request: The request object used to generate the output.
|
||||||
enable_auto_tools: Whether to enable automatic tool call parsing.
|
enable_auto_tools: Whether to enable automatic tool call parsing.
|
||||||
|
model_output_token_ids: The generated raw output token IDs.
|
||||||
|
|
||||||
Returns:
|
Returns:
|
||||||
A tuple of (reasoning, content, tool_calls).
|
A tuple of (reasoning, content, tool_calls).
|
||||||
@@ -642,6 +644,7 @@ class DelegatingParser(Parser):
|
|||||||
model_output: str,
|
model_output: str,
|
||||||
request: ChatCompletionRequest | ResponsesRequest,
|
request: ChatCompletionRequest | ResponsesRequest,
|
||||||
enable_auto_tools: bool = False,
|
enable_auto_tools: bool = False,
|
||||||
|
model_output_token_ids: Sequence[int] = (),
|
||||||
) -> tuple[str | None, str | None, list[FunctionCall] | None]:
|
) -> tuple[str | None, str | None, list[FunctionCall] | None]:
|
||||||
reasoning, content = self.extract_reasoning(model_output, request)
|
reasoning, content = self.extract_reasoning(model_output, request)
|
||||||
tool_calls, content = self._extract_tool_calls(
|
tool_calls, content = self._extract_tool_calls(
|
||||||
|
|||||||
@@ -0,0 +1,240 @@
|
|||||||
|
# SPDX-License-Identifier: Apache-2.0
|
||||||
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import json
|
||||||
|
from collections.abc import Sequence
|
||||||
|
from dataclasses import dataclass
|
||||||
|
from enum import Enum, auto
|
||||||
|
from typing import TYPE_CHECKING, NamedTuple
|
||||||
|
|
||||||
|
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
|
||||||
|
from vllm.entrypoints.openai.engine.protocol import (
|
||||||
|
DeltaMessage,
|
||||||
|
FunctionCall,
|
||||||
|
)
|
||||||
|
from vllm.entrypoints.openai.parser.harmony_utils import (
|
||||||
|
extract_function_from_recipient,
|
||||||
|
get_streamable_parser_for_assistant,
|
||||||
|
is_function_recipient,
|
||||||
|
)
|
||||||
|
from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
|
||||||
|
from vllm.parser.abstract_parser import DelegatingParser
|
||||||
|
from vllm.reasoning.gptoss_reasoning_parser import GptOssReasoningParser
|
||||||
|
from vllm.tool_parsers.gptoss_tool_parser import GptOssToolParser
|
||||||
|
|
||||||
|
if TYPE_CHECKING:
|
||||||
|
from openai_harmony import Message, Role
|
||||||
|
from openai_harmony import StreamState as HarmonyStreamState
|
||||||
|
|
||||||
|
|
||||||
|
class _SegmentType(Enum):
|
||||||
|
TOOL = auto()
|
||||||
|
REASONING = auto()
|
||||||
|
CONTENT = auto()
|
||||||
|
IGNORE = auto()
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def from_channel_and_recipient(
|
||||||
|
channel: str | None, recipient: str | None
|
||||||
|
) -> _SegmentType:
|
||||||
|
if recipient and is_function_recipient(recipient):
|
||||||
|
return _SegmentType.TOOL
|
||||||
|
if channel == "analysis":
|
||||||
|
return _SegmentType.REASONING
|
||||||
|
if channel == "final" or (channel == "commentary" and recipient is None):
|
||||||
|
return _SegmentType.CONTENT
|
||||||
|
return _SegmentType.IGNORE
|
||||||
|
|
||||||
|
|
||||||
|
class Segment(NamedTuple):
|
||||||
|
channel: str | None
|
||||||
|
recipient: str | None
|
||||||
|
delta: str
|
||||||
|
is_boundary: bool = False
|
||||||
|
completed_message: Message | None = None
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass
|
||||||
|
class ChunkResult:
|
||||||
|
segments: list[Segment]
|
||||||
|
reasoning_token_count: int
|
||||||
|
|
||||||
|
|
||||||
|
class HarmonyParser(DelegatingParser):
|
||||||
|
def __init__(self, tokenizer, tools=None, *args, **kwargs):
|
||||||
|
super().__init__(tokenizer, tools, *args, **kwargs)
|
||||||
|
|
||||||
|
if self._reasoning_parser and not isinstance(
|
||||||
|
self._reasoning_parser, GptOssReasoningParser
|
||||||
|
):
|
||||||
|
raise ValueError(
|
||||||
|
"Harmony requires GptOssReasoningParser, "
|
||||||
|
f"got {self._reasoning_parser.__class__.__name__}."
|
||||||
|
)
|
||||||
|
|
||||||
|
if self._tool_parser and not isinstance(self._tool_parser, GptOssToolParser):
|
||||||
|
raise ValueError(
|
||||||
|
"Harmony requires GptOssToolParser, "
|
||||||
|
f"got {self._tool_parser.__class__.__name__}."
|
||||||
|
)
|
||||||
|
|
||||||
|
self._harmony_parser = get_streamable_parser_for_assistant()
|
||||||
|
|
||||||
|
@property
|
||||||
|
def messages(self) -> list[Message]:
|
||||||
|
return self._harmony_parser.messages
|
||||||
|
|
||||||
|
@property
|
||||||
|
def state(self) -> HarmonyStreamState:
|
||||||
|
return self._harmony_parser.state
|
||||||
|
|
||||||
|
@property
|
||||||
|
def current_role(self) -> Role | None:
|
||||||
|
return self._harmony_parser.current_role
|
||||||
|
|
||||||
|
@property
|
||||||
|
def current_channel(self) -> str | None:
|
||||||
|
return self._harmony_parser.current_channel
|
||||||
|
|
||||||
|
@property
|
||||||
|
def current_recipient(self) -> str | None:
|
||||||
|
return self._harmony_parser.current_recipient
|
||||||
|
|
||||||
|
@property
|
||||||
|
def current_content(self) -> str:
|
||||||
|
return self._harmony_parser.current_content
|
||||||
|
|
||||||
|
@property
|
||||||
|
def current_content_type(self) -> str | None:
|
||||||
|
return self._harmony_parser.current_content_type
|
||||||
|
|
||||||
|
def parse(
|
||||||
|
self,
|
||||||
|
model_output: str,
|
||||||
|
request: ChatCompletionRequest | ResponsesRequest,
|
||||||
|
enable_auto_tools: bool = False,
|
||||||
|
model_output_token_ids: Sequence[int] = (),
|
||||||
|
) -> tuple[str | None, str | None, list[FunctionCall] | None]:
|
||||||
|
"""Parse Harmony output from token IDs.
|
||||||
|
|
||||||
|
Tool calls are always extracted regardless of ``enable_auto_tools``.
|
||||||
|
Callers must decide whether to surface them.
|
||||||
|
"""
|
||||||
|
result = self.process_chunk(model_output_token_ids)
|
||||||
|
|
||||||
|
reasoning_parts: list[str] = []
|
||||||
|
content_parts: list[str] = []
|
||||||
|
tool_calls: list[FunctionCall] = []
|
||||||
|
|
||||||
|
def _append_parsed_message(
|
||||||
|
channel: str | None,
|
||||||
|
recipient: str | None,
|
||||||
|
text: str,
|
||||||
|
content_type: str | None = None,
|
||||||
|
) -> None:
|
||||||
|
segment_type = _SegmentType.from_channel_and_recipient(channel, recipient)
|
||||||
|
match segment_type:
|
||||||
|
case _SegmentType.REASONING if self.reasoning_parser and text:
|
||||||
|
reasoning_parts.append(text)
|
||||||
|
case _SegmentType.CONTENT if text:
|
||||||
|
content_parts.append(text)
|
||||||
|
case _SegmentType.TOOL if self.tool_parser:
|
||||||
|
assert recipient is not None
|
||||||
|
if content_type is not None and "json" not in content_type:
|
||||||
|
arguments = text
|
||||||
|
else:
|
||||||
|
try:
|
||||||
|
arguments = json.dumps(json.loads(text))
|
||||||
|
except json.JSONDecodeError:
|
||||||
|
arguments = text
|
||||||
|
tool_calls.append(
|
||||||
|
FunctionCall(
|
||||||
|
name=extract_function_from_recipient(recipient),
|
||||||
|
arguments=arguments,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
for segment in result.segments:
|
||||||
|
msg = segment.completed_message
|
||||||
|
if msg is None:
|
||||||
|
continue
|
||||||
|
if msg.author.role != "assistant" or not msg.content:
|
||||||
|
continue
|
||||||
|
_append_parsed_message(
|
||||||
|
channel=msg.channel,
|
||||||
|
recipient=msg.recipient,
|
||||||
|
text=msg.content[0].text,
|
||||||
|
content_type=msg.content_type,
|
||||||
|
)
|
||||||
|
|
||||||
|
if (
|
||||||
|
self.current_channel is not None
|
||||||
|
or self.current_recipient is not None
|
||||||
|
or self.current_content
|
||||||
|
):
|
||||||
|
_append_parsed_message(
|
||||||
|
channel=self.current_channel,
|
||||||
|
recipient=self.current_recipient,
|
||||||
|
text=self.current_content,
|
||||||
|
content_type=self.current_content_type,
|
||||||
|
)
|
||||||
|
|
||||||
|
reasoning = "\n".join(reasoning_parts) or None
|
||||||
|
content = "\n".join(content_parts) or None
|
||||||
|
return reasoning, content, tool_calls or None
|
||||||
|
|
||||||
|
def parse_delta(
|
||||||
|
self,
|
||||||
|
delta_text: str,
|
||||||
|
delta_token_ids: list[int],
|
||||||
|
request: ChatCompletionRequest | ResponsesRequest,
|
||||||
|
prompt_token_ids: list[int] | None = None,
|
||||||
|
*,
|
||||||
|
finished: bool,
|
||||||
|
) -> DeltaMessage | None:
|
||||||
|
raise NotImplementedError(
|
||||||
|
"HarmonyParser streaming parsing is deferred. "
|
||||||
|
"Use the existing harmony streaming path."
|
||||||
|
)
|
||||||
|
|
||||||
|
def process_chunk(self, token_ids: Sequence[int]) -> ChunkResult:
|
||||||
|
if not token_ids:
|
||||||
|
return ChunkResult(segments=[], reasoning_token_count=0)
|
||||||
|
|
||||||
|
from openai_harmony import StreamState
|
||||||
|
|
||||||
|
segments: list[Segment] = []
|
||||||
|
reasoning_token_count = 0
|
||||||
|
for token_id in token_ids:
|
||||||
|
self._harmony_parser.process(token_id)
|
||||||
|
channel = self.current_channel
|
||||||
|
recipient = self.current_recipient
|
||||||
|
delta = self._harmony_parser.last_content_delta or ""
|
||||||
|
completed_message = None
|
||||||
|
is_boundary = self.state == StreamState.EXPECT_START
|
||||||
|
if is_boundary and self.messages:
|
||||||
|
completed_message = self.messages[-1]
|
||||||
|
|
||||||
|
if channel == "analysis" or (
|
||||||
|
channel == "commentary" and recipient is not None
|
||||||
|
):
|
||||||
|
reasoning_token_count += 1
|
||||||
|
|
||||||
|
segments.append(
|
||||||
|
Segment(
|
||||||
|
channel=channel,
|
||||||
|
recipient=recipient,
|
||||||
|
delta=delta,
|
||||||
|
is_boundary=is_boundary,
|
||||||
|
completed_message=completed_message,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
# TODO: Optionally merge and suppress empty Segments
|
||||||
|
|
||||||
|
return ChunkResult(
|
||||||
|
segments=segments,
|
||||||
|
reasoning_token_count=reasoning_token_count,
|
||||||
|
)
|
||||||
@@ -3,6 +3,7 @@
|
|||||||
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from collections.abc import Sequence
|
||||||
from typing import TYPE_CHECKING
|
from typing import TYPE_CHECKING
|
||||||
|
|
||||||
from vllm.entrypoints.openai.engine.protocol import DeltaMessage, FunctionCall
|
from vllm.entrypoints.openai.engine.protocol import DeltaMessage, FunctionCall
|
||||||
@@ -43,10 +44,14 @@ class MistralParser(DelegatingParser):
|
|||||||
model_output: str,
|
model_output: str,
|
||||||
request: ChatCompletionRequest | ResponsesRequest,
|
request: ChatCompletionRequest | ResponsesRequest,
|
||||||
enable_auto_tools: bool = False,
|
enable_auto_tools: bool = False,
|
||||||
|
model_output_token_ids: Sequence[int] = (),
|
||||||
) -> tuple[str | None, str | None, list[FunctionCall] | None]:
|
) -> tuple[str | None, str | None, list[FunctionCall] | None]:
|
||||||
self._maybe_force_auto_tool_parsing(request)
|
self._maybe_force_auto_tool_parsing(request)
|
||||||
reasoning, content, tool_calls = super().parse(
|
reasoning, content, tool_calls = super().parse(
|
||||||
model_output, request, enable_auto_tools
|
model_output,
|
||||||
|
request,
|
||||||
|
enable_auto_tools,
|
||||||
|
model_output_token_ids,
|
||||||
)
|
)
|
||||||
if tool_calls:
|
if tool_calls:
|
||||||
from vllm.tool_parsers.mistral_tool_parser import MistralToolCall
|
from vllm.tool_parsers.mistral_tool_parser import MistralToolCall
|
||||||
|
|||||||
@@ -79,6 +79,7 @@ class ParserManager:
|
|||||||
reasoning_parser_name: str | None = None,
|
reasoning_parser_name: str | None = None,
|
||||||
enable_auto_tools: bool = False,
|
enable_auto_tools: bool = False,
|
||||||
model_name: str | None = None,
|
model_name: str | None = None,
|
||||||
|
is_harmony: bool = False,
|
||||||
) -> type[Parser] | None:
|
) -> type[Parser] | None:
|
||||||
"""
|
"""
|
||||||
Get a Parser that handles both reasoning and tool parsing.
|
Get a Parser that handles both reasoning and tool parsing.
|
||||||
@@ -91,6 +92,8 @@ class ParserManager:
|
|||||||
reasoning_parser_name: The name of the reasoning parser.
|
reasoning_parser_name: The name of the reasoning parser.
|
||||||
enable_auto_tools: Whether auto tool choice is enabled.
|
enable_auto_tools: Whether auto tool choice is enabled.
|
||||||
model_name: The model name for parser-specific warnings.
|
model_name: The model name for parser-specific warnings.
|
||||||
|
is_harmony: Whether the selected model uses the Harmony format.
|
||||||
|
If True, HarmonyParser is always returned.
|
||||||
|
|
||||||
Returns:
|
Returns:
|
||||||
A Parser class, or None if neither parser is specified.
|
A Parser class, or None if neither parser is specified.
|
||||||
@@ -108,6 +111,13 @@ class ParserManager:
|
|||||||
|
|
||||||
from vllm.utils.mistral import is_mistral_tool_parser
|
from vllm.utils.mistral import is_mistral_tool_parser
|
||||||
|
|
||||||
|
if is_harmony:
|
||||||
|
from vllm.parser.harmony import HarmonyParser
|
||||||
|
|
||||||
|
HarmonyParser.reasoning_parser_cls = reasoning_parser_cls
|
||||||
|
HarmonyParser.tool_parser_cls = tool_parser_cls
|
||||||
|
return HarmonyParser
|
||||||
|
|
||||||
if is_mistral_tool_parser(tool_parser_cls):
|
if is_mistral_tool_parser(tool_parser_cls):
|
||||||
from vllm.parser.mistral import MistralParser
|
from vllm.parser.mistral import MistralParser
|
||||||
|
|
||||||
|
|||||||
@@ -8,7 +8,6 @@ from transformers import PreTrainedTokenizerBase
|
|||||||
|
|
||||||
from vllm.entrypoints.mcp.tool_server import ToolServer
|
from vllm.entrypoints.mcp.tool_server import ToolServer
|
||||||
from vllm.entrypoints.openai.engine.protocol import DeltaMessage
|
from vllm.entrypoints.openai.engine.protocol import DeltaMessage
|
||||||
from vllm.entrypoints.openai.parser.harmony_utils import parse_chat_output
|
|
||||||
from vllm.logger import init_logger
|
from vllm.logger import init_logger
|
||||||
from vllm.reasoning import ReasoningParser
|
from vllm.reasoning import ReasoningParser
|
||||||
|
|
||||||
@@ -132,10 +131,10 @@ class GptOssReasoningParser(ReasoningParser):
|
|||||||
return self.is_reasoning_end(input_ids[n - window :])
|
return self.is_reasoning_end(input_ids[n - window :])
|
||||||
|
|
||||||
def extract_content_ids(self, input_ids: list[int]) -> list[int]:
|
def extract_content_ids(self, input_ids: list[int]) -> list[int]:
|
||||||
_, content, _ = parse_chat_output(input_ids)
|
raise NotImplementedError(
|
||||||
if content is None:
|
"GptOssReasoningParser only provides boundary detection. "
|
||||||
return []
|
"Use HarmonyParser for output parsing."
|
||||||
return self.model_tokenizer.encode(content)
|
)
|
||||||
|
|
||||||
def extract_reasoning_streaming(
|
def extract_reasoning_streaming(
|
||||||
self,
|
self,
|
||||||
@@ -146,25 +145,10 @@ class GptOssReasoningParser(ReasoningParser):
|
|||||||
current_token_ids: Sequence[int],
|
current_token_ids: Sequence[int],
|
||||||
delta_token_ids: Sequence[int],
|
delta_token_ids: Sequence[int],
|
||||||
) -> DeltaMessage | None:
|
) -> DeltaMessage | None:
|
||||||
prev_reasoning, prev_content, _ = parse_chat_output(list(previous_token_ids))
|
raise NotImplementedError(
|
||||||
cur_reasoning, cur_content, _ = parse_chat_output(list(current_token_ids))
|
"GptOssReasoningParser only provides boundary detection. "
|
||||||
reasoning_delta = None
|
"Use HarmonyParser for output parsing."
|
||||||
content_delta = None
|
)
|
||||||
if cur_reasoning is not None:
|
|
||||||
prev_r = prev_reasoning or ""
|
|
||||||
if cur_reasoning.startswith(prev_r):
|
|
||||||
reasoning_delta = cur_reasoning[len(prev_r) :] or None
|
|
||||||
else:
|
|
||||||
reasoning_delta = cur_reasoning
|
|
||||||
if cur_content is not None:
|
|
||||||
prev_c = prev_content or ""
|
|
||||||
if cur_content.startswith(prev_c):
|
|
||||||
content_delta = cur_content[len(prev_c) :] or None
|
|
||||||
else:
|
|
||||||
content_delta = cur_content
|
|
||||||
if reasoning_delta is None and content_delta is None:
|
|
||||||
return None
|
|
||||||
return DeltaMessage(reasoning=reasoning_delta, content=content_delta)
|
|
||||||
|
|
||||||
def extract_reasoning(
|
def extract_reasoning(
|
||||||
self,
|
self,
|
||||||
@@ -172,7 +156,8 @@ class GptOssReasoningParser(ReasoningParser):
|
|||||||
request: "ChatCompletionRequest | ResponsesRequest",
|
request: "ChatCompletionRequest | ResponsesRequest",
|
||||||
) -> tuple[str | None, str | None]:
|
) -> tuple[str | None, str | None]:
|
||||||
raise NotImplementedError(
|
raise NotImplementedError(
|
||||||
"gpt-oss has a special branch for parsing reasoning in non-streaming mode. This method shouldn't be used." # noqa: E501
|
"GptOssReasoningParser only provides boundary detection. "
|
||||||
|
"Use HarmonyParser for output parsing."
|
||||||
)
|
)
|
||||||
|
|
||||||
# This function prepares the structural tag to format reasoning output
|
# This function prepares the structural tag to format reasoning output
|
||||||
|
|||||||
@@ -143,8 +143,8 @@ _TOOL_PARSERS_TO_REGISTER = {
|
|||||||
"Olmo3PythonicToolParser",
|
"Olmo3PythonicToolParser",
|
||||||
),
|
),
|
||||||
"openai": (
|
"openai": (
|
||||||
"openai_tool_parser",
|
"gptoss_tool_parser",
|
||||||
"OpenAIToolParser",
|
"GptOssToolParser",
|
||||||
),
|
),
|
||||||
"phi4_mini_json": (
|
"phi4_mini_json": (
|
||||||
"phi4mini_tool_parser",
|
"phi4mini_tool_parser",
|
||||||
|
|||||||
@@ -0,0 +1,47 @@
|
|||||||
|
# SPDX-License-Identifier: Apache-2.0
|
||||||
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||||
|
|
||||||
|
from collections.abc import Sequence
|
||||||
|
from typing import TYPE_CHECKING
|
||||||
|
|
||||||
|
from vllm.entrypoints.openai.engine.protocol import (
|
||||||
|
DeltaMessage,
|
||||||
|
ExtractedToolCallInformation,
|
||||||
|
)
|
||||||
|
from vllm.tool_parsers.abstract_tool_parser import Tool, ToolParser
|
||||||
|
|
||||||
|
if TYPE_CHECKING:
|
||||||
|
from vllm.tokenizers import TokenizerLike
|
||||||
|
|
||||||
|
|
||||||
|
class GptOssToolParser(ToolParser):
|
||||||
|
"""
|
||||||
|
Stub tool parser for gpt-oss/harmony models.
|
||||||
|
|
||||||
|
All output parsing is handled by HarmonyParser. This stub exists as a
|
||||||
|
capability declaration via HarmonyParser.tool_parser_cls.
|
||||||
|
"""
|
||||||
|
|
||||||
|
def __init__(self, tokenizer: "TokenizerLike", tools: list[Tool] | None = None):
|
||||||
|
super().__init__(tokenizer, tools)
|
||||||
|
|
||||||
|
def extract_tool_calls(
|
||||||
|
self, model_output, request, **kwargs
|
||||||
|
) -> ExtractedToolCallInformation:
|
||||||
|
raise NotImplementedError(
|
||||||
|
"GptOssToolParser is a stub. Use HarmonyParser for tool parsing."
|
||||||
|
)
|
||||||
|
|
||||||
|
def extract_tool_calls_streaming(
|
||||||
|
self,
|
||||||
|
previous_text: str,
|
||||||
|
current_text: str,
|
||||||
|
delta_text: str,
|
||||||
|
previous_token_ids: Sequence[int],
|
||||||
|
current_token_ids: Sequence[int],
|
||||||
|
delta_token_ids: Sequence[int],
|
||||||
|
request,
|
||||||
|
) -> DeltaMessage | None:
|
||||||
|
raise NotImplementedError(
|
||||||
|
"GptOssToolParser is a stub. Use HarmonyParser for tool parsing."
|
||||||
|
)
|
||||||
@@ -1,120 +0,0 @@
|
|||||||
# SPDX-License-Identifier: Apache-2.0
|
|
||||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
|
||||||
import json
|
|
||||||
from collections.abc import Sequence
|
|
||||||
from typing import TYPE_CHECKING
|
|
||||||
|
|
||||||
from vllm.entrypoints.openai.chat_completion.protocol import (
|
|
||||||
ChatCompletionRequest,
|
|
||||||
)
|
|
||||||
from vllm.entrypoints.openai.engine.protocol import (
|
|
||||||
DeltaMessage,
|
|
||||||
ExtractedToolCallInformation,
|
|
||||||
FunctionCall,
|
|
||||||
ToolCall,
|
|
||||||
)
|
|
||||||
from vllm.entrypoints.openai.parser.harmony_utils import (
|
|
||||||
extract_function_from_recipient,
|
|
||||||
is_function_recipient,
|
|
||||||
parse_output_into_messages,
|
|
||||||
)
|
|
||||||
from vllm.logger import init_logger
|
|
||||||
from vllm.tool_parsers.abstract_tool_parser import (
|
|
||||||
Tool,
|
|
||||||
ToolParser,
|
|
||||||
)
|
|
||||||
|
|
||||||
if TYPE_CHECKING:
|
|
||||||
from vllm.tokenizers import TokenizerLike
|
|
||||||
else:
|
|
||||||
TokenizerLike = object
|
|
||||||
|
|
||||||
logger = init_logger(__name__)
|
|
||||||
|
|
||||||
|
|
||||||
class OpenAIToolParser(ToolParser):
|
|
||||||
def __init__(self, tokenizer: "TokenizerLike", tools: list[Tool] | None = None):
|
|
||||||
super().__init__(tokenizer, tools)
|
|
||||||
|
|
||||||
def extract_tool_calls(
|
|
||||||
self,
|
|
||||||
model_output: str,
|
|
||||||
request: ChatCompletionRequest,
|
|
||||||
token_ids: Sequence[int] | None = None,
|
|
||||||
) -> ExtractedToolCallInformation:
|
|
||||||
if token_ids is None:
|
|
||||||
raise NotImplementedError(
|
|
||||||
"OpenAIToolParser requires token IDs and does not support text-based extraction." # noqa: E501
|
|
||||||
)
|
|
||||||
|
|
||||||
parser = parse_output_into_messages(token_ids)
|
|
||||||
tool_calls = []
|
|
||||||
final_content = None
|
|
||||||
commentary_content = None
|
|
||||||
|
|
||||||
if len(parser.messages) > 0:
|
|
||||||
for msg in parser.messages:
|
|
||||||
if msg.author.role != "assistant":
|
|
||||||
continue
|
|
||||||
if len(msg.content) < 1:
|
|
||||||
continue
|
|
||||||
msg_text = msg.content[0].text
|
|
||||||
if msg.recipient and is_function_recipient(msg.recipient):
|
|
||||||
# If no content-type is given assume JSON, as that's the
|
|
||||||
# most common case with gpt-oss models.
|
|
||||||
if not msg.content_type or "json" in msg.content_type:
|
|
||||||
# load and dump the JSON text to check validity and
|
|
||||||
# remove any extra newlines or other odd formatting
|
|
||||||
try:
|
|
||||||
tool_args = json.dumps(json.loads(msg_text))
|
|
||||||
except json.JSONDecodeError:
|
|
||||||
logger.exception(
|
|
||||||
"Error decoding JSON tool call from response."
|
|
||||||
)
|
|
||||||
tool_args = msg_text
|
|
||||||
else:
|
|
||||||
tool_args = msg_text
|
|
||||||
tool_calls.append(
|
|
||||||
ToolCall(
|
|
||||||
type="function",
|
|
||||||
function=FunctionCall(
|
|
||||||
name=extract_function_from_recipient(msg.recipient),
|
|
||||||
arguments=tool_args,
|
|
||||||
),
|
|
||||||
)
|
|
||||||
)
|
|
||||||
elif msg.channel == "final":
|
|
||||||
final_content = msg_text
|
|
||||||
elif msg.channel == "commentary" and not msg.recipient:
|
|
||||||
commentary_content = msg_text
|
|
||||||
|
|
||||||
# Extract partial content from the parser state if the generation was truncated
|
|
||||||
if parser.current_content:
|
|
||||||
if parser.current_channel == "final":
|
|
||||||
final_content = parser.current_content
|
|
||||||
elif (
|
|
||||||
parser.current_channel == "commentary" and not parser.current_recipient
|
|
||||||
):
|
|
||||||
commentary_content = parser.current_content
|
|
||||||
|
|
||||||
return ExtractedToolCallInformation(
|
|
||||||
tools_called=len(tool_calls) > 0,
|
|
||||||
tool_calls=tool_calls,
|
|
||||||
# prefer final content over commentary content if both are present
|
|
||||||
# commentary content is tool call preambles meant to be shown to the user
|
|
||||||
content=final_content or commentary_content,
|
|
||||||
)
|
|
||||||
|
|
||||||
def extract_tool_calls_streaming(
|
|
||||||
self,
|
|
||||||
previous_text: str,
|
|
||||||
current_text: str,
|
|
||||||
delta_text: str,
|
|
||||||
previous_token_ids: Sequence[int],
|
|
||||||
current_token_ids: Sequence[int],
|
|
||||||
delta_token_ids: Sequence[int],
|
|
||||||
request: ChatCompletionRequest,
|
|
||||||
) -> DeltaMessage | None:
|
|
||||||
raise NotImplementedError(
|
|
||||||
"Not being used, manual parsing in serving_chat.py" # noqa: E501
|
|
||||||
)
|
|
||||||
Reference in New Issue
Block a user