iHore/vLLM
8
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

[Refactor] Chat Completions Harmony Refactor, non-streaming path. (#45171)

Browse Source 
Signed-off-by: Yifan Zong <yzong@redhat.com>
This commit is contained in:
yzong-rh
2026-06-11 21:18:30 +00:00
committed by GitHub
parent 5a6c7b7ab5
commit f712fd0d7d
16 changed files with 822 additions and 816 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/entrypoints/openai/chat_completion/test_serving_chat.py
+38 -10
View File
@@ -38,12 +38,12 @@ from vllm.entrypoints.serve.render.serving import OpenAIServingRender
from vllm.exceptions import VLLMValidationError
from vllm.inputs import TokensPrompt
from vllm.outputs import CompletionOutput, RequestOutput
from vllm.parser import HarmonyParser
from vllm.renderers.hf import HfRenderer
from vllm.renderers.mistral import MistralRenderer
from vllm.tokenizers import get_tokenizer
from vllm.tokenizers.mistral import MistralTokenizer
from vllm.tokenizers.registry import cached_tokenizer_from_config
from vllm.tool_parsers import ToolParserManager
from vllm.v1.engine.async_llm import AsyncLLM
GPT_OSS_MODEL_NAME = "openai/gpt-oss-20b"
@@ -575,7 +575,13 @@ def _build_serving_render(
)
def _build_serving_chat(engine: AsyncLLM) -> OpenAIServingChat:
def _build_serving_chat(
engine: AsyncLLM,
*,
reasoning_parser: str = "",
tool_parser: str | None = None,
enable_auto_tools: bool = False,
) -> OpenAIServingChat:
models = OpenAIServingModels(
engine_client=engine,
base_model_paths=BASE_MODEL_PATHS,
@@ -590,6 +596,9 @@ def _build_serving_chat(engine: AsyncLLM) -> OpenAIServingChat:
chat_template=CHAT_TEMPLATE,
chat_template_content_format="auto",
request_logger=None,
reasoning_parser=reasoning_parser,
tool_parser=tool_parser,
enable_auto_tools=enable_auto_tools,
)
return serving_chat
@@ -637,7 +646,7 @@ async def test_serving_chat_returns_correct_model_name():
serving_chat = _build_serving_chat(mock_engine)
messages = [{"role": "user", "content": "what is 1+1?"}]
async def return_model_name(*args):
async def return_model_name(*args, **kwargs):
return args[3]
serving_chat.chat_completion_full_generator = return_model_name
@@ -1210,15 +1219,21 @@ class TestServingChatWithHarmony:
mock_engine = MagicMock(spec=AsyncLLM)
mock_engine.errored = False
mock_engine.model_config = MockModelConfig()
mock_engine.model_config.hf_config = MockHFConfig(model_type="gpt_oss")
mock_engine.model_config.hf_text_config = MockHFConfig(model_type="gpt_oss")
mock_engine.input_processor = MagicMock()
mock_engine.renderer = _build_renderer(mock_engine.model_config)
return mock_engine
@pytest.fixture()
def serving_chat(self, mock_engine) -> OpenAIServingChat:
chat = _build_serving_chat(mock_engine)
chat.use_harmony = True
chat.tool_parser = ToolParserManager.get_tool_parser("openai")
chat = _build_serving_chat(
mock_engine,
reasoning_parser="openai_gptoss",
tool_parser="openai",
enable_auto_tools=True,
)
assert chat.parser_cls is HarmonyParser
return chat
def mock_request_output_from_req_and_token_ids(
@@ -1277,6 +1292,7 @@ class TestServingChatWithHarmony:
stream: bool = False,
) -> ChatCompletionResponse:
harmony_token_ids = get_encoding().encode(harmony_str, allowed_special="all")
tokenizer = get_tokenizer(GPT_OSS_MODEL_NAME)
async def result_generator():
if stream:
@@ -1304,11 +1320,12 @@ class TestServingChatWithHarmony:
request_id=req.request_id,
model_name=req.model,
conversation=[],
tokenizer=get_tokenizer(req.model),
tokenizer=tokenizer,
request_metadata=RequestResponseMetadata(
request_id=req.request_id,
model_name=req.model,
),
chat_template_kwargs=serving_chat._effective_chat_template_kwargs(req),
)
if stream:
@@ -1316,11 +1333,18 @@ class TestServingChatWithHarmony:
return await result
@pytest.mark.asyncio
async def test_simple_chat(self, serving_chat, stream):
@pytest.mark.parametrize(
"include_reasoning", [True, False], ids=["with_reasoning", "no_reasoning"]
)
async def test_simple_chat(self, serving_chat, stream, include_reasoning):
messages = [{"role": "user", "content": "what is 1+1?"}]
# Test the Harmony messages for the first turn's input
req = ChatCompletionRequest(model=MODEL_NAME, messages=messages)
req = ChatCompletionRequest(
model=MODEL_NAME,
messages=messages,
include_reasoning=include_reasoning,
)
input_messages, _ = (
serving_chat.openai_serving_render._make_request_with_harmony(req)
)
@@ -1342,7 +1366,11 @@ class TestServingChatWithHarmony:
response = await self.generate_response_from_harmony_str(
serving_chat, req, response_str, stream=stream
)
verify_chat_response(response, content=final_str, reasoning=reasoning_str)
verify_chat_response(
response,
content=final_str,
reasoning=reasoning_str if include_reasoning else None,
)
# Add the output messages from the first turn as input to the second turn
for choice in response.choices:
tests/entrypoints/openai/parser/test_harmony_utils.py
-106
View File
@@ -11,12 +11,10 @@ from vllm.entrypoints.openai.parser.harmony_utils import (
auto_drop_analysis_messages,
create_tool_definition,
extract_function_from_recipient,
get_encoding,
get_system_message,
has_custom_tools,
is_function_recipient,
parse_chat_input_to_harmony_message,
parse_chat_output,
)
from vllm.entrypoints.openai.responses.harmony import (
response_input_to_harmony,
@@ -941,110 +939,6 @@ class TestAutoDropAnalysisMessages:
assert cleaned_messages == messages[1:]
class TestParseChatOutput:
def test_parse_chat_output_interrupted_first_message(self) -> None:
harmony_str = "<|channel|>final<|message|>I'm in the middle of answering"
token_ids = get_encoding().encode(harmony_str, allowed_special="all")
reasoning, final_content, _ = parse_chat_output(token_ids)
assert reasoning is None
assert final_content == "I'm in the middle of answering"
def test_parse_chat_output_interrupted_reasoning_first_message(self) -> None:
harmony_str = "<|channel|>analysis<|message|>I'm in the middle of thinking"
token_ids = get_encoding().encode(harmony_str, allowed_special="all")
reasoning, final_content, _ = parse_chat_output(token_ids)
assert reasoning == "I'm in the middle of thinking"
assert final_content is None
def test_parse_chat_output_complete_reasoning_interrupted_content(self) -> None:
harmony_str = (
"<|channel|>analysis<|message|>I'm thinking.<|end|>"
"<|start|>assistant<|channel|>final"
"<|message|>I'm in the middle of answering"
)
token_ids = get_encoding().encode(harmony_str, allowed_special="all")
reasoning, final_content, _ = parse_chat_output(token_ids)
assert reasoning == "I'm thinking."
assert final_content == "I'm in the middle of answering"
def test_parse_chat_output_complete_content(self) -> None:
harmony_str = "<|channel|>final<|message|>The answer is 4.<|end|>"
token_ids = get_encoding().encode(harmony_str, allowed_special="all")
reasoning, final_content, _ = parse_chat_output(token_ids)
assert reasoning is None
assert final_content == "The answer is 4."
def test_parse_chat_output_complete_commentary(self) -> None:
harmony_str = (
"<|channel|>commentary<|message|>I need to call some tools.<|end|>"
)
token_ids = get_encoding().encode(harmony_str, allowed_special="all")
reasoning, final_content, _ = parse_chat_output(token_ids)
assert reasoning is None
assert final_content == "I need to call some tools."
def test_parse_chat_output_complete_reasoning(self) -> None:
harmony_str = (
"<|channel|>analysis<|message|>I've thought hard about this.<|end|>"
)
token_ids = get_encoding().encode(harmony_str, allowed_special="all")
reasoning, final_content, _ = parse_chat_output(token_ids)
assert reasoning == "I've thought hard about this."
assert final_content is None
def test_parse_chat_output_complete_reasoning_and_content(self) -> None:
harmony_str = (
"<|channel|>analysis<|message|>I've thought hard about this.<|end|>"
"<|start|>assistant<|channel|>final<|message|>The answer is 4.<|end|>"
)
token_ids = get_encoding().encode(harmony_str, allowed_special="all")
reasoning, final_content, _ = parse_chat_output(token_ids)
assert reasoning == "I've thought hard about this."
assert final_content == "The answer is 4."
def test_parse_chat_output_commentary_with_recipient_excluded(self) -> None:
"""Commentary with a recipient (tool call) should not appear in
final_content — those are handled separately by the tool parser.
The first message is a preamble (visible), the second is a tool
call (excluded). Only the preamble should appear in final_content.
"""
harmony_str = (
"<|channel|>commentary"
"<|message|>Let me check the weather.<|end|>"
"<|start|>assistant to=functions.get_weather"
"<|channel|>commentary"
'<|message|>{"location": "SF"}<|end|>'
)
token_ids = get_encoding().encode(harmony_str, allowed_special="all")
reasoning, final_content, _ = parse_chat_output(token_ids)
assert reasoning is None
assert final_content == "Let me check the weather."
def test_parse_chat_output_interrupted_preamble(self) -> None:
"""Partial/interrupted preamble (commentary without recipient) should
appear in final_content, not reasoning."""
harmony_str = "<|channel|>commentary<|message|>I'll search for that"
token_ids = get_encoding().encode(harmony_str, allowed_special="all")
reasoning, final_content, _ = parse_chat_output(token_ids)
assert reasoning is None
assert final_content == "I'll search for that"
def test_parse_chat_output_preamble_then_final(self) -> None:
"""Preamble followed by a final message should both appear in
final_content, joined by newline."""
harmony_str = (
"<|channel|>commentary"
"<|message|>Let me look that up.<|end|>"
"<|start|>assistant<|channel|>final"
"<|message|>The answer is 42.<|end|>"
)
token_ids = get_encoding().encode(harmony_str, allowed_special="all")
reasoning, final_content, _ = parse_chat_output(token_ids)
assert reasoning is None
assert final_content == "Let me look that up.\nThe answer is 42."
def test_has_custom_tools() -> None:
assert not has_custom_tools(set())
assert not has_custom_tools({"web_search_preview", "code_interpreter", "container"})
tests/parser/test_harmony.py
+452
View File
@@ -0,0 +1,452 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import json
from collections.abc import Sequence
import pytest
from openai_harmony import (
Conversation,
Message,
RenderConversationConfig,
Role,
)
from transformers import AutoTokenizer
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
from vllm.entrypoints.openai.engine.protocol import FunctionCall
from vllm.entrypoints.openai.parser.harmony_utils import (
get_encoding,
)
from vllm.parser.harmony import HarmonyParser
from vllm.parser.parser_manager import ParserManager
REASONING_MODEL_NAME = "openai/gpt-oss-20b"
@pytest.fixture(scope="module")
def gpt_oss_tokenizer():
return AutoTokenizer.from_pretrained(REASONING_MODEL_NAME)
@pytest.fixture
def harmony_parser(gpt_oss_tokenizer):
parser_cls = ParserManager.get_parser(
tool_parser_name="openai",
reasoning_parser_name="openai_gptoss",
enable_auto_tools=True,
model_name=REASONING_MODEL_NAME,
is_harmony=True,
)
assert parser_cls is HarmonyParser
return parser_cls(gpt_oss_tokenizer)
@pytest.fixture
def chat_request():
return ChatCompletionRequest(
model="openai/gpt-oss-20b",
messages=[{"role": "user", "content": "Hello"}],
)
def encode_output(harmony_str: str) -> list[int]:
return get_encoding().encode(harmony_str, allowed_special="all")
def assistant(content: str, channel: str) -> Message:
return Message.from_role_and_content(Role.ASSISTANT, content).with_channel(channel)
def tool_call(
recipient: str,
content: str,
channel: str = "commentary",
content_type: str | None = "json",
) -> Message:
message = assistant(content, channel).with_recipient(recipient)
return message if content_type is None else message.with_content_type(content_type)
def get_model_output_tokens(
prompt_messages: Sequence[Message],
response_messages: Sequence[Message],
) -> list[int]:
enc = get_encoding()
# Keep analysis messages when synthesizing model-output-only token sequences
# for parser tests; the default render path drops them after a later final turn.
config = RenderConversationConfig(auto_drop_analysis=False)
prompt_ids = enc.render_conversation_for_completion(
Conversation.from_messages(list(prompt_messages)),
Role.ASSISTANT,
config=config,
)
full_ids = enc.render_conversation_for_completion(
Conversation.from_messages([*prompt_messages, *response_messages]),
Role.ASSISTANT,
config=config,
)
assert full_ids[: len(prompt_ids)] == prompt_ids
return full_ids[len(prompt_ids) :]
def get_text(msg: Message) -> str:
return msg.content[0].text if msg.content else ""
def visible_segments(result) -> list[tuple[str | None, str | None, str]]:
return [
(segment.channel, segment.recipient, segment.delta)
for segment in result.segments
if not segment.is_boundary and segment.delta
]
def tool_call_tuples(tool_calls: list[FunctionCall] | None) -> list[tuple[str, str]]:
return [] if tool_calls is None else [(tc.name, tc.arguments) for tc in tool_calls]
class TestParse:
# Rendered conversation outputs.
def test_reasoning_only(self, harmony_parser, chat_request):
prompt = [Message.from_role_and_content(Role.USER, "Why?")]
response = [assistant("This is reasoning", "analysis")]
reasoning, content, tool_calls = harmony_parser.parse(
"",
chat_request,
model_output_token_ids=get_model_output_tokens(prompt, response),
)
assert reasoning == "This is reasoning"
assert content is None
assert tool_calls is None
def test_content_only(self, harmony_parser, chat_request):
prompt = [Message.from_role_and_content(Role.USER, "Hello")]
response = [assistant("This is a test", "final")]
reasoning, content, tool_calls = harmony_parser.parse(
"",
chat_request,
model_output_token_ids=get_model_output_tokens(prompt, response),
)
assert reasoning is None
assert content == "This is a test"
assert tool_calls is None
def test_reasoning_and_content(self, harmony_parser, chat_request):
prompt = [Message.from_role_and_content(Role.USER, "What is 2+2?")]
response = [
assistant("I should think first.", "analysis"),
assistant("The answer is 4.", "final"),
]
reasoning, content, tool_calls = harmony_parser.parse(
"",
chat_request,
model_output_token_ids=get_model_output_tokens(prompt, response),
)
assert reasoning == "I should think first."
assert content == "The answer is 4."
assert tool_calls is None
@pytest.mark.parametrize(
"tool_args",
[
'{"location": "Tokyo"}',
'{\n"location": "Tokyo"\n}',
],
)
@pytest.mark.parametrize("tool_channel", ["commentary", "analysis"])
def test_single_tool_call(
self, harmony_parser, chat_request, tool_args, tool_channel
):
prompt = [
Message.from_role_and_content(Role.USER, "What is the weather in Tokyo?")
]
response = [tool_call("functions.get_current_weather", tool_args, tool_channel)]
reasoning, content, tool_calls = harmony_parser.parse(
"",
chat_request,
model_output_token_ids=get_model_output_tokens(prompt, response),
)
assert reasoning is None
assert content is None
assert tool_call_tuples(tool_calls) == [
("get_current_weather", json.dumps({"location": "Tokyo"}))
]
def test_multiple_tool_calls_varied_formats(self, harmony_parser, chat_request):
prompt = [
Message.from_role_and_content(
Role.USER, "What is the weather in Tokyo based on where I'm at?"
)
]
response = [
tool_call("functions.get_current_weather", '{"location": "Tokyo"}'),
tool_call("functions.get_user_location", '{"location": "Tokyo"}'),
tool_call(
"functions.no_content_type",
'{"location": "Tokyo"}',
content_type=None,
),
tool_call("functions.not_json_no_content_type", "foo", content_type=None),
tool_call("functions.empty_args", "{}"),
tool_call("functions.no_args", ""),
]
_, content, tool_calls = harmony_parser.parse(
"",
chat_request,
model_output_token_ids=get_model_output_tokens(prompt, response),
)
assert content is None
assert tool_call_tuples(tool_calls) == [
("get_current_weather", json.dumps({"location": "Tokyo"})),
("get_user_location", json.dumps({"location": "Tokyo"})),
("no_content_type", json.dumps({"location": "Tokyo"})),
("not_json_no_content_type", "foo"),
("empty_args", json.dumps({})),
("no_args", ""),
]
def test_tool_call_bare_recipient(self, harmony_parser, chat_request):
prompt = [Message.from_role_and_content(Role.USER, "Weather?")]
response = [tool_call("get_current_weather", '{"location": "Tokyo"}')]
_, _, tool_calls = harmony_parser.parse(
"",
chat_request,
model_output_token_ids=get_model_output_tokens(prompt, response),
)
assert tool_call_tuples(tool_calls) == [
("get_current_weather", json.dumps({"location": "Tokyo"}))
]
def test_multiple_tool_calls_bare_recipients(self, harmony_parser, chat_request):
prompt = [Message.from_role_and_content(Role.USER, "Use both tools.")]
response = [
tool_call("get_current_weather", '{"location": "Tokyo"}'),
tool_call("get_user_location", "{}"),
]
_, _, tool_calls = harmony_parser.parse(
"",
chat_request,
model_output_token_ids=get_model_output_tokens(prompt, response),
)
assert tool_call_tuples(tool_calls) == [
("get_current_weather", json.dumps({"location": "Tokyo"})),
("get_user_location", json.dumps({})),
]
def test_assistant_recipient_not_tool(self, harmony_parser, chat_request):
prompt = [Message.from_role_and_content(Role.USER, "Hello")]
response = [
tool_call("assistant", "Some tool response", content_type=None),
assistant("Here is the answer", "final"),
]
reasoning, content, tool_calls = harmony_parser.parse(
"",
chat_request,
model_output_token_ids=get_model_output_tokens(prompt, response),
)
assert reasoning is None
assert content == "Here is the answer"
assert tool_calls is None
def test_tool_call_dotted_name(self, harmony_parser, chat_request):
prompt = [Message.from_role_and_content(Role.USER, "Compute 2+3")]
response = [tool_call("math.sum", '{"a": 2, "b": 3}')]
_, _, tool_calls = harmony_parser.parse(
"",
chat_request,
model_output_token_ids=get_model_output_tokens(prompt, response),
)
assert tool_call_tuples(tool_calls) == [
("math.sum", json.dumps({"a": 2, "b": 3}))
]
def test_tool_calls_with_final_content(self, harmony_parser, chat_request):
prompt = [Message.from_role_and_content(Role.USER, "What is the weather?")]
response = [
assistant("User asked about the weather.", "analysis"),
tool_call("functions.get_current_weather", '{"location": "Tokyo"}'),
assistant("This tool call will get the weather.", "final"),
]
reasoning, content, tool_calls = harmony_parser.parse(
"",
chat_request,
model_output_token_ids=get_model_output_tokens(prompt, response),
)
assert reasoning == "User asked about the weather."
assert content == "This tool call will get the weather."
assert tool_call_tuples(tool_calls) == [
("get_current_weather", json.dumps({"location": "Tokyo"}))
]
# Raw/truncated Harmony output streams.
def test_interrupted_first_message(self, harmony_parser, chat_request):
reasoning, content, tool_calls = harmony_parser.parse(
"",
chat_request,
model_output_token_ids=encode_output(
"<|channel|>final<|message|>I'm in the middle of answering"
),
)
assert reasoning is None
assert content == "I'm in the middle of answering"
assert tool_calls is None
def test_interrupted_reasoning_first_message(self, harmony_parser, chat_request):
reasoning, content, tool_calls = harmony_parser.parse(
"",
chat_request,
model_output_token_ids=encode_output(
"<|channel|>analysis<|message|>I'm in the middle of thinking"
),
)
assert reasoning == "I'm in the middle of thinking"
assert content is None
assert tool_calls is None
def test_truncated_output(self, harmony_parser, chat_request):
reasoning, content, tool_calls = harmony_parser.parse(
"",
chat_request,
model_output_token_ids=encode_output(
"<|channel|>analysis<|message|>I'm thinking.<|end|>"
"<|start|>assistant<|channel|>final<|message|>"
"I'm in the middle of answering"
),
)
assert reasoning == "I'm thinking."
assert content == "I'm in the middle of answering"
assert tool_calls is None
@pytest.mark.parametrize(
("harmony_str", "expected_content"),
[
(
"<|channel|>commentary<|message|>I'll search for that",
"I'll search for that",
),
(
"<|channel|>commentary<|message|>Let me look that up.<|end|>"
"<|start|>assistant<|channel|>final<|message|>The answer is 42.<|end|>",
"Let me look that up.\nThe answer is 42.",
),
],
)
def test_commentary_preambles(
self,
harmony_parser,
chat_request,
harmony_str,
expected_content,
):
reasoning, content, tool_calls = harmony_parser.parse(
"",
chat_request,
model_output_token_ids=encode_output(harmony_str),
)
assert reasoning is None
assert content == expected_content
assert tool_calls is None
def test_commentary_with_recipient_excluded(self, harmony_parser, chat_request):
reasoning, content, tool_calls = harmony_parser.parse(
"",
chat_request,
model_output_token_ids=encode_output(
"<|channel|>commentary"
"<|message|>Let me check the weather.<|end|>"
"<|start|>assistant to=functions.get_weather"
"<|channel|>commentary"
'<|message|>{"location": "SF"}<|end|>'
),
)
assert reasoning is None
assert content == "Let me check the weather."
assert tool_call_tuples(tool_calls) == [
("get_weather", json.dumps({"location": "SF"}))
]
class TestProcessChunk:
def test_empty(self, harmony_parser):
result = harmony_parser.process_chunk([])
assert result.segments == []
assert result.reasoning_token_count == 0
def test_single_channel(self, harmony_parser):
result = harmony_parser.process_chunk(
encode_output("<|channel|>final<|message|>Hello")
)
assert visible_segments(result) == [("final", None, "Hello")]
def test_cross_channel(self, harmony_parser):
result = harmony_parser.process_chunk(
encode_output(
"<|channel|>analysis<|message|>Think<|end|>"
"<|start|>assistant<|channel|>final<|message|>Answer"
)
)
assert visible_segments(result) == [
("analysis", None, "Think"),
("final", None, "Answer"),
]
def test_boundary_detection(self, harmony_parser):
result = harmony_parser.process_chunk(
encode_output("<|channel|>final<|message|>Done<|end|>")
)
boundary_segments = [
segment for segment in result.segments if segment.is_boundary
]
assert len(boundary_segments) == 1
assert boundary_segments[0].completed_message is not None
assert boundary_segments[0].completed_message.channel == "final"
assert get_text(boundary_segments[0].completed_message) == "Done"
def test_multi_boundary(self, harmony_parser):
result = harmony_parser.process_chunk(
encode_output(
"<|channel|>analysis<|message|>One<|end|>"
"<|start|>assistant<|channel|>final<|message|>Two<|end|>"
)
)
boundary_segments = [
segment for segment in result.segments if segment.is_boundary
]
assert [
get_text(segment.completed_message) for segment in boundary_segments
] == [
"One",
"Two",
]
tests/tool_parsers/test_openai_tool_parser.py
-415
View File
@@ -1,415 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import json
import pytest
from openai_harmony import (
Conversation,
DeveloperContent,
HarmonyEncodingName,
Message,
Role,
SystemContent,
load_harmony_encoding,
)
from vllm.entrypoints.openai.engine.protocol import FunctionCall, ToolCall
from vllm.tokenizers import get_tokenizer
from vllm.tool_parsers.openai_tool_parser import OpenAIToolParser
MODEL = "gpt2"
@pytest.fixture(scope="module")
def openai_tokenizer():
# The parser does not use the tokenizer, but the constructor requires it.
return get_tokenizer(MODEL)
@pytest.fixture
def openai_tool_parser(openai_tokenizer):
return OpenAIToolParser(openai_tokenizer)
@pytest.fixture(scope="module")
def harmony_encoding():
return load_harmony_encoding(HarmonyEncodingName.HARMONY_GPT_OSS)
def assert_tool_calls(
actual_tool_calls: list[ToolCall],
expected_tool_calls: list[ToolCall],
):
assert len(actual_tool_calls) == len(expected_tool_calls)
for actual_tool_call, expected_tool_call in zip(
actual_tool_calls, expected_tool_calls
):
assert isinstance(actual_tool_call.id, str)
assert len(actual_tool_call.id) > 16 # Default from protocol.py
assert actual_tool_call.type == "function"
assert actual_tool_call.function == expected_tool_call.function
def test_extract_tool_calls_no_tools(openai_tool_parser, harmony_encoding):
convo = Conversation.from_messages(
[
Message.from_role_and_content(
Role.SYSTEM,
SystemContent.new(),
),
Message.from_role_and_content(
Role.DEVELOPER,
DeveloperContent.new().with_instructions("Talk like a pirate!"),
),
Message.from_role_and_content(Role.USER, "Arrr, how be you?"),
Message.from_role_and_content(
Role.ASSISTANT, "This is a test"
).with_channel("final"),
]
)
token_ids = harmony_encoding.render_conversation_for_completion(
convo, Role.ASSISTANT
)
extracted_info = openai_tool_parser.extract_tool_calls(
"",
request=None,
token_ids=token_ids,
)
assert not extracted_info.tools_called
assert extracted_info.tool_calls == []
assert extracted_info.content == "This is a test"
@pytest.mark.parametrize(
"tool_args",
[
'{"location": "Tokyo"}',
'{\n"location": "Tokyo"\n}',
],
)
def test_extract_tool_calls_single_tool(
openai_tool_parser, harmony_encoding, tool_args
):
convo = Conversation.from_messages(
[
Message.from_role_and_content(Role.USER, "What is the weather in Tokyo?"),
Message.from_role_and_content(
Role.ASSISTANT,
'User asks: "What is the weather in Tokyo?" We need to use get_current_weather tool.', # noqa: E501
).with_channel("analysis"),
Message.from_role_and_content(Role.ASSISTANT, tool_args)
.with_channel("commentary")
.with_recipient("functions.get_current_weather")
.with_content_type("json"),
]
)
token_ids = harmony_encoding.render_conversation_for_completion(
convo, Role.ASSISTANT
)
extracted_info = openai_tool_parser.extract_tool_calls(
"",
request=None,
token_ids=token_ids,
)
assert extracted_info.tools_called
expected_tool_calls = [
ToolCall(
function=FunctionCall(
name="get_current_weather",
arguments=json.dumps({"location": "Tokyo"}),
)
)
]
assert_tool_calls(extracted_info.tool_calls, expected_tool_calls)
assert extracted_info.content is None
def test_extract_tool_calls_multiple_tools(
openai_tool_parser,
harmony_encoding,
):
convo = Conversation.from_messages(
[
Message.from_role_and_content(
Role.USER, "What is the weather in Tokyo based on where I'm at?"
),
Message.from_role_and_content(
Role.ASSISTANT,
'User asks: "What is the weather in Tokyo?" based on their location. We need to use get_current_weather tool and get_user_location tool.', # noqa: E501
).with_channel("analysis"),
Message.from_role_and_content(Role.ASSISTANT, '{"location": "Tokyo"}')
.with_channel("commentary")
.with_recipient("functions.get_current_weather")
.with_content_type("json"),
Message.from_role_and_content(Role.ASSISTANT, '{"location": "Tokyo"}')
.with_channel("commentary")
.with_recipient("functions.get_user_location")
.with_content_type("json"),
Message.from_role_and_content(Role.ASSISTANT, '{"location": "Tokyo"}')
.with_channel("commentary")
.with_recipient("functions.no_content_type"),
Message.from_role_and_content(Role.ASSISTANT, "foo")
.with_channel("commentary")
.with_recipient("functions.not_json_no_content_type"),
Message.from_role_and_content(Role.ASSISTANT, "{}")
.with_channel("commentary")
.with_recipient("functions.empty_args")
.with_content_type("json"),
Message.from_role_and_content(Role.ASSISTANT, "")
.with_channel("commentary")
.with_recipient("functions.no_args")
.with_content_type("json"),
]
)
token_ids = harmony_encoding.render_conversation_for_completion(
convo,
Role.ASSISTANT,
)
extracted_info = openai_tool_parser.extract_tool_calls(
"",
request=None,
token_ids=token_ids,
)
assert extracted_info.tools_called
expected_tool_calls = [
ToolCall(
function=FunctionCall(
name="get_current_weather",
arguments=json.dumps({"location": "Tokyo"}),
)
),
ToolCall(
function=FunctionCall(
name="get_user_location",
arguments=json.dumps({"location": "Tokyo"}),
)
),
ToolCall(
function=FunctionCall(
name="no_content_type",
arguments=json.dumps({"location": "Tokyo"}),
)
),
ToolCall(
function=FunctionCall(
name="not_json_no_content_type",
arguments="foo",
)
),
ToolCall(
function=FunctionCall(
name="empty_args",
arguments=json.dumps({}),
)
),
ToolCall(
function=FunctionCall(
name="no_args",
arguments="",
)
),
]
assert_tool_calls(extracted_info.tool_calls, expected_tool_calls)
assert extracted_info.content is None
def test_extract_tool_calls_bare_function_name(
openai_tool_parser,
harmony_encoding,
):
convo = Conversation.from_messages(
[
Message.from_role_and_content(Role.USER, "What is the weather in Tokyo?"),
Message.from_role_and_content(
Role.ASSISTANT,
"We need to use get_current_weather tool.",
).with_channel("analysis"),
Message.from_role_and_content(Role.ASSISTANT, '{"location": "Tokyo"}')
.with_channel("commentary")
.with_recipient("get_current_weather")
.with_content_type("json"),
]
)
token_ids = harmony_encoding.render_conversation_for_completion(
convo, Role.ASSISTANT
)
extracted_info = openai_tool_parser.extract_tool_calls(
"",
request=None,
token_ids=token_ids,
)
assert extracted_info.tools_called
expected_tool_calls = [
ToolCall(
function=FunctionCall(
name="get_current_weather",
arguments=json.dumps({"location": "Tokyo"}),
)
)
]
assert_tool_calls(extracted_info.tool_calls, expected_tool_calls)
assert extracted_info.content is None
def test_extract_tool_calls_bare_function_name_multiple(
openai_tool_parser,
harmony_encoding,
):
convo = Conversation.from_messages(
[
Message.from_role_and_content(
Role.USER, "What is the weather in Tokyo based on where I'm at?"
),
Message.from_role_and_content(
Role.ASSISTANT,
"We need to use both tools.",
).with_channel("analysis"),
Message.from_role_and_content(Role.ASSISTANT, '{"location": "Tokyo"}')
.with_channel("commentary")
.with_recipient("get_current_weather")
.with_content_type("json"),
Message.from_role_and_content(Role.ASSISTANT, "{}")
.with_channel("commentary")
.with_recipient("get_user_location")
.with_content_type("json"),
]
)
token_ids = harmony_encoding.render_conversation_for_completion(
convo, Role.ASSISTANT
)
extracted_info = openai_tool_parser.extract_tool_calls(
"",
request=None,
token_ids=token_ids,
)
assert extracted_info.tools_called
expected_tool_calls = [
ToolCall(
function=FunctionCall(
name="get_current_weather",
arguments=json.dumps({"location": "Tokyo"}),
)
),
ToolCall(
function=FunctionCall(
name="get_user_location",
arguments=json.dumps({}),
)
),
]
assert_tool_calls(extracted_info.tool_calls, expected_tool_calls)
def test_extract_tool_calls_assistant_recipient_ignored(
openai_tool_parser,
harmony_encoding,
):
convo = Conversation.from_messages(
[
Message.from_role_and_content(Role.USER, "Hello"),
Message.from_role_and_content(Role.ASSISTANT, "Some tool response")
.with_channel("commentary")
.with_recipient("assistant"),
Message.from_role_and_content(
Role.ASSISTANT, "Here is the answer"
).with_channel("final"),
]
)
token_ids = harmony_encoding.render_conversation_for_completion(
convo, Role.ASSISTANT
)
extracted_info = openai_tool_parser.extract_tool_calls(
"",
request=None,
token_ids=token_ids,
)
assert not extracted_info.tools_called
assert extracted_info.tool_calls == []
assert extracted_info.content == "Here is the answer"
def test_extract_tool_calls_dotted_function_name(
openai_tool_parser,
harmony_encoding,
):
convo = Conversation.from_messages(
[
Message.from_role_and_content(Role.USER, "Compute 2+3"),
Message.from_role_and_content(Role.ASSISTANT, '{"a": 2, "b": 3}')
.with_channel("commentary")
.with_recipient("math.sum")
.with_content_type("json"),
]
)
token_ids = harmony_encoding.render_conversation_for_completion(
convo, Role.ASSISTANT
)
extracted_info = openai_tool_parser.extract_tool_calls(
"",
request=None,
token_ids=token_ids,
)
assert extracted_info.tools_called
expected_tool_calls = [
ToolCall(
function=FunctionCall(
name="math.sum",
arguments=json.dumps({"a": 2, "b": 3}),
)
)
]
assert_tool_calls(extracted_info.tool_calls, expected_tool_calls)
def test_extract_tool_calls_with_content(
openai_tool_parser,
harmony_encoding,
):
final_content = "This tool call will get the weather."
convo = Conversation.from_messages(
[
Message.from_role_and_content(
Role.USER, "What is the weather in Tokyo based on where I'm at?"
),
Message.from_role_and_content(
Role.ASSISTANT,
'User asks: "What is the weather in Tokyo?" based on their location. We need to use get_current_weather tool and get_user_location tool.', # noqa: E501
).with_channel("analysis"),
Message.from_role_and_content(Role.ASSISTANT, '{"location": "Tokyo"}')
.with_channel("commentary")
.with_recipient("functions.get_current_weather")
.with_content_type("json"),
Message.from_role_and_content(Role.ASSISTANT, final_content).with_channel(
"final"
),
]
)
token_ids = harmony_encoding.render_conversation_for_completion(
convo,
Role.ASSISTANT,
)
extracted_info = openai_tool_parser.extract_tool_calls(
"",
request=None,
token_ids=token_ids,
)
assert extracted_info.tools_called
expected_tool_calls = [
ToolCall(
function=FunctionCall(
name="get_current_weather",
arguments=json.dumps({"location": "Tokyo"}),
)
),
]
assert_tool_calls(extracted_info.tool_calls, expected_tool_calls)
assert extracted_info.content == final_content
vllm/entrypoints/openai/chat_completion/serving.py
+10 -74
View File
@@ -54,7 +54,6 @@ from vllm.entrypoints.openai.engine.serving import (
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
from vllm.entrypoints.openai.parser.harmony_utils import (
get_streamable_parser_for_assistant,
parse_chat_output,
)
from vllm.entrypoints.serve.utils.api_utils import get_max_tokens, should_include_usage
from vllm.entrypoints.serve.utils.request_logger import RequestLogger
@@ -135,6 +134,7 @@ class OpenAIServingChat(OpenAIServing):
reasoning_parser_name=reasoning_parser,
enable_auto_tools=enable_auto_tools,
model_name=self.model_config.model,
is_harmony=self.model_config.hf_config.model_type == "gpt_oss",
)
if (
is_mistral_tool_parser(self.tool_parser)
@@ -359,14 +359,6 @@ class OpenAIServingChat(OpenAIServing):
assert len(generators) == 1
(result_generator,) = generators
parser: Parser | None = None
if self.parser_cls is not None:
parser = self.parser_cls(
tokenizer,
request.tools,
chat_template_kwargs=chat_template_kwargs,
)
if request.stream:
return self.chat_completion_stream_generator(
request,
@@ -387,7 +379,7 @@ class OpenAIServingChat(OpenAIServing):
conversation,
tokenizer,
request_metadata,
parser,
chat_template_kwargs=chat_template_kwargs,
)
def get_chat_request_role(self, request: ChatCompletionRequest) -> str:
@@ -840,7 +832,7 @@ class OpenAIServingChat(OpenAIServing):
conversation: list[ConversationMessage],
tokenizer: TokenizerLike,
request_metadata: RequestResponseMetadata,
parser: Parser | None = None,
chat_template_kwargs: dict[str, Any] | None = None,
) -> ErrorResponse | ChatCompletionResponse:
created_time = int(time.time())
final_res: RequestOutput | None = None
@@ -871,7 +863,6 @@ class OpenAIServingChat(OpenAIServing):
self._raise_if_error(output.finish_reason, request_id)
token_ids = output.token_ids
out_logprobs = output.logprobs
tool_call_info = None
if request.logprobs and request.top_logprobs is not None:
assert out_logprobs is not None, "Did not output logprobs"
@@ -885,75 +876,20 @@ class OpenAIServingChat(OpenAIServing):
else:
logprobs = None
if self.use_harmony:
reasoning, content, _ = parse_chat_output(token_ids)
if not request.include_reasoning:
reasoning = None
if self.tool_parser is not None:
if tokenizer is None:
raise ValueError(
"Tokenizer not available when `skip_tokenizer_init=True`"
)
tool_parser = self.tool_parser(tokenizer, request.tools)
# NOTE: We use token_ids for openai tool parser
tool_call_info = tool_parser.extract_tool_calls(
"",
request=request,
token_ids=token_ids, # type: ignore
)
content = tool_call_info.content
message = ChatMessage(
role=role,
reasoning=reasoning,
content=content,
tool_calls=tool_call_info.tool_calls,
)
else:
message = ChatMessage(
role=role,
reasoning=reasoning,
content=content,
)
# Encode routed_experts for transport. JSON can't carry raw
# bytes, so we write the ndarray as a ``.npy`` byte stream
# and base64-encode it. ``pybase64`` is ~3x faster than the
# stdlib ``base64`` on large payloads thanks to SIMD.
routed_experts_b64 = None
if output.routed_experts is not None:
buf = io.BytesIO()
np.save(buf, output.routed_experts)
routed_experts_b64 = base64.b64encode(buf.getvalue()).decode(
"ascii"
)
choice_data = ChatCompletionResponseChoice(
index=output.index,
message=message,
logprobs=logprobs,
finish_reason=(
"tool_calls"
if (tool_call_info is not None and tool_call_info.tools_called)
else output.finish_reason
if output.finish_reason
else "stop"
),
stop_reason=output.stop_reason,
token_ids=(
as_list(output.token_ids) if request.return_token_ids else None
),
routed_experts=routed_experts_b64,
parser: Parser | None = None
if self.parser_cls is not None:
parser = self.parser_cls(
tokenizer,
request.tools,
chat_template_kwargs=chat_template_kwargs,
)
choices.append(choice_data)
continue
if parser is not None:
reasoning, content, tool_calls = parser.parse(
output.text,
request,
enable_auto_tools=self.enable_auto_tools,
model_output_token_ids=token_ids,
)
if not request.include_reasoning:
reasoning = None
vllm/entrypoints/openai/parser/harmony_utils.py
+1 -63
View File
@@ -2,7 +2,7 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import datetime
from collections.abc import Iterable, Sequence
from collections.abc import Sequence
from typing import Any
from openai.types.responses.tool import Tool
@@ -456,65 +456,3 @@ def render_for_completion(messages: list[Message]) -> list[int]:
def get_streamable_parser_for_assistant() -> StreamableParser:
return StreamableParser(get_encoding(), role=Role.ASSISTANT)
def parse_output_into_messages(token_ids: Iterable[int]) -> StreamableParser:
parser = get_streamable_parser_for_assistant()
for token_id in token_ids:
parser.process(token_id)
return parser
def parse_chat_output(
token_ids: Sequence[int],
) -> tuple[str | None, str | None, bool]:
"""
Parse the output of a Harmony chat completion into reasoning and final content.
Note that when the `openai` tool parser is used, serving_chat only uses this
for the reasoning content and gets the final content from the tool call parser.
When the `openai` tool parser is not enabled, or when `GptOssReasoningParser` is
in use,this needs to return the final content without any tool calls parsed.
Empty reasoning or final content is returned as None instead of an empty string.
"""
parser = parse_output_into_messages(token_ids)
output_msgs = parser.messages
is_tool_call = False # TODO: update this when tool call is supported
# Get completed messages from the parser
# - analysis channel: hidden reasoning
# - commentary channel without recipient (preambles): visible to user
# - final channel: visible to user
# - commentary with recipient (tool calls): handled separately by tool parser
reasoning_texts = [
msg.content[0].text for msg in output_msgs if msg.channel == "analysis"
]
final_texts = [
msg.content[0].text
for msg in output_msgs
if msg.channel == "final" or (msg.channel == "commentary" and not msg.recipient)
]
# Extract partial messages from the parser
if parser.current_channel == "analysis" and parser.current_content:
reasoning_texts.append(parser.current_content)
elif parser.current_channel == "final" and parser.current_content:
final_texts.append(parser.current_content)
elif (
parser.current_channel == "commentary"
and not parser.current_recipient
and parser.current_content
):
# Preambles (commentary without recipient) are visible to user
final_texts.append(parser.current_content)
# Flatten multiple messages into a single string
reasoning: str | None = "\n".join(reasoning_texts)
final_content: str | None = "\n".join(final_texts)
# Return None instead of empty string since existing callers check for None
reasoning = reasoning or None
final_content = final_content or None
return reasoning, final_content, is_tool_call
vllm/entrypoints/openai/responses/serving.py
+1
View File
@@ -191,6 +191,7 @@ class OpenAIServingResponses(OpenAIServing):
reasoning_parser_name=reasoning_parser,
enable_auto_tools=enable_auto_tools,
model_name=self.model_config.model,
is_harmony=self.model_config.hf_config.model_type == "gpt_oss",
)
self.enable_prompt_tokens_details = enable_prompt_tokens_details
self.enable_force_include_usage = enable_force_include_usage
vllm/parser/__init__.py
+2
View File
@@ -5,10 +5,12 @@ from vllm.parser.abstract_parser import (
DelegatingParser,
Parser,
)
from vllm.parser.harmony import HarmonyParser
from vllm.parser.parser_manager import ParserManager
__all__ = [
"Parser",
"DelegatingParser",
"HarmonyParser",
"ParserManager",
]
vllm/parser/abstract_parser.py
+3
View File
@@ -282,6 +282,7 @@ class Parser:
model_output: str,
request: ChatCompletionRequest | ResponsesRequest,
enable_auto_tools: bool = False,
model_output_token_ids: Sequence[int] = (),
) -> tuple[str | None, str | None, list[FunctionCall] | None]:
"""Parse a complete model output, extracting reasoning and tool calls.
@@ -289,6 +290,7 @@ class Parser:
model_output: The complete model-generated string.
request: The request object used to generate the output.
enable_auto_tools: Whether to enable automatic tool call parsing.
model_output_token_ids: The generated raw output token IDs.
Returns:
A tuple of (reasoning, content, tool_calls).
@@ -642,6 +644,7 @@ class DelegatingParser(Parser):
model_output: str,
request: ChatCompletionRequest | ResponsesRequest,
enable_auto_tools: bool = False,
model_output_token_ids: Sequence[int] = (),
) -> tuple[str | None, str | None, list[FunctionCall] | None]:
reasoning, content = self.extract_reasoning(model_output, request)
tool_calls, content = self._extract_tool_calls(
vllm/parser/harmony.py
+240
View File
@@ -0,0 +1,240 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from __future__ import annotations
import json
from collections.abc import Sequence
from dataclasses import dataclass
from enum import Enum, auto
from typing import TYPE_CHECKING, NamedTuple
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
from vllm.entrypoints.openai.engine.protocol import (
DeltaMessage,
FunctionCall,
)
from vllm.entrypoints.openai.parser.harmony_utils import (
extract_function_from_recipient,
get_streamable_parser_for_assistant,
is_function_recipient,
)
from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
from vllm.parser.abstract_parser import DelegatingParser
from vllm.reasoning.gptoss_reasoning_parser import GptOssReasoningParser
from vllm.tool_parsers.gptoss_tool_parser import GptOssToolParser
if TYPE_CHECKING:
from openai_harmony import Message, Role
from openai_harmony import StreamState as HarmonyStreamState
class _SegmentType(Enum):
TOOL = auto()
REASONING = auto()
CONTENT = auto()
IGNORE = auto()
@staticmethod
def from_channel_and_recipient(
channel: str | None, recipient: str | None
) -> _SegmentType:
if recipient and is_function_recipient(recipient):
return _SegmentType.TOOL
if channel == "analysis":
return _SegmentType.REASONING
if channel == "final" or (channel == "commentary" and recipient is None):
return _SegmentType.CONTENT
return _SegmentType.IGNORE
class Segment(NamedTuple):
channel: str | None
recipient: str | None
delta: str
is_boundary: bool = False
completed_message: Message | None = None
@dataclass
class ChunkResult:
segments: list[Segment]
reasoning_token_count: int
class HarmonyParser(DelegatingParser):
def __init__(self, tokenizer, tools=None, *args, **kwargs):
super().__init__(tokenizer, tools, *args, **kwargs)
if self._reasoning_parser and not isinstance(
self._reasoning_parser, GptOssReasoningParser
):
raise ValueError(
"Harmony requires GptOssReasoningParser, "
f"got {self._reasoning_parser.__class__.__name__}."
)
if self._tool_parser and not isinstance(self._tool_parser, GptOssToolParser):
raise ValueError(
"Harmony requires GptOssToolParser, "
f"got {self._tool_parser.__class__.__name__}."
)
self._harmony_parser = get_streamable_parser_for_assistant()
@property
def messages(self) -> list[Message]:
return self._harmony_parser.messages
@property
def state(self) -> HarmonyStreamState:
return self._harmony_parser.state
@property
def current_role(self) -> Role | None:
return self._harmony_parser.current_role
@property
def current_channel(self) -> str | None:
return self._harmony_parser.current_channel
@property
def current_recipient(self) -> str | None:
return self._harmony_parser.current_recipient
@property
def current_content(self) -> str:
return self._harmony_parser.current_content
@property
def current_content_type(self) -> str | None:
return self._harmony_parser.current_content_type
def parse(
self,
model_output: str,
request: ChatCompletionRequest | ResponsesRequest,
enable_auto_tools: bool = False,
model_output_token_ids: Sequence[int] = (),
) -> tuple[str | None, str | None, list[FunctionCall] | None]:
"""Parse Harmony output from token IDs.
Tool calls are always extracted regardless of ``enable_auto_tools``.
Callers must decide whether to surface them.
"""
result = self.process_chunk(model_output_token_ids)
reasoning_parts: list[str] = []
content_parts: list[str] = []
tool_calls: list[FunctionCall] = []
def _append_parsed_message(
channel: str | None,
recipient: str | None,
text: str,
content_type: str | None = None,
) -> None:
segment_type = _SegmentType.from_channel_and_recipient(channel, recipient)
match segment_type:
case _SegmentType.REASONING if self.reasoning_parser and text:
reasoning_parts.append(text)
case _SegmentType.CONTENT if text:
content_parts.append(text)
case _SegmentType.TOOL if self.tool_parser:
assert recipient is not None
if content_type is not None and "json" not in content_type:
arguments = text
else:
try:
arguments = json.dumps(json.loads(text))
except json.JSONDecodeError:
arguments = text
tool_calls.append(
FunctionCall(
name=extract_function_from_recipient(recipient),
arguments=arguments,
)
)
for segment in result.segments:
msg = segment.completed_message
if msg is None:
continue
if msg.author.role != "assistant" or not msg.content:
continue
_append_parsed_message(
channel=msg.channel,
recipient=msg.recipient,
text=msg.content[0].text,
content_type=msg.content_type,
)
if (
self.current_channel is not None
or self.current_recipient is not None
or self.current_content
):
_append_parsed_message(
channel=self.current_channel,
recipient=self.current_recipient,
text=self.current_content,
content_type=self.current_content_type,
)
reasoning = "\n".join(reasoning_parts) or None
content = "\n".join(content_parts) or None
return reasoning, content, tool_calls or None
def parse_delta(
self,
delta_text: str,
delta_token_ids: list[int],
request: ChatCompletionRequest | ResponsesRequest,
prompt_token_ids: list[int] | None = None,
*,
finished: bool,
) -> DeltaMessage | None:
raise NotImplementedError(
"HarmonyParser streaming parsing is deferred. "
"Use the existing harmony streaming path."
)
def process_chunk(self, token_ids: Sequence[int]) -> ChunkResult:
if not token_ids:
return ChunkResult(segments=[], reasoning_token_count=0)
from openai_harmony import StreamState
segments: list[Segment] = []
reasoning_token_count = 0
for token_id in token_ids:
self._harmony_parser.process(token_id)
channel = self.current_channel
recipient = self.current_recipient
delta = self._harmony_parser.last_content_delta or ""
completed_message = None
is_boundary = self.state == StreamState.EXPECT_START
if is_boundary and self.messages:
completed_message = self.messages[-1]
if channel == "analysis" or (
channel == "commentary" and recipient is not None
):
reasoning_token_count += 1
segments.append(
Segment(
channel=channel,
recipient=recipient,
delta=delta,
is_boundary=is_boundary,
completed_message=completed_message,
)
)
# TODO: Optionally merge and suppress empty Segments
return ChunkResult(
segments=segments,
reasoning_token_count=reasoning_token_count,
)
vllm/parser/mistral.py
+6 -1
View File
@@ -3,6 +3,7 @@
from __future__ import annotations
from collections.abc import Sequence
from typing import TYPE_CHECKING
from vllm.entrypoints.openai.engine.protocol import DeltaMessage, FunctionCall
@@ -43,10 +44,14 @@ class MistralParser(DelegatingParser):
model_output: str,
request: ChatCompletionRequest | ResponsesRequest,
enable_auto_tools: bool = False,
model_output_token_ids: Sequence[int] = (),
) -> tuple[str | None, str | None, list[FunctionCall] | None]:
self._maybe_force_auto_tool_parsing(request)
reasoning, content, tool_calls = super().parse(
model_output, request, enable_auto_tools
model_output,
request,
enable_auto_tools,
model_output_token_ids,
)
if tool_calls:
from vllm.tool_parsers.mistral_tool_parser import MistralToolCall
vllm/parser/parser_manager.py
+10
View File
@@ -79,6 +79,7 @@ class ParserManager:
reasoning_parser_name: str | None = None,
enable_auto_tools: bool = False,
model_name: str | None = None,
is_harmony: bool = False,
) -> type[Parser] | None:
"""
Get a Parser that handles both reasoning and tool parsing.
@@ -91,6 +92,8 @@ class ParserManager:
reasoning_parser_name: The name of the reasoning parser.
enable_auto_tools: Whether auto tool choice is enabled.
model_name: The model name for parser-specific warnings.
is_harmony: Whether the selected model uses the Harmony format.
If True, HarmonyParser is always returned.
Returns:
A Parser class, or None if neither parser is specified.
@@ -108,6 +111,13 @@ class ParserManager:
from vllm.utils.mistral import is_mistral_tool_parser
if is_harmony:
from vllm.parser.harmony import HarmonyParser
HarmonyParser.reasoning_parser_cls = reasoning_parser_cls
HarmonyParser.tool_parser_cls = tool_parser_cls
return HarmonyParser
if is_mistral_tool_parser(tool_parser_cls):
from vllm.parser.mistral import MistralParser
vllm/reasoning/gptoss_reasoning_parser.py
+10 -25
View File
@@ -8,7 +8,6 @@ from transformers import PreTrainedTokenizerBase
from vllm.entrypoints.mcp.tool_server import ToolServer
from vllm.entrypoints.openai.engine.protocol import DeltaMessage
from vllm.entrypoints.openai.parser.harmony_utils import parse_chat_output
from vllm.logger import init_logger
from vllm.reasoning import ReasoningParser
@@ -132,10 +131,10 @@ class GptOssReasoningParser(ReasoningParser):
return self.is_reasoning_end(input_ids[n - window :])
def extract_content_ids(self, input_ids: list[int]) -> list[int]:
_, content, _ = parse_chat_output(input_ids)
if content is None:
return []
return self.model_tokenizer.encode(content)
raise NotImplementedError(
"GptOssReasoningParser only provides boundary detection. "
"Use HarmonyParser for output parsing."
)
def extract_reasoning_streaming(
self,
@@ -146,25 +145,10 @@ class GptOssReasoningParser(ReasoningParser):
current_token_ids: Sequence[int],
delta_token_ids: Sequence[int],
) -> DeltaMessage | None:
prev_reasoning, prev_content, _ = parse_chat_output(list(previous_token_ids))
cur_reasoning, cur_content, _ = parse_chat_output(list(current_token_ids))
reasoning_delta = None
content_delta = None
if cur_reasoning is not None:
prev_r = prev_reasoning or ""
if cur_reasoning.startswith(prev_r):
reasoning_delta = cur_reasoning[len(prev_r) :] or None
else:
reasoning_delta = cur_reasoning
if cur_content is not None:
prev_c = prev_content or ""
if cur_content.startswith(prev_c):
content_delta = cur_content[len(prev_c) :] or None
else:
content_delta = cur_content
if reasoning_delta is None and content_delta is None:
return None
return DeltaMessage(reasoning=reasoning_delta, content=content_delta)
raise NotImplementedError(
"GptOssReasoningParser only provides boundary detection. "
"Use HarmonyParser for output parsing."
)
def extract_reasoning(
self,
@@ -172,7 +156,8 @@ class GptOssReasoningParser(ReasoningParser):
request: "ChatCompletionRequest | ResponsesRequest",
) -> tuple[str | None, str | None]:
raise NotImplementedError(
"gpt-oss has a special branch for parsing reasoning in non-streaming mode. This method shouldn't be used." # noqa: E501
"GptOssReasoningParser only provides boundary detection. "
"Use HarmonyParser for output parsing."
)
# This function prepares the structural tag to format reasoning output
vllm/tool_parsers/__init__.py
+2 -2
View File
@@ -143,8 +143,8 @@ _TOOL_PARSERS_TO_REGISTER = {
"Olmo3PythonicToolParser",
),
"openai": (
"openai_tool_parser",
"OpenAIToolParser",
"gptoss_tool_parser",
"GptOssToolParser",
),
"phi4_mini_json": (
"phi4mini_tool_parser",
vllm/tool_parsers/gptoss_tool_parser.py
+47
View File
@@ -0,0 +1,47 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from collections.abc import Sequence
from typing import TYPE_CHECKING
from vllm.entrypoints.openai.engine.protocol import (
DeltaMessage,
ExtractedToolCallInformation,
)
from vllm.tool_parsers.abstract_tool_parser import Tool, ToolParser
if TYPE_CHECKING:
from vllm.tokenizers import TokenizerLike
class GptOssToolParser(ToolParser):
"""
Stub tool parser for gpt-oss/harmony models.
All output parsing is handled by HarmonyParser. This stub exists as a
capability declaration via HarmonyParser.tool_parser_cls.
"""
def __init__(self, tokenizer: "TokenizerLike", tools: list[Tool] | None = None):
super().__init__(tokenizer, tools)
def extract_tool_calls(
self, model_output, request, **kwargs
) -> ExtractedToolCallInformation:
raise NotImplementedError(
"GptOssToolParser is a stub. Use HarmonyParser for tool parsing."
)
def extract_tool_calls_streaming(
self,
previous_text: str,
current_text: str,
delta_text: str,
previous_token_ids: Sequence[int],
current_token_ids: Sequence[int],
delta_token_ids: Sequence[int],
request,
) -> DeltaMessage | None:
raise NotImplementedError(
"GptOssToolParser is a stub. Use HarmonyParser for tool parsing."
)
vllm/tool_parsers/openai_tool_parser.py
-120
View File
@@ -1,120 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import json
from collections.abc import Sequence
from typing import TYPE_CHECKING
from vllm.entrypoints.openai.chat_completion.protocol import (
ChatCompletionRequest,
)
from vllm.entrypoints.openai.engine.protocol import (
DeltaMessage,
ExtractedToolCallInformation,
FunctionCall,
ToolCall,
)
from vllm.entrypoints.openai.parser.harmony_utils import (
extract_function_from_recipient,
is_function_recipient,
parse_output_into_messages,
)
from vllm.logger import init_logger
from vllm.tool_parsers.abstract_tool_parser import (
Tool,
ToolParser,
)
if TYPE_CHECKING:
from vllm.tokenizers import TokenizerLike
else:
TokenizerLike = object
logger = init_logger(__name__)
class OpenAIToolParser(ToolParser):
def __init__(self, tokenizer: "TokenizerLike", tools: list[Tool] | None = None):
super().__init__(tokenizer, tools)
def extract_tool_calls(
self,
model_output: str,
request: ChatCompletionRequest,
token_ids: Sequence[int] | None = None,
) -> ExtractedToolCallInformation:
if token_ids is None:
raise NotImplementedError(
"OpenAIToolParser requires token IDs and does not support text-based extraction." # noqa: E501
)
parser = parse_output_into_messages(token_ids)
tool_calls = []
final_content = None
commentary_content = None
if len(parser.messages) > 0:
for msg in parser.messages:
if msg.author.role != "assistant":
continue
if len(msg.content) < 1:
continue
msg_text = msg.content[0].text
if msg.recipient and is_function_recipient(msg.recipient):
# If no content-type is given assume JSON, as that's the
# most common case with gpt-oss models.
if not msg.content_type or "json" in msg.content_type:
# load and dump the JSON text to check validity and
# remove any extra newlines or other odd formatting
try:
tool_args = json.dumps(json.loads(msg_text))
except json.JSONDecodeError:
logger.exception(
"Error decoding JSON tool call from response."
)
tool_args = msg_text
else:
tool_args = msg_text
tool_calls.append(
ToolCall(
type="function",
function=FunctionCall(
name=extract_function_from_recipient(msg.recipient),
arguments=tool_args,
),
)
)
elif msg.channel == "final":
final_content = msg_text
elif msg.channel == "commentary" and not msg.recipient:
commentary_content = msg_text
# Extract partial content from the parser state if the generation was truncated
if parser.current_content:
if parser.current_channel == "final":
final_content = parser.current_content
elif (
parser.current_channel == "commentary" and not parser.current_recipient
):
commentary_content = parser.current_content
return ExtractedToolCallInformation(
tools_called=len(tool_calls) > 0,
tool_calls=tool_calls,
# prefer final content over commentary content if both are present
# commentary content is tool call preambles meant to be shown to the user
content=final_content or commentary_content,
)
def extract_tool_calls_streaming(
self,
previous_text: str,
current_text: str,
delta_text: str,
previous_token_ids: Sequence[int],
current_token_ids: Sequence[int],
delta_token_ids: Sequence[int],
request: ChatCompletionRequest,
) -> DeltaMessage | None:
raise NotImplementedError(
"Not being used, manual parsing in serving_chat.py" # noqa: E501
)