From f712fd0d7db6e0b2c7fbdb6e77cae155c81fd8c5 Mon Sep 17 00:00:00 2001 From: yzong-rh Date: Thu, 11 Jun 2026 17:18:30 -0400 Subject: [PATCH] [Refactor] Chat Completions Harmony Refactor, non-streaming path. (#45171) Signed-off-by: Yifan Zong --- .../chat_completion/test_serving_chat.py | 48 +- .../openai/parser/test_harmony_utils.py | 106 ---- tests/parser/test_harmony.py | 452 ++++++++++++++++++ tests/tool_parsers/test_openai_tool_parser.py | 415 ---------------- .../openai/chat_completion/serving.py | 84 +--- .../openai/parser/harmony_utils.py | 64 +-- vllm/entrypoints/openai/responses/serving.py | 1 + vllm/parser/__init__.py | 2 + vllm/parser/abstract_parser.py | 3 + vllm/parser/harmony.py | 240 ++++++++++ vllm/parser/mistral.py | 7 +- vllm/parser/parser_manager.py | 10 + vllm/reasoning/gptoss_reasoning_parser.py | 35 +- vllm/tool_parsers/__init__.py | 4 +- vllm/tool_parsers/gptoss_tool_parser.py | 47 ++ vllm/tool_parsers/openai_tool_parser.py | 120 ----- 16 files changed, 822 insertions(+), 816 deletions(-) create mode 100644 tests/parser/test_harmony.py delete mode 100644 tests/tool_parsers/test_openai_tool_parser.py create mode 100644 vllm/parser/harmony.py create mode 100644 vllm/tool_parsers/gptoss_tool_parser.py delete mode 100644 vllm/tool_parsers/openai_tool_parser.py diff --git a/tests/entrypoints/openai/chat_completion/test_serving_chat.py b/tests/entrypoints/openai/chat_completion/test_serving_chat.py index 22077bd4a3..e523cc2d4a 100644 --- a/tests/entrypoints/openai/chat_completion/test_serving_chat.py +++ b/tests/entrypoints/openai/chat_completion/test_serving_chat.py @@ -38,12 +38,12 @@ from vllm.entrypoints.serve.render.serving import OpenAIServingRender from vllm.exceptions import VLLMValidationError from vllm.inputs import TokensPrompt from vllm.outputs import CompletionOutput, RequestOutput +from vllm.parser import HarmonyParser from vllm.renderers.hf import HfRenderer from vllm.renderers.mistral import MistralRenderer from vllm.tokenizers import get_tokenizer from vllm.tokenizers.mistral import MistralTokenizer from vllm.tokenizers.registry import cached_tokenizer_from_config -from vllm.tool_parsers import ToolParserManager from vllm.v1.engine.async_llm import AsyncLLM GPT_OSS_MODEL_NAME = "openai/gpt-oss-20b" @@ -575,7 +575,13 @@ def _build_serving_render( ) -def _build_serving_chat(engine: AsyncLLM) -> OpenAIServingChat: +def _build_serving_chat( + engine: AsyncLLM, + *, + reasoning_parser: str = "", + tool_parser: str | None = None, + enable_auto_tools: bool = False, +) -> OpenAIServingChat: models = OpenAIServingModels( engine_client=engine, base_model_paths=BASE_MODEL_PATHS, @@ -590,6 +596,9 @@ def _build_serving_chat(engine: AsyncLLM) -> OpenAIServingChat: chat_template=CHAT_TEMPLATE, chat_template_content_format="auto", request_logger=None, + reasoning_parser=reasoning_parser, + tool_parser=tool_parser, + enable_auto_tools=enable_auto_tools, ) return serving_chat @@ -637,7 +646,7 @@ async def test_serving_chat_returns_correct_model_name(): serving_chat = _build_serving_chat(mock_engine) messages = [{"role": "user", "content": "what is 1+1?"}] - async def return_model_name(*args): + async def return_model_name(*args, **kwargs): return args[3] serving_chat.chat_completion_full_generator = return_model_name @@ -1210,15 +1219,21 @@ class TestServingChatWithHarmony: mock_engine = MagicMock(spec=AsyncLLM) mock_engine.errored = False mock_engine.model_config = MockModelConfig() + mock_engine.model_config.hf_config = MockHFConfig(model_type="gpt_oss") + mock_engine.model_config.hf_text_config = MockHFConfig(model_type="gpt_oss") mock_engine.input_processor = MagicMock() mock_engine.renderer = _build_renderer(mock_engine.model_config) return mock_engine @pytest.fixture() def serving_chat(self, mock_engine) -> OpenAIServingChat: - chat = _build_serving_chat(mock_engine) - chat.use_harmony = True - chat.tool_parser = ToolParserManager.get_tool_parser("openai") + chat = _build_serving_chat( + mock_engine, + reasoning_parser="openai_gptoss", + tool_parser="openai", + enable_auto_tools=True, + ) + assert chat.parser_cls is HarmonyParser return chat def mock_request_output_from_req_and_token_ids( @@ -1277,6 +1292,7 @@ class TestServingChatWithHarmony: stream: bool = False, ) -> ChatCompletionResponse: harmony_token_ids = get_encoding().encode(harmony_str, allowed_special="all") + tokenizer = get_tokenizer(GPT_OSS_MODEL_NAME) async def result_generator(): if stream: @@ -1304,11 +1320,12 @@ class TestServingChatWithHarmony: request_id=req.request_id, model_name=req.model, conversation=[], - tokenizer=get_tokenizer(req.model), + tokenizer=tokenizer, request_metadata=RequestResponseMetadata( request_id=req.request_id, model_name=req.model, ), + chat_template_kwargs=serving_chat._effective_chat_template_kwargs(req), ) if stream: @@ -1316,11 +1333,18 @@ class TestServingChatWithHarmony: return await result @pytest.mark.asyncio - async def test_simple_chat(self, serving_chat, stream): + @pytest.mark.parametrize( + "include_reasoning", [True, False], ids=["with_reasoning", "no_reasoning"] + ) + async def test_simple_chat(self, serving_chat, stream, include_reasoning): messages = [{"role": "user", "content": "what is 1+1?"}] # Test the Harmony messages for the first turn's input - req = ChatCompletionRequest(model=MODEL_NAME, messages=messages) + req = ChatCompletionRequest( + model=MODEL_NAME, + messages=messages, + include_reasoning=include_reasoning, + ) input_messages, _ = ( serving_chat.openai_serving_render._make_request_with_harmony(req) ) @@ -1342,7 +1366,11 @@ class TestServingChatWithHarmony: response = await self.generate_response_from_harmony_str( serving_chat, req, response_str, stream=stream ) - verify_chat_response(response, content=final_str, reasoning=reasoning_str) + verify_chat_response( + response, + content=final_str, + reasoning=reasoning_str if include_reasoning else None, + ) # Add the output messages from the first turn as input to the second turn for choice in response.choices: diff --git a/tests/entrypoints/openai/parser/test_harmony_utils.py b/tests/entrypoints/openai/parser/test_harmony_utils.py index d2985264e0..0027c2763f 100644 --- a/tests/entrypoints/openai/parser/test_harmony_utils.py +++ b/tests/entrypoints/openai/parser/test_harmony_utils.py @@ -11,12 +11,10 @@ from vllm.entrypoints.openai.parser.harmony_utils import ( auto_drop_analysis_messages, create_tool_definition, extract_function_from_recipient, - get_encoding, get_system_message, has_custom_tools, is_function_recipient, parse_chat_input_to_harmony_message, - parse_chat_output, ) from vllm.entrypoints.openai.responses.harmony import ( response_input_to_harmony, @@ -941,110 +939,6 @@ class TestAutoDropAnalysisMessages: assert cleaned_messages == messages[1:] -class TestParseChatOutput: - def test_parse_chat_output_interrupted_first_message(self) -> None: - harmony_str = "<|channel|>final<|message|>I'm in the middle of answering" - token_ids = get_encoding().encode(harmony_str, allowed_special="all") - reasoning, final_content, _ = parse_chat_output(token_ids) - assert reasoning is None - assert final_content == "I'm in the middle of answering" - - def test_parse_chat_output_interrupted_reasoning_first_message(self) -> None: - harmony_str = "<|channel|>analysis<|message|>I'm in the middle of thinking" - token_ids = get_encoding().encode(harmony_str, allowed_special="all") - reasoning, final_content, _ = parse_chat_output(token_ids) - assert reasoning == "I'm in the middle of thinking" - assert final_content is None - - def test_parse_chat_output_complete_reasoning_interrupted_content(self) -> None: - harmony_str = ( - "<|channel|>analysis<|message|>I'm thinking.<|end|>" - "<|start|>assistant<|channel|>final" - "<|message|>I'm in the middle of answering" - ) - token_ids = get_encoding().encode(harmony_str, allowed_special="all") - reasoning, final_content, _ = parse_chat_output(token_ids) - assert reasoning == "I'm thinking." - assert final_content == "I'm in the middle of answering" - - def test_parse_chat_output_complete_content(self) -> None: - harmony_str = "<|channel|>final<|message|>The answer is 4.<|end|>" - token_ids = get_encoding().encode(harmony_str, allowed_special="all") - reasoning, final_content, _ = parse_chat_output(token_ids) - assert reasoning is None - assert final_content == "The answer is 4." - - def test_parse_chat_output_complete_commentary(self) -> None: - harmony_str = ( - "<|channel|>commentary<|message|>I need to call some tools.<|end|>" - ) - token_ids = get_encoding().encode(harmony_str, allowed_special="all") - reasoning, final_content, _ = parse_chat_output(token_ids) - assert reasoning is None - assert final_content == "I need to call some tools." - - def test_parse_chat_output_complete_reasoning(self) -> None: - harmony_str = ( - "<|channel|>analysis<|message|>I've thought hard about this.<|end|>" - ) - token_ids = get_encoding().encode(harmony_str, allowed_special="all") - reasoning, final_content, _ = parse_chat_output(token_ids) - assert reasoning == "I've thought hard about this." - assert final_content is None - - def test_parse_chat_output_complete_reasoning_and_content(self) -> None: - harmony_str = ( - "<|channel|>analysis<|message|>I've thought hard about this.<|end|>" - "<|start|>assistant<|channel|>final<|message|>The answer is 4.<|end|>" - ) - token_ids = get_encoding().encode(harmony_str, allowed_special="all") - reasoning, final_content, _ = parse_chat_output(token_ids) - assert reasoning == "I've thought hard about this." - assert final_content == "The answer is 4." - - def test_parse_chat_output_commentary_with_recipient_excluded(self) -> None: - """Commentary with a recipient (tool call) should not appear in - final_content — those are handled separately by the tool parser. - - The first message is a preamble (visible), the second is a tool - call (excluded). Only the preamble should appear in final_content. - """ - harmony_str = ( - "<|channel|>commentary" - "<|message|>Let me check the weather.<|end|>" - "<|start|>assistant to=functions.get_weather" - "<|channel|>commentary" - '<|message|>{"location": "SF"}<|end|>' - ) - token_ids = get_encoding().encode(harmony_str, allowed_special="all") - reasoning, final_content, _ = parse_chat_output(token_ids) - assert reasoning is None - assert final_content == "Let me check the weather." - - def test_parse_chat_output_interrupted_preamble(self) -> None: - """Partial/interrupted preamble (commentary without recipient) should - appear in final_content, not reasoning.""" - harmony_str = "<|channel|>commentary<|message|>I'll search for that" - token_ids = get_encoding().encode(harmony_str, allowed_special="all") - reasoning, final_content, _ = parse_chat_output(token_ids) - assert reasoning is None - assert final_content == "I'll search for that" - - def test_parse_chat_output_preamble_then_final(self) -> None: - """Preamble followed by a final message should both appear in - final_content, joined by newline.""" - harmony_str = ( - "<|channel|>commentary" - "<|message|>Let me look that up.<|end|>" - "<|start|>assistant<|channel|>final" - "<|message|>The answer is 42.<|end|>" - ) - token_ids = get_encoding().encode(harmony_str, allowed_special="all") - reasoning, final_content, _ = parse_chat_output(token_ids) - assert reasoning is None - assert final_content == "Let me look that up.\nThe answer is 42." - - def test_has_custom_tools() -> None: assert not has_custom_tools(set()) assert not has_custom_tools({"web_search_preview", "code_interpreter", "container"}) diff --git a/tests/parser/test_harmony.py b/tests/parser/test_harmony.py new file mode 100644 index 0000000000..98687b08ed --- /dev/null +++ b/tests/parser/test_harmony.py @@ -0,0 +1,452 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +import json +from collections.abc import Sequence + +import pytest +from openai_harmony import ( + Conversation, + Message, + RenderConversationConfig, + Role, +) +from transformers import AutoTokenizer + +from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest +from vllm.entrypoints.openai.engine.protocol import FunctionCall +from vllm.entrypoints.openai.parser.harmony_utils import ( + get_encoding, +) +from vllm.parser.harmony import HarmonyParser +from vllm.parser.parser_manager import ParserManager + +REASONING_MODEL_NAME = "openai/gpt-oss-20b" + + +@pytest.fixture(scope="module") +def gpt_oss_tokenizer(): + return AutoTokenizer.from_pretrained(REASONING_MODEL_NAME) + + +@pytest.fixture +def harmony_parser(gpt_oss_tokenizer): + parser_cls = ParserManager.get_parser( + tool_parser_name="openai", + reasoning_parser_name="openai_gptoss", + enable_auto_tools=True, + model_name=REASONING_MODEL_NAME, + is_harmony=True, + ) + assert parser_cls is HarmonyParser + return parser_cls(gpt_oss_tokenizer) + + +@pytest.fixture +def chat_request(): + return ChatCompletionRequest( + model="openai/gpt-oss-20b", + messages=[{"role": "user", "content": "Hello"}], + ) + + +def encode_output(harmony_str: str) -> list[int]: + return get_encoding().encode(harmony_str, allowed_special="all") + + +def assistant(content: str, channel: str) -> Message: + return Message.from_role_and_content(Role.ASSISTANT, content).with_channel(channel) + + +def tool_call( + recipient: str, + content: str, + channel: str = "commentary", + content_type: str | None = "json", +) -> Message: + message = assistant(content, channel).with_recipient(recipient) + return message if content_type is None else message.with_content_type(content_type) + + +def get_model_output_tokens( + prompt_messages: Sequence[Message], + response_messages: Sequence[Message], +) -> list[int]: + enc = get_encoding() + # Keep analysis messages when synthesizing model-output-only token sequences + # for parser tests; the default render path drops them after a later final turn. + config = RenderConversationConfig(auto_drop_analysis=False) + prompt_ids = enc.render_conversation_for_completion( + Conversation.from_messages(list(prompt_messages)), + Role.ASSISTANT, + config=config, + ) + full_ids = enc.render_conversation_for_completion( + Conversation.from_messages([*prompt_messages, *response_messages]), + Role.ASSISTANT, + config=config, + ) + assert full_ids[: len(prompt_ids)] == prompt_ids + return full_ids[len(prompt_ids) :] + + +def get_text(msg: Message) -> str: + return msg.content[0].text if msg.content else "" + + +def visible_segments(result) -> list[tuple[str | None, str | None, str]]: + return [ + (segment.channel, segment.recipient, segment.delta) + for segment in result.segments + if not segment.is_boundary and segment.delta + ] + + +def tool_call_tuples(tool_calls: list[FunctionCall] | None) -> list[tuple[str, str]]: + return [] if tool_calls is None else [(tc.name, tc.arguments) for tc in tool_calls] + + +class TestParse: + # Rendered conversation outputs. + + def test_reasoning_only(self, harmony_parser, chat_request): + prompt = [Message.from_role_and_content(Role.USER, "Why?")] + response = [assistant("This is reasoning", "analysis")] + + reasoning, content, tool_calls = harmony_parser.parse( + "", + chat_request, + model_output_token_ids=get_model_output_tokens(prompt, response), + ) + + assert reasoning == "This is reasoning" + assert content is None + assert tool_calls is None + + def test_content_only(self, harmony_parser, chat_request): + prompt = [Message.from_role_and_content(Role.USER, "Hello")] + response = [assistant("This is a test", "final")] + + reasoning, content, tool_calls = harmony_parser.parse( + "", + chat_request, + model_output_token_ids=get_model_output_tokens(prompt, response), + ) + + assert reasoning is None + assert content == "This is a test" + assert tool_calls is None + + def test_reasoning_and_content(self, harmony_parser, chat_request): + prompt = [Message.from_role_and_content(Role.USER, "What is 2+2?")] + response = [ + assistant("I should think first.", "analysis"), + assistant("The answer is 4.", "final"), + ] + + reasoning, content, tool_calls = harmony_parser.parse( + "", + chat_request, + model_output_token_ids=get_model_output_tokens(prompt, response), + ) + + assert reasoning == "I should think first." + assert content == "The answer is 4." + assert tool_calls is None + + @pytest.mark.parametrize( + "tool_args", + [ + '{"location": "Tokyo"}', + '{\n"location": "Tokyo"\n}', + ], + ) + @pytest.mark.parametrize("tool_channel", ["commentary", "analysis"]) + def test_single_tool_call( + self, harmony_parser, chat_request, tool_args, tool_channel + ): + prompt = [ + Message.from_role_and_content(Role.USER, "What is the weather in Tokyo?") + ] + response = [tool_call("functions.get_current_weather", tool_args, tool_channel)] + + reasoning, content, tool_calls = harmony_parser.parse( + "", + chat_request, + model_output_token_ids=get_model_output_tokens(prompt, response), + ) + + assert reasoning is None + assert content is None + assert tool_call_tuples(tool_calls) == [ + ("get_current_weather", json.dumps({"location": "Tokyo"})) + ] + + def test_multiple_tool_calls_varied_formats(self, harmony_parser, chat_request): + prompt = [ + Message.from_role_and_content( + Role.USER, "What is the weather in Tokyo based on where I'm at?" + ) + ] + response = [ + tool_call("functions.get_current_weather", '{"location": "Tokyo"}'), + tool_call("functions.get_user_location", '{"location": "Tokyo"}'), + tool_call( + "functions.no_content_type", + '{"location": "Tokyo"}', + content_type=None, + ), + tool_call("functions.not_json_no_content_type", "foo", content_type=None), + tool_call("functions.empty_args", "{}"), + tool_call("functions.no_args", ""), + ] + + _, content, tool_calls = harmony_parser.parse( + "", + chat_request, + model_output_token_ids=get_model_output_tokens(prompt, response), + ) + + assert content is None + assert tool_call_tuples(tool_calls) == [ + ("get_current_weather", json.dumps({"location": "Tokyo"})), + ("get_user_location", json.dumps({"location": "Tokyo"})), + ("no_content_type", json.dumps({"location": "Tokyo"})), + ("not_json_no_content_type", "foo"), + ("empty_args", json.dumps({})), + ("no_args", ""), + ] + + def test_tool_call_bare_recipient(self, harmony_parser, chat_request): + prompt = [Message.from_role_and_content(Role.USER, "Weather?")] + response = [tool_call("get_current_weather", '{"location": "Tokyo"}')] + + _, _, tool_calls = harmony_parser.parse( + "", + chat_request, + model_output_token_ids=get_model_output_tokens(prompt, response), + ) + + assert tool_call_tuples(tool_calls) == [ + ("get_current_weather", json.dumps({"location": "Tokyo"})) + ] + + def test_multiple_tool_calls_bare_recipients(self, harmony_parser, chat_request): + prompt = [Message.from_role_and_content(Role.USER, "Use both tools.")] + response = [ + tool_call("get_current_weather", '{"location": "Tokyo"}'), + tool_call("get_user_location", "{}"), + ] + + _, _, tool_calls = harmony_parser.parse( + "", + chat_request, + model_output_token_ids=get_model_output_tokens(prompt, response), + ) + + assert tool_call_tuples(tool_calls) == [ + ("get_current_weather", json.dumps({"location": "Tokyo"})), + ("get_user_location", json.dumps({})), + ] + + def test_assistant_recipient_not_tool(self, harmony_parser, chat_request): + prompt = [Message.from_role_and_content(Role.USER, "Hello")] + response = [ + tool_call("assistant", "Some tool response", content_type=None), + assistant("Here is the answer", "final"), + ] + + reasoning, content, tool_calls = harmony_parser.parse( + "", + chat_request, + model_output_token_ids=get_model_output_tokens(prompt, response), + ) + + assert reasoning is None + assert content == "Here is the answer" + assert tool_calls is None + + def test_tool_call_dotted_name(self, harmony_parser, chat_request): + prompt = [Message.from_role_and_content(Role.USER, "Compute 2+3")] + response = [tool_call("math.sum", '{"a": 2, "b": 3}')] + + _, _, tool_calls = harmony_parser.parse( + "", + chat_request, + model_output_token_ids=get_model_output_tokens(prompt, response), + ) + + assert tool_call_tuples(tool_calls) == [ + ("math.sum", json.dumps({"a": 2, "b": 3})) + ] + + def test_tool_calls_with_final_content(self, harmony_parser, chat_request): + prompt = [Message.from_role_and_content(Role.USER, "What is the weather?")] + response = [ + assistant("User asked about the weather.", "analysis"), + tool_call("functions.get_current_weather", '{"location": "Tokyo"}'), + assistant("This tool call will get the weather.", "final"), + ] + + reasoning, content, tool_calls = harmony_parser.parse( + "", + chat_request, + model_output_token_ids=get_model_output_tokens(prompt, response), + ) + + assert reasoning == "User asked about the weather." + assert content == "This tool call will get the weather." + assert tool_call_tuples(tool_calls) == [ + ("get_current_weather", json.dumps({"location": "Tokyo"})) + ] + + # Raw/truncated Harmony output streams. + + def test_interrupted_first_message(self, harmony_parser, chat_request): + reasoning, content, tool_calls = harmony_parser.parse( + "", + chat_request, + model_output_token_ids=encode_output( + "<|channel|>final<|message|>I'm in the middle of answering" + ), + ) + + assert reasoning is None + assert content == "I'm in the middle of answering" + assert tool_calls is None + + def test_interrupted_reasoning_first_message(self, harmony_parser, chat_request): + reasoning, content, tool_calls = harmony_parser.parse( + "", + chat_request, + model_output_token_ids=encode_output( + "<|channel|>analysis<|message|>I'm in the middle of thinking" + ), + ) + + assert reasoning == "I'm in the middle of thinking" + assert content is None + assert tool_calls is None + + def test_truncated_output(self, harmony_parser, chat_request): + reasoning, content, tool_calls = harmony_parser.parse( + "", + chat_request, + model_output_token_ids=encode_output( + "<|channel|>analysis<|message|>I'm thinking.<|end|>" + "<|start|>assistant<|channel|>final<|message|>" + "I'm in the middle of answering" + ), + ) + + assert reasoning == "I'm thinking." + assert content == "I'm in the middle of answering" + assert tool_calls is None + + @pytest.mark.parametrize( + ("harmony_str", "expected_content"), + [ + ( + "<|channel|>commentary<|message|>I'll search for that", + "I'll search for that", + ), + ( + "<|channel|>commentary<|message|>Let me look that up.<|end|>" + "<|start|>assistant<|channel|>final<|message|>The answer is 42.<|end|>", + "Let me look that up.\nThe answer is 42.", + ), + ], + ) + def test_commentary_preambles( + self, + harmony_parser, + chat_request, + harmony_str, + expected_content, + ): + reasoning, content, tool_calls = harmony_parser.parse( + "", + chat_request, + model_output_token_ids=encode_output(harmony_str), + ) + + assert reasoning is None + assert content == expected_content + assert tool_calls is None + + def test_commentary_with_recipient_excluded(self, harmony_parser, chat_request): + reasoning, content, tool_calls = harmony_parser.parse( + "", + chat_request, + model_output_token_ids=encode_output( + "<|channel|>commentary" + "<|message|>Let me check the weather.<|end|>" + "<|start|>assistant to=functions.get_weather" + "<|channel|>commentary" + '<|message|>{"location": "SF"}<|end|>' + ), + ) + + assert reasoning is None + assert content == "Let me check the weather." + assert tool_call_tuples(tool_calls) == [ + ("get_weather", json.dumps({"location": "SF"})) + ] + + +class TestProcessChunk: + def test_empty(self, harmony_parser): + result = harmony_parser.process_chunk([]) + assert result.segments == [] + assert result.reasoning_token_count == 0 + + def test_single_channel(self, harmony_parser): + result = harmony_parser.process_chunk( + encode_output("<|channel|>final<|message|>Hello") + ) + + assert visible_segments(result) == [("final", None, "Hello")] + + def test_cross_channel(self, harmony_parser): + result = harmony_parser.process_chunk( + encode_output( + "<|channel|>analysis<|message|>Think<|end|>" + "<|start|>assistant<|channel|>final<|message|>Answer" + ) + ) + + assert visible_segments(result) == [ + ("analysis", None, "Think"), + ("final", None, "Answer"), + ] + + def test_boundary_detection(self, harmony_parser): + result = harmony_parser.process_chunk( + encode_output("<|channel|>final<|message|>Done<|end|>") + ) + + boundary_segments = [ + segment for segment in result.segments if segment.is_boundary + ] + assert len(boundary_segments) == 1 + assert boundary_segments[0].completed_message is not None + assert boundary_segments[0].completed_message.channel == "final" + assert get_text(boundary_segments[0].completed_message) == "Done" + + def test_multi_boundary(self, harmony_parser): + result = harmony_parser.process_chunk( + encode_output( + "<|channel|>analysis<|message|>One<|end|>" + "<|start|>assistant<|channel|>final<|message|>Two<|end|>" + ) + ) + + boundary_segments = [ + segment for segment in result.segments if segment.is_boundary + ] + assert [ + get_text(segment.completed_message) for segment in boundary_segments + ] == [ + "One", + "Two", + ] diff --git a/tests/tool_parsers/test_openai_tool_parser.py b/tests/tool_parsers/test_openai_tool_parser.py deleted file mode 100644 index 843fbca621..0000000000 --- a/tests/tool_parsers/test_openai_tool_parser.py +++ /dev/null @@ -1,415 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project - -import json - -import pytest -from openai_harmony import ( - Conversation, - DeveloperContent, - HarmonyEncodingName, - Message, - Role, - SystemContent, - load_harmony_encoding, -) - -from vllm.entrypoints.openai.engine.protocol import FunctionCall, ToolCall -from vllm.tokenizers import get_tokenizer -from vllm.tool_parsers.openai_tool_parser import OpenAIToolParser - -MODEL = "gpt2" - - -@pytest.fixture(scope="module") -def openai_tokenizer(): - # The parser does not use the tokenizer, but the constructor requires it. - return get_tokenizer(MODEL) - - -@pytest.fixture -def openai_tool_parser(openai_tokenizer): - return OpenAIToolParser(openai_tokenizer) - - -@pytest.fixture(scope="module") -def harmony_encoding(): - return load_harmony_encoding(HarmonyEncodingName.HARMONY_GPT_OSS) - - -def assert_tool_calls( - actual_tool_calls: list[ToolCall], - expected_tool_calls: list[ToolCall], -): - assert len(actual_tool_calls) == len(expected_tool_calls) - - for actual_tool_call, expected_tool_call in zip( - actual_tool_calls, expected_tool_calls - ): - assert isinstance(actual_tool_call.id, str) - assert len(actual_tool_call.id) > 16 # Default from protocol.py - assert actual_tool_call.type == "function" - assert actual_tool_call.function == expected_tool_call.function - - -def test_extract_tool_calls_no_tools(openai_tool_parser, harmony_encoding): - convo = Conversation.from_messages( - [ - Message.from_role_and_content( - Role.SYSTEM, - SystemContent.new(), - ), - Message.from_role_and_content( - Role.DEVELOPER, - DeveloperContent.new().with_instructions("Talk like a pirate!"), - ), - Message.from_role_and_content(Role.USER, "Arrr, how be you?"), - Message.from_role_and_content( - Role.ASSISTANT, "This is a test" - ).with_channel("final"), - ] - ) - token_ids = harmony_encoding.render_conversation_for_completion( - convo, Role.ASSISTANT - ) - extracted_info = openai_tool_parser.extract_tool_calls( - "", - request=None, - token_ids=token_ids, - ) - assert not extracted_info.tools_called - assert extracted_info.tool_calls == [] - assert extracted_info.content == "This is a test" - - -@pytest.mark.parametrize( - "tool_args", - [ - '{"location": "Tokyo"}', - '{\n"location": "Tokyo"\n}', - ], -) -def test_extract_tool_calls_single_tool( - openai_tool_parser, harmony_encoding, tool_args -): - convo = Conversation.from_messages( - [ - Message.from_role_and_content(Role.USER, "What is the weather in Tokyo?"), - Message.from_role_and_content( - Role.ASSISTANT, - 'User asks: "What is the weather in Tokyo?" We need to use get_current_weather tool.', # noqa: E501 - ).with_channel("analysis"), - Message.from_role_and_content(Role.ASSISTANT, tool_args) - .with_channel("commentary") - .with_recipient("functions.get_current_weather") - .with_content_type("json"), - ] - ) - token_ids = harmony_encoding.render_conversation_for_completion( - convo, Role.ASSISTANT - ) - - extracted_info = openai_tool_parser.extract_tool_calls( - "", - request=None, - token_ids=token_ids, - ) - assert extracted_info.tools_called - expected_tool_calls = [ - ToolCall( - function=FunctionCall( - name="get_current_weather", - arguments=json.dumps({"location": "Tokyo"}), - ) - ) - ] - assert_tool_calls(extracted_info.tool_calls, expected_tool_calls) - assert extracted_info.content is None - - -def test_extract_tool_calls_multiple_tools( - openai_tool_parser, - harmony_encoding, -): - convo = Conversation.from_messages( - [ - Message.from_role_and_content( - Role.USER, "What is the weather in Tokyo based on where I'm at?" - ), - Message.from_role_and_content( - Role.ASSISTANT, - 'User asks: "What is the weather in Tokyo?" based on their location. We need to use get_current_weather tool and get_user_location tool.', # noqa: E501 - ).with_channel("analysis"), - Message.from_role_and_content(Role.ASSISTANT, '{"location": "Tokyo"}') - .with_channel("commentary") - .with_recipient("functions.get_current_weather") - .with_content_type("json"), - Message.from_role_and_content(Role.ASSISTANT, '{"location": "Tokyo"}') - .with_channel("commentary") - .with_recipient("functions.get_user_location") - .with_content_type("json"), - Message.from_role_and_content(Role.ASSISTANT, '{"location": "Tokyo"}') - .with_channel("commentary") - .with_recipient("functions.no_content_type"), - Message.from_role_and_content(Role.ASSISTANT, "foo") - .with_channel("commentary") - .with_recipient("functions.not_json_no_content_type"), - Message.from_role_and_content(Role.ASSISTANT, "{}") - .with_channel("commentary") - .with_recipient("functions.empty_args") - .with_content_type("json"), - Message.from_role_and_content(Role.ASSISTANT, "") - .with_channel("commentary") - .with_recipient("functions.no_args") - .with_content_type("json"), - ] - ) - token_ids = harmony_encoding.render_conversation_for_completion( - convo, - Role.ASSISTANT, - ) - - extracted_info = openai_tool_parser.extract_tool_calls( - "", - request=None, - token_ids=token_ids, - ) - assert extracted_info.tools_called - expected_tool_calls = [ - ToolCall( - function=FunctionCall( - name="get_current_weather", - arguments=json.dumps({"location": "Tokyo"}), - ) - ), - ToolCall( - function=FunctionCall( - name="get_user_location", - arguments=json.dumps({"location": "Tokyo"}), - ) - ), - ToolCall( - function=FunctionCall( - name="no_content_type", - arguments=json.dumps({"location": "Tokyo"}), - ) - ), - ToolCall( - function=FunctionCall( - name="not_json_no_content_type", - arguments="foo", - ) - ), - ToolCall( - function=FunctionCall( - name="empty_args", - arguments=json.dumps({}), - ) - ), - ToolCall( - function=FunctionCall( - name="no_args", - arguments="", - ) - ), - ] - assert_tool_calls(extracted_info.tool_calls, expected_tool_calls) - assert extracted_info.content is None - - -def test_extract_tool_calls_bare_function_name( - openai_tool_parser, - harmony_encoding, -): - convo = Conversation.from_messages( - [ - Message.from_role_and_content(Role.USER, "What is the weather in Tokyo?"), - Message.from_role_and_content( - Role.ASSISTANT, - "We need to use get_current_weather tool.", - ).with_channel("analysis"), - Message.from_role_and_content(Role.ASSISTANT, '{"location": "Tokyo"}') - .with_channel("commentary") - .with_recipient("get_current_weather") - .with_content_type("json"), - ] - ) - token_ids = harmony_encoding.render_conversation_for_completion( - convo, Role.ASSISTANT - ) - - extracted_info = openai_tool_parser.extract_tool_calls( - "", - request=None, - token_ids=token_ids, - ) - assert extracted_info.tools_called - expected_tool_calls = [ - ToolCall( - function=FunctionCall( - name="get_current_weather", - arguments=json.dumps({"location": "Tokyo"}), - ) - ) - ] - assert_tool_calls(extracted_info.tool_calls, expected_tool_calls) - assert extracted_info.content is None - - -def test_extract_tool_calls_bare_function_name_multiple( - openai_tool_parser, - harmony_encoding, -): - convo = Conversation.from_messages( - [ - Message.from_role_and_content( - Role.USER, "What is the weather in Tokyo based on where I'm at?" - ), - Message.from_role_and_content( - Role.ASSISTANT, - "We need to use both tools.", - ).with_channel("analysis"), - Message.from_role_and_content(Role.ASSISTANT, '{"location": "Tokyo"}') - .with_channel("commentary") - .with_recipient("get_current_weather") - .with_content_type("json"), - Message.from_role_and_content(Role.ASSISTANT, "{}") - .with_channel("commentary") - .with_recipient("get_user_location") - .with_content_type("json"), - ] - ) - token_ids = harmony_encoding.render_conversation_for_completion( - convo, Role.ASSISTANT - ) - - extracted_info = openai_tool_parser.extract_tool_calls( - "", - request=None, - token_ids=token_ids, - ) - assert extracted_info.tools_called - expected_tool_calls = [ - ToolCall( - function=FunctionCall( - name="get_current_weather", - arguments=json.dumps({"location": "Tokyo"}), - ) - ), - ToolCall( - function=FunctionCall( - name="get_user_location", - arguments=json.dumps({}), - ) - ), - ] - assert_tool_calls(extracted_info.tool_calls, expected_tool_calls) - - -def test_extract_tool_calls_assistant_recipient_ignored( - openai_tool_parser, - harmony_encoding, -): - convo = Conversation.from_messages( - [ - Message.from_role_and_content(Role.USER, "Hello"), - Message.from_role_and_content(Role.ASSISTANT, "Some tool response") - .with_channel("commentary") - .with_recipient("assistant"), - Message.from_role_and_content( - Role.ASSISTANT, "Here is the answer" - ).with_channel("final"), - ] - ) - token_ids = harmony_encoding.render_conversation_for_completion( - convo, Role.ASSISTANT - ) - - extracted_info = openai_tool_parser.extract_tool_calls( - "", - request=None, - token_ids=token_ids, - ) - assert not extracted_info.tools_called - assert extracted_info.tool_calls == [] - assert extracted_info.content == "Here is the answer" - - -def test_extract_tool_calls_dotted_function_name( - openai_tool_parser, - harmony_encoding, -): - convo = Conversation.from_messages( - [ - Message.from_role_and_content(Role.USER, "Compute 2+3"), - Message.from_role_and_content(Role.ASSISTANT, '{"a": 2, "b": 3}') - .with_channel("commentary") - .with_recipient("math.sum") - .with_content_type("json"), - ] - ) - token_ids = harmony_encoding.render_conversation_for_completion( - convo, Role.ASSISTANT - ) - - extracted_info = openai_tool_parser.extract_tool_calls( - "", - request=None, - token_ids=token_ids, - ) - assert extracted_info.tools_called - expected_tool_calls = [ - ToolCall( - function=FunctionCall( - name="math.sum", - arguments=json.dumps({"a": 2, "b": 3}), - ) - ) - ] - assert_tool_calls(extracted_info.tool_calls, expected_tool_calls) - - -def test_extract_tool_calls_with_content( - openai_tool_parser, - harmony_encoding, -): - final_content = "This tool call will get the weather." - convo = Conversation.from_messages( - [ - Message.from_role_and_content( - Role.USER, "What is the weather in Tokyo based on where I'm at?" - ), - Message.from_role_and_content( - Role.ASSISTANT, - 'User asks: "What is the weather in Tokyo?" based on their location. We need to use get_current_weather tool and get_user_location tool.', # noqa: E501 - ).with_channel("analysis"), - Message.from_role_and_content(Role.ASSISTANT, '{"location": "Tokyo"}') - .with_channel("commentary") - .with_recipient("functions.get_current_weather") - .with_content_type("json"), - Message.from_role_and_content(Role.ASSISTANT, final_content).with_channel( - "final" - ), - ] - ) - token_ids = harmony_encoding.render_conversation_for_completion( - convo, - Role.ASSISTANT, - ) - - extracted_info = openai_tool_parser.extract_tool_calls( - "", - request=None, - token_ids=token_ids, - ) - assert extracted_info.tools_called - expected_tool_calls = [ - ToolCall( - function=FunctionCall( - name="get_current_weather", - arguments=json.dumps({"location": "Tokyo"}), - ) - ), - ] - assert_tool_calls(extracted_info.tool_calls, expected_tool_calls) - assert extracted_info.content == final_content diff --git a/vllm/entrypoints/openai/chat_completion/serving.py b/vllm/entrypoints/openai/chat_completion/serving.py index 2da89917a8..4924ceb8b5 100644 --- a/vllm/entrypoints/openai/chat_completion/serving.py +++ b/vllm/entrypoints/openai/chat_completion/serving.py @@ -54,7 +54,6 @@ from vllm.entrypoints.openai.engine.serving import ( from vllm.entrypoints.openai.models.serving import OpenAIServingModels from vllm.entrypoints.openai.parser.harmony_utils import ( get_streamable_parser_for_assistant, - parse_chat_output, ) from vllm.entrypoints.serve.utils.api_utils import get_max_tokens, should_include_usage from vllm.entrypoints.serve.utils.request_logger import RequestLogger @@ -135,6 +134,7 @@ class OpenAIServingChat(OpenAIServing): reasoning_parser_name=reasoning_parser, enable_auto_tools=enable_auto_tools, model_name=self.model_config.model, + is_harmony=self.model_config.hf_config.model_type == "gpt_oss", ) if ( is_mistral_tool_parser(self.tool_parser) @@ -359,14 +359,6 @@ class OpenAIServingChat(OpenAIServing): assert len(generators) == 1 (result_generator,) = generators - parser: Parser | None = None - if self.parser_cls is not None: - parser = self.parser_cls( - tokenizer, - request.tools, - chat_template_kwargs=chat_template_kwargs, - ) - if request.stream: return self.chat_completion_stream_generator( request, @@ -387,7 +379,7 @@ class OpenAIServingChat(OpenAIServing): conversation, tokenizer, request_metadata, - parser, + chat_template_kwargs=chat_template_kwargs, ) def get_chat_request_role(self, request: ChatCompletionRequest) -> str: @@ -840,7 +832,7 @@ class OpenAIServingChat(OpenAIServing): conversation: list[ConversationMessage], tokenizer: TokenizerLike, request_metadata: RequestResponseMetadata, - parser: Parser | None = None, + chat_template_kwargs: dict[str, Any] | None = None, ) -> ErrorResponse | ChatCompletionResponse: created_time = int(time.time()) final_res: RequestOutput | None = None @@ -871,7 +863,6 @@ class OpenAIServingChat(OpenAIServing): self._raise_if_error(output.finish_reason, request_id) token_ids = output.token_ids out_logprobs = output.logprobs - tool_call_info = None if request.logprobs and request.top_logprobs is not None: assert out_logprobs is not None, "Did not output logprobs" @@ -885,75 +876,20 @@ class OpenAIServingChat(OpenAIServing): else: logprobs = None - if self.use_harmony: - reasoning, content, _ = parse_chat_output(token_ids) - if not request.include_reasoning: - reasoning = None - - if self.tool_parser is not None: - if tokenizer is None: - raise ValueError( - "Tokenizer not available when `skip_tokenizer_init=True`" - ) - - tool_parser = self.tool_parser(tokenizer, request.tools) - # NOTE: We use token_ids for openai tool parser - tool_call_info = tool_parser.extract_tool_calls( - "", - request=request, - token_ids=token_ids, # type: ignore - ) - content = tool_call_info.content - message = ChatMessage( - role=role, - reasoning=reasoning, - content=content, - tool_calls=tool_call_info.tool_calls, - ) - else: - message = ChatMessage( - role=role, - reasoning=reasoning, - content=content, - ) - - # Encode routed_experts for transport. JSON can't carry raw - # bytes, so we write the ndarray as a ``.npy`` byte stream - # and base64-encode it. ``pybase64`` is ~3x faster than the - # stdlib ``base64`` on large payloads thanks to SIMD. - routed_experts_b64 = None - if output.routed_experts is not None: - buf = io.BytesIO() - np.save(buf, output.routed_experts) - routed_experts_b64 = base64.b64encode(buf.getvalue()).decode( - "ascii" - ) - - choice_data = ChatCompletionResponseChoice( - index=output.index, - message=message, - logprobs=logprobs, - finish_reason=( - "tool_calls" - if (tool_call_info is not None and tool_call_info.tools_called) - else output.finish_reason - if output.finish_reason - else "stop" - ), - stop_reason=output.stop_reason, - token_ids=( - as_list(output.token_ids) if request.return_token_ids else None - ), - routed_experts=routed_experts_b64, + parser: Parser | None = None + if self.parser_cls is not None: + parser = self.parser_cls( + tokenizer, + request.tools, + chat_template_kwargs=chat_template_kwargs, ) - choices.append(choice_data) - continue if parser is not None: reasoning, content, tool_calls = parser.parse( output.text, request, enable_auto_tools=self.enable_auto_tools, + model_output_token_ids=token_ids, ) if not request.include_reasoning: reasoning = None diff --git a/vllm/entrypoints/openai/parser/harmony_utils.py b/vllm/entrypoints/openai/parser/harmony_utils.py index 771faabe60..82316efb86 100644 --- a/vllm/entrypoints/openai/parser/harmony_utils.py +++ b/vllm/entrypoints/openai/parser/harmony_utils.py @@ -2,7 +2,7 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project import datetime -from collections.abc import Iterable, Sequence +from collections.abc import Sequence from typing import Any from openai.types.responses.tool import Tool @@ -456,65 +456,3 @@ def render_for_completion(messages: list[Message]) -> list[int]: def get_streamable_parser_for_assistant() -> StreamableParser: return StreamableParser(get_encoding(), role=Role.ASSISTANT) - - -def parse_output_into_messages(token_ids: Iterable[int]) -> StreamableParser: - parser = get_streamable_parser_for_assistant() - for token_id in token_ids: - parser.process(token_id) - return parser - - -def parse_chat_output( - token_ids: Sequence[int], -) -> tuple[str | None, str | None, bool]: - """ - Parse the output of a Harmony chat completion into reasoning and final content. - Note that when the `openai` tool parser is used, serving_chat only uses this - for the reasoning content and gets the final content from the tool call parser. - - When the `openai` tool parser is not enabled, or when `GptOssReasoningParser` is - in use,this needs to return the final content without any tool calls parsed. - - Empty reasoning or final content is returned as None instead of an empty string. - """ - parser = parse_output_into_messages(token_ids) - output_msgs = parser.messages - is_tool_call = False # TODO: update this when tool call is supported - - # Get completed messages from the parser - # - analysis channel: hidden reasoning - # - commentary channel without recipient (preambles): visible to user - # - final channel: visible to user - # - commentary with recipient (tool calls): handled separately by tool parser - reasoning_texts = [ - msg.content[0].text for msg in output_msgs if msg.channel == "analysis" - ] - final_texts = [ - msg.content[0].text - for msg in output_msgs - if msg.channel == "final" or (msg.channel == "commentary" and not msg.recipient) - ] - - # Extract partial messages from the parser - if parser.current_channel == "analysis" and parser.current_content: - reasoning_texts.append(parser.current_content) - elif parser.current_channel == "final" and parser.current_content: - final_texts.append(parser.current_content) - elif ( - parser.current_channel == "commentary" - and not parser.current_recipient - and parser.current_content - ): - # Preambles (commentary without recipient) are visible to user - final_texts.append(parser.current_content) - - # Flatten multiple messages into a single string - reasoning: str | None = "\n".join(reasoning_texts) - final_content: str | None = "\n".join(final_texts) - - # Return None instead of empty string since existing callers check for None - reasoning = reasoning or None - final_content = final_content or None - - return reasoning, final_content, is_tool_call diff --git a/vllm/entrypoints/openai/responses/serving.py b/vllm/entrypoints/openai/responses/serving.py index 51831f6083..69fbcce818 100644 --- a/vllm/entrypoints/openai/responses/serving.py +++ b/vllm/entrypoints/openai/responses/serving.py @@ -191,6 +191,7 @@ class OpenAIServingResponses(OpenAIServing): reasoning_parser_name=reasoning_parser, enable_auto_tools=enable_auto_tools, model_name=self.model_config.model, + is_harmony=self.model_config.hf_config.model_type == "gpt_oss", ) self.enable_prompt_tokens_details = enable_prompt_tokens_details self.enable_force_include_usage = enable_force_include_usage diff --git a/vllm/parser/__init__.py b/vllm/parser/__init__.py index de815b2e1f..e13c2ece9f 100644 --- a/vllm/parser/__init__.py +++ b/vllm/parser/__init__.py @@ -5,10 +5,12 @@ from vllm.parser.abstract_parser import ( DelegatingParser, Parser, ) +from vllm.parser.harmony import HarmonyParser from vllm.parser.parser_manager import ParserManager __all__ = [ "Parser", "DelegatingParser", + "HarmonyParser", "ParserManager", ] diff --git a/vllm/parser/abstract_parser.py b/vllm/parser/abstract_parser.py index 48db01c14e..4fe7b7ec4d 100644 --- a/vllm/parser/abstract_parser.py +++ b/vllm/parser/abstract_parser.py @@ -282,6 +282,7 @@ class Parser: model_output: str, request: ChatCompletionRequest | ResponsesRequest, enable_auto_tools: bool = False, + model_output_token_ids: Sequence[int] = (), ) -> tuple[str | None, str | None, list[FunctionCall] | None]: """Parse a complete model output, extracting reasoning and tool calls. @@ -289,6 +290,7 @@ class Parser: model_output: The complete model-generated string. request: The request object used to generate the output. enable_auto_tools: Whether to enable automatic tool call parsing. + model_output_token_ids: The generated raw output token IDs. Returns: A tuple of (reasoning, content, tool_calls). @@ -642,6 +644,7 @@ class DelegatingParser(Parser): model_output: str, request: ChatCompletionRequest | ResponsesRequest, enable_auto_tools: bool = False, + model_output_token_ids: Sequence[int] = (), ) -> tuple[str | None, str | None, list[FunctionCall] | None]: reasoning, content = self.extract_reasoning(model_output, request) tool_calls, content = self._extract_tool_calls( diff --git a/vllm/parser/harmony.py b/vllm/parser/harmony.py new file mode 100644 index 0000000000..c1eb7ea042 --- /dev/null +++ b/vllm/parser/harmony.py @@ -0,0 +1,240 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +from __future__ import annotations + +import json +from collections.abc import Sequence +from dataclasses import dataclass +from enum import Enum, auto +from typing import TYPE_CHECKING, NamedTuple + +from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest +from vllm.entrypoints.openai.engine.protocol import ( + DeltaMessage, + FunctionCall, +) +from vllm.entrypoints.openai.parser.harmony_utils import ( + extract_function_from_recipient, + get_streamable_parser_for_assistant, + is_function_recipient, +) +from vllm.entrypoints.openai.responses.protocol import ResponsesRequest +from vllm.parser.abstract_parser import DelegatingParser +from vllm.reasoning.gptoss_reasoning_parser import GptOssReasoningParser +from vllm.tool_parsers.gptoss_tool_parser import GptOssToolParser + +if TYPE_CHECKING: + from openai_harmony import Message, Role + from openai_harmony import StreamState as HarmonyStreamState + + +class _SegmentType(Enum): + TOOL = auto() + REASONING = auto() + CONTENT = auto() + IGNORE = auto() + + @staticmethod + def from_channel_and_recipient( + channel: str | None, recipient: str | None + ) -> _SegmentType: + if recipient and is_function_recipient(recipient): + return _SegmentType.TOOL + if channel == "analysis": + return _SegmentType.REASONING + if channel == "final" or (channel == "commentary" and recipient is None): + return _SegmentType.CONTENT + return _SegmentType.IGNORE + + +class Segment(NamedTuple): + channel: str | None + recipient: str | None + delta: str + is_boundary: bool = False + completed_message: Message | None = None + + +@dataclass +class ChunkResult: + segments: list[Segment] + reasoning_token_count: int + + +class HarmonyParser(DelegatingParser): + def __init__(self, tokenizer, tools=None, *args, **kwargs): + super().__init__(tokenizer, tools, *args, **kwargs) + + if self._reasoning_parser and not isinstance( + self._reasoning_parser, GptOssReasoningParser + ): + raise ValueError( + "Harmony requires GptOssReasoningParser, " + f"got {self._reasoning_parser.__class__.__name__}." + ) + + if self._tool_parser and not isinstance(self._tool_parser, GptOssToolParser): + raise ValueError( + "Harmony requires GptOssToolParser, " + f"got {self._tool_parser.__class__.__name__}." + ) + + self._harmony_parser = get_streamable_parser_for_assistant() + + @property + def messages(self) -> list[Message]: + return self._harmony_parser.messages + + @property + def state(self) -> HarmonyStreamState: + return self._harmony_parser.state + + @property + def current_role(self) -> Role | None: + return self._harmony_parser.current_role + + @property + def current_channel(self) -> str | None: + return self._harmony_parser.current_channel + + @property + def current_recipient(self) -> str | None: + return self._harmony_parser.current_recipient + + @property + def current_content(self) -> str: + return self._harmony_parser.current_content + + @property + def current_content_type(self) -> str | None: + return self._harmony_parser.current_content_type + + def parse( + self, + model_output: str, + request: ChatCompletionRequest | ResponsesRequest, + enable_auto_tools: bool = False, + model_output_token_ids: Sequence[int] = (), + ) -> tuple[str | None, str | None, list[FunctionCall] | None]: + """Parse Harmony output from token IDs. + + Tool calls are always extracted regardless of ``enable_auto_tools``. + Callers must decide whether to surface them. + """ + result = self.process_chunk(model_output_token_ids) + + reasoning_parts: list[str] = [] + content_parts: list[str] = [] + tool_calls: list[FunctionCall] = [] + + def _append_parsed_message( + channel: str | None, + recipient: str | None, + text: str, + content_type: str | None = None, + ) -> None: + segment_type = _SegmentType.from_channel_and_recipient(channel, recipient) + match segment_type: + case _SegmentType.REASONING if self.reasoning_parser and text: + reasoning_parts.append(text) + case _SegmentType.CONTENT if text: + content_parts.append(text) + case _SegmentType.TOOL if self.tool_parser: + assert recipient is not None + if content_type is not None and "json" not in content_type: + arguments = text + else: + try: + arguments = json.dumps(json.loads(text)) + except json.JSONDecodeError: + arguments = text + tool_calls.append( + FunctionCall( + name=extract_function_from_recipient(recipient), + arguments=arguments, + ) + ) + + for segment in result.segments: + msg = segment.completed_message + if msg is None: + continue + if msg.author.role != "assistant" or not msg.content: + continue + _append_parsed_message( + channel=msg.channel, + recipient=msg.recipient, + text=msg.content[0].text, + content_type=msg.content_type, + ) + + if ( + self.current_channel is not None + or self.current_recipient is not None + or self.current_content + ): + _append_parsed_message( + channel=self.current_channel, + recipient=self.current_recipient, + text=self.current_content, + content_type=self.current_content_type, + ) + + reasoning = "\n".join(reasoning_parts) or None + content = "\n".join(content_parts) or None + return reasoning, content, tool_calls or None + + def parse_delta( + self, + delta_text: str, + delta_token_ids: list[int], + request: ChatCompletionRequest | ResponsesRequest, + prompt_token_ids: list[int] | None = None, + *, + finished: bool, + ) -> DeltaMessage | None: + raise NotImplementedError( + "HarmonyParser streaming parsing is deferred. " + "Use the existing harmony streaming path." + ) + + def process_chunk(self, token_ids: Sequence[int]) -> ChunkResult: + if not token_ids: + return ChunkResult(segments=[], reasoning_token_count=0) + + from openai_harmony import StreamState + + segments: list[Segment] = [] + reasoning_token_count = 0 + for token_id in token_ids: + self._harmony_parser.process(token_id) + channel = self.current_channel + recipient = self.current_recipient + delta = self._harmony_parser.last_content_delta or "" + completed_message = None + is_boundary = self.state == StreamState.EXPECT_START + if is_boundary and self.messages: + completed_message = self.messages[-1] + + if channel == "analysis" or ( + channel == "commentary" and recipient is not None + ): + reasoning_token_count += 1 + + segments.append( + Segment( + channel=channel, + recipient=recipient, + delta=delta, + is_boundary=is_boundary, + completed_message=completed_message, + ) + ) + + # TODO: Optionally merge and suppress empty Segments + + return ChunkResult( + segments=segments, + reasoning_token_count=reasoning_token_count, + ) diff --git a/vllm/parser/mistral.py b/vllm/parser/mistral.py index c7f557a5a9..52f16136ee 100644 --- a/vllm/parser/mistral.py +++ b/vllm/parser/mistral.py @@ -3,6 +3,7 @@ from __future__ import annotations +from collections.abc import Sequence from typing import TYPE_CHECKING from vllm.entrypoints.openai.engine.protocol import DeltaMessage, FunctionCall @@ -43,10 +44,14 @@ class MistralParser(DelegatingParser): model_output: str, request: ChatCompletionRequest | ResponsesRequest, enable_auto_tools: bool = False, + model_output_token_ids: Sequence[int] = (), ) -> tuple[str | None, str | None, list[FunctionCall] | None]: self._maybe_force_auto_tool_parsing(request) reasoning, content, tool_calls = super().parse( - model_output, request, enable_auto_tools + model_output, + request, + enable_auto_tools, + model_output_token_ids, ) if tool_calls: from vllm.tool_parsers.mistral_tool_parser import MistralToolCall diff --git a/vllm/parser/parser_manager.py b/vllm/parser/parser_manager.py index 6c2fdf52dd..1b5133f5a8 100644 --- a/vllm/parser/parser_manager.py +++ b/vllm/parser/parser_manager.py @@ -79,6 +79,7 @@ class ParserManager: reasoning_parser_name: str | None = None, enable_auto_tools: bool = False, model_name: str | None = None, + is_harmony: bool = False, ) -> type[Parser] | None: """ Get a Parser that handles both reasoning and tool parsing. @@ -91,6 +92,8 @@ class ParserManager: reasoning_parser_name: The name of the reasoning parser. enable_auto_tools: Whether auto tool choice is enabled. model_name: The model name for parser-specific warnings. + is_harmony: Whether the selected model uses the Harmony format. + If True, HarmonyParser is always returned. Returns: A Parser class, or None if neither parser is specified. @@ -108,6 +111,13 @@ class ParserManager: from vllm.utils.mistral import is_mistral_tool_parser + if is_harmony: + from vllm.parser.harmony import HarmonyParser + + HarmonyParser.reasoning_parser_cls = reasoning_parser_cls + HarmonyParser.tool_parser_cls = tool_parser_cls + return HarmonyParser + if is_mistral_tool_parser(tool_parser_cls): from vllm.parser.mistral import MistralParser diff --git a/vllm/reasoning/gptoss_reasoning_parser.py b/vllm/reasoning/gptoss_reasoning_parser.py index 1ba933cca3..d7bdca8291 100644 --- a/vllm/reasoning/gptoss_reasoning_parser.py +++ b/vllm/reasoning/gptoss_reasoning_parser.py @@ -8,7 +8,6 @@ from transformers import PreTrainedTokenizerBase from vllm.entrypoints.mcp.tool_server import ToolServer from vllm.entrypoints.openai.engine.protocol import DeltaMessage -from vllm.entrypoints.openai.parser.harmony_utils import parse_chat_output from vllm.logger import init_logger from vllm.reasoning import ReasoningParser @@ -132,10 +131,10 @@ class GptOssReasoningParser(ReasoningParser): return self.is_reasoning_end(input_ids[n - window :]) def extract_content_ids(self, input_ids: list[int]) -> list[int]: - _, content, _ = parse_chat_output(input_ids) - if content is None: - return [] - return self.model_tokenizer.encode(content) + raise NotImplementedError( + "GptOssReasoningParser only provides boundary detection. " + "Use HarmonyParser for output parsing." + ) def extract_reasoning_streaming( self, @@ -146,25 +145,10 @@ class GptOssReasoningParser(ReasoningParser): current_token_ids: Sequence[int], delta_token_ids: Sequence[int], ) -> DeltaMessage | None: - prev_reasoning, prev_content, _ = parse_chat_output(list(previous_token_ids)) - cur_reasoning, cur_content, _ = parse_chat_output(list(current_token_ids)) - reasoning_delta = None - content_delta = None - if cur_reasoning is not None: - prev_r = prev_reasoning or "" - if cur_reasoning.startswith(prev_r): - reasoning_delta = cur_reasoning[len(prev_r) :] or None - else: - reasoning_delta = cur_reasoning - if cur_content is not None: - prev_c = prev_content or "" - if cur_content.startswith(prev_c): - content_delta = cur_content[len(prev_c) :] or None - else: - content_delta = cur_content - if reasoning_delta is None and content_delta is None: - return None - return DeltaMessage(reasoning=reasoning_delta, content=content_delta) + raise NotImplementedError( + "GptOssReasoningParser only provides boundary detection. " + "Use HarmonyParser for output parsing." + ) def extract_reasoning( self, @@ -172,7 +156,8 @@ class GptOssReasoningParser(ReasoningParser): request: "ChatCompletionRequest | ResponsesRequest", ) -> tuple[str | None, str | None]: raise NotImplementedError( - "gpt-oss has a special branch for parsing reasoning in non-streaming mode. This method shouldn't be used." # noqa: E501 + "GptOssReasoningParser only provides boundary detection. " + "Use HarmonyParser for output parsing." ) # This function prepares the structural tag to format reasoning output diff --git a/vllm/tool_parsers/__init__.py b/vllm/tool_parsers/__init__.py index bf832f178b..9c534e77f6 100644 --- a/vllm/tool_parsers/__init__.py +++ b/vllm/tool_parsers/__init__.py @@ -143,8 +143,8 @@ _TOOL_PARSERS_TO_REGISTER = { "Olmo3PythonicToolParser", ), "openai": ( - "openai_tool_parser", - "OpenAIToolParser", + "gptoss_tool_parser", + "GptOssToolParser", ), "phi4_mini_json": ( "phi4mini_tool_parser", diff --git a/vllm/tool_parsers/gptoss_tool_parser.py b/vllm/tool_parsers/gptoss_tool_parser.py new file mode 100644 index 0000000000..6857e6bbe7 --- /dev/null +++ b/vllm/tool_parsers/gptoss_tool_parser.py @@ -0,0 +1,47 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +from collections.abc import Sequence +from typing import TYPE_CHECKING + +from vllm.entrypoints.openai.engine.protocol import ( + DeltaMessage, + ExtractedToolCallInformation, +) +from vllm.tool_parsers.abstract_tool_parser import Tool, ToolParser + +if TYPE_CHECKING: + from vllm.tokenizers import TokenizerLike + + +class GptOssToolParser(ToolParser): + """ + Stub tool parser for gpt-oss/harmony models. + + All output parsing is handled by HarmonyParser. This stub exists as a + capability declaration via HarmonyParser.tool_parser_cls. + """ + + def __init__(self, tokenizer: "TokenizerLike", tools: list[Tool] | None = None): + super().__init__(tokenizer, tools) + + def extract_tool_calls( + self, model_output, request, **kwargs + ) -> ExtractedToolCallInformation: + raise NotImplementedError( + "GptOssToolParser is a stub. Use HarmonyParser for tool parsing." + ) + + def extract_tool_calls_streaming( + self, + previous_text: str, + current_text: str, + delta_text: str, + previous_token_ids: Sequence[int], + current_token_ids: Sequence[int], + delta_token_ids: Sequence[int], + request, + ) -> DeltaMessage | None: + raise NotImplementedError( + "GptOssToolParser is a stub. Use HarmonyParser for tool parsing." + ) diff --git a/vllm/tool_parsers/openai_tool_parser.py b/vllm/tool_parsers/openai_tool_parser.py deleted file mode 100644 index e5c37fbd3d..0000000000 --- a/vllm/tool_parsers/openai_tool_parser.py +++ /dev/null @@ -1,120 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -import json -from collections.abc import Sequence -from typing import TYPE_CHECKING - -from vllm.entrypoints.openai.chat_completion.protocol import ( - ChatCompletionRequest, -) -from vllm.entrypoints.openai.engine.protocol import ( - DeltaMessage, - ExtractedToolCallInformation, - FunctionCall, - ToolCall, -) -from vllm.entrypoints.openai.parser.harmony_utils import ( - extract_function_from_recipient, - is_function_recipient, - parse_output_into_messages, -) -from vllm.logger import init_logger -from vllm.tool_parsers.abstract_tool_parser import ( - Tool, - ToolParser, -) - -if TYPE_CHECKING: - from vllm.tokenizers import TokenizerLike -else: - TokenizerLike = object - -logger = init_logger(__name__) - - -class OpenAIToolParser(ToolParser): - def __init__(self, tokenizer: "TokenizerLike", tools: list[Tool] | None = None): - super().__init__(tokenizer, tools) - - def extract_tool_calls( - self, - model_output: str, - request: ChatCompletionRequest, - token_ids: Sequence[int] | None = None, - ) -> ExtractedToolCallInformation: - if token_ids is None: - raise NotImplementedError( - "OpenAIToolParser requires token IDs and does not support text-based extraction." # noqa: E501 - ) - - parser = parse_output_into_messages(token_ids) - tool_calls = [] - final_content = None - commentary_content = None - - if len(parser.messages) > 0: - for msg in parser.messages: - if msg.author.role != "assistant": - continue - if len(msg.content) < 1: - continue - msg_text = msg.content[0].text - if msg.recipient and is_function_recipient(msg.recipient): - # If no content-type is given assume JSON, as that's the - # most common case with gpt-oss models. - if not msg.content_type or "json" in msg.content_type: - # load and dump the JSON text to check validity and - # remove any extra newlines or other odd formatting - try: - tool_args = json.dumps(json.loads(msg_text)) - except json.JSONDecodeError: - logger.exception( - "Error decoding JSON tool call from response." - ) - tool_args = msg_text - else: - tool_args = msg_text - tool_calls.append( - ToolCall( - type="function", - function=FunctionCall( - name=extract_function_from_recipient(msg.recipient), - arguments=tool_args, - ), - ) - ) - elif msg.channel == "final": - final_content = msg_text - elif msg.channel == "commentary" and not msg.recipient: - commentary_content = msg_text - - # Extract partial content from the parser state if the generation was truncated - if parser.current_content: - if parser.current_channel == "final": - final_content = parser.current_content - elif ( - parser.current_channel == "commentary" and not parser.current_recipient - ): - commentary_content = parser.current_content - - return ExtractedToolCallInformation( - tools_called=len(tool_calls) > 0, - tool_calls=tool_calls, - # prefer final content over commentary content if both are present - # commentary content is tool call preambles meant to be shown to the user - content=final_content or commentary_content, - ) - - def extract_tool_calls_streaming( - self, - previous_text: str, - current_text: str, - delta_text: str, - previous_token_ids: Sequence[int], - current_token_ids: Sequence[int], - delta_token_ids: Sequence[int], - request: ChatCompletionRequest, - ) -> DeltaMessage | None: - raise NotImplementedError( - "Not being used, manual parsing in serving_chat.py" # noqa: E501 - )