[Refactor] Chat Completions Harmony Refactor, non-streaming path. (#45171)
Signed-off-by: Yifan Zong <yzong@redhat.com>
This commit is contained in:
@@ -38,12 +38,12 @@ from vllm.entrypoints.serve.render.serving import OpenAIServingRender
|
||||
from vllm.exceptions import VLLMValidationError
|
||||
from vllm.inputs import TokensPrompt
|
||||
from vllm.outputs import CompletionOutput, RequestOutput
|
||||
from vllm.parser import HarmonyParser
|
||||
from vllm.renderers.hf import HfRenderer
|
||||
from vllm.renderers.mistral import MistralRenderer
|
||||
from vllm.tokenizers import get_tokenizer
|
||||
from vllm.tokenizers.mistral import MistralTokenizer
|
||||
from vllm.tokenizers.registry import cached_tokenizer_from_config
|
||||
from vllm.tool_parsers import ToolParserManager
|
||||
from vllm.v1.engine.async_llm import AsyncLLM
|
||||
|
||||
GPT_OSS_MODEL_NAME = "openai/gpt-oss-20b"
|
||||
@@ -575,7 +575,13 @@ def _build_serving_render(
|
||||
)
|
||||
|
||||
|
||||
def _build_serving_chat(engine: AsyncLLM) -> OpenAIServingChat:
|
||||
def _build_serving_chat(
|
||||
engine: AsyncLLM,
|
||||
*,
|
||||
reasoning_parser: str = "",
|
||||
tool_parser: str | None = None,
|
||||
enable_auto_tools: bool = False,
|
||||
) -> OpenAIServingChat:
|
||||
models = OpenAIServingModels(
|
||||
engine_client=engine,
|
||||
base_model_paths=BASE_MODEL_PATHS,
|
||||
@@ -590,6 +596,9 @@ def _build_serving_chat(engine: AsyncLLM) -> OpenAIServingChat:
|
||||
chat_template=CHAT_TEMPLATE,
|
||||
chat_template_content_format="auto",
|
||||
request_logger=None,
|
||||
reasoning_parser=reasoning_parser,
|
||||
tool_parser=tool_parser,
|
||||
enable_auto_tools=enable_auto_tools,
|
||||
)
|
||||
|
||||
return serving_chat
|
||||
@@ -637,7 +646,7 @@ async def test_serving_chat_returns_correct_model_name():
|
||||
serving_chat = _build_serving_chat(mock_engine)
|
||||
messages = [{"role": "user", "content": "what is 1+1?"}]
|
||||
|
||||
async def return_model_name(*args):
|
||||
async def return_model_name(*args, **kwargs):
|
||||
return args[3]
|
||||
|
||||
serving_chat.chat_completion_full_generator = return_model_name
|
||||
@@ -1210,15 +1219,21 @@ class TestServingChatWithHarmony:
|
||||
mock_engine = MagicMock(spec=AsyncLLM)
|
||||
mock_engine.errored = False
|
||||
mock_engine.model_config = MockModelConfig()
|
||||
mock_engine.model_config.hf_config = MockHFConfig(model_type="gpt_oss")
|
||||
mock_engine.model_config.hf_text_config = MockHFConfig(model_type="gpt_oss")
|
||||
mock_engine.input_processor = MagicMock()
|
||||
mock_engine.renderer = _build_renderer(mock_engine.model_config)
|
||||
return mock_engine
|
||||
|
||||
@pytest.fixture()
|
||||
def serving_chat(self, mock_engine) -> OpenAIServingChat:
|
||||
chat = _build_serving_chat(mock_engine)
|
||||
chat.use_harmony = True
|
||||
chat.tool_parser = ToolParserManager.get_tool_parser("openai")
|
||||
chat = _build_serving_chat(
|
||||
mock_engine,
|
||||
reasoning_parser="openai_gptoss",
|
||||
tool_parser="openai",
|
||||
enable_auto_tools=True,
|
||||
)
|
||||
assert chat.parser_cls is HarmonyParser
|
||||
return chat
|
||||
|
||||
def mock_request_output_from_req_and_token_ids(
|
||||
@@ -1277,6 +1292,7 @@ class TestServingChatWithHarmony:
|
||||
stream: bool = False,
|
||||
) -> ChatCompletionResponse:
|
||||
harmony_token_ids = get_encoding().encode(harmony_str, allowed_special="all")
|
||||
tokenizer = get_tokenizer(GPT_OSS_MODEL_NAME)
|
||||
|
||||
async def result_generator():
|
||||
if stream:
|
||||
@@ -1304,11 +1320,12 @@ class TestServingChatWithHarmony:
|
||||
request_id=req.request_id,
|
||||
model_name=req.model,
|
||||
conversation=[],
|
||||
tokenizer=get_tokenizer(req.model),
|
||||
tokenizer=tokenizer,
|
||||
request_metadata=RequestResponseMetadata(
|
||||
request_id=req.request_id,
|
||||
model_name=req.model,
|
||||
),
|
||||
chat_template_kwargs=serving_chat._effective_chat_template_kwargs(req),
|
||||
)
|
||||
|
||||
if stream:
|
||||
@@ -1316,11 +1333,18 @@ class TestServingChatWithHarmony:
|
||||
return await result
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_simple_chat(self, serving_chat, stream):
|
||||
@pytest.mark.parametrize(
|
||||
"include_reasoning", [True, False], ids=["with_reasoning", "no_reasoning"]
|
||||
)
|
||||
async def test_simple_chat(self, serving_chat, stream, include_reasoning):
|
||||
messages = [{"role": "user", "content": "what is 1+1?"}]
|
||||
|
||||
# Test the Harmony messages for the first turn's input
|
||||
req = ChatCompletionRequest(model=MODEL_NAME, messages=messages)
|
||||
req = ChatCompletionRequest(
|
||||
model=MODEL_NAME,
|
||||
messages=messages,
|
||||
include_reasoning=include_reasoning,
|
||||
)
|
||||
input_messages, _ = (
|
||||
serving_chat.openai_serving_render._make_request_with_harmony(req)
|
||||
)
|
||||
@@ -1342,7 +1366,11 @@ class TestServingChatWithHarmony:
|
||||
response = await self.generate_response_from_harmony_str(
|
||||
serving_chat, req, response_str, stream=stream
|
||||
)
|
||||
verify_chat_response(response, content=final_str, reasoning=reasoning_str)
|
||||
verify_chat_response(
|
||||
response,
|
||||
content=final_str,
|
||||
reasoning=reasoning_str if include_reasoning else None,
|
||||
)
|
||||
|
||||
# Add the output messages from the first turn as input to the second turn
|
||||
for choice in response.choices:
|
||||
|
||||
@@ -11,12 +11,10 @@ from vllm.entrypoints.openai.parser.harmony_utils import (
|
||||
auto_drop_analysis_messages,
|
||||
create_tool_definition,
|
||||
extract_function_from_recipient,
|
||||
get_encoding,
|
||||
get_system_message,
|
||||
has_custom_tools,
|
||||
is_function_recipient,
|
||||
parse_chat_input_to_harmony_message,
|
||||
parse_chat_output,
|
||||
)
|
||||
from vllm.entrypoints.openai.responses.harmony import (
|
||||
response_input_to_harmony,
|
||||
@@ -941,110 +939,6 @@ class TestAutoDropAnalysisMessages:
|
||||
assert cleaned_messages == messages[1:]
|
||||
|
||||
|
||||
class TestParseChatOutput:
|
||||
def test_parse_chat_output_interrupted_first_message(self) -> None:
|
||||
harmony_str = "<|channel|>final<|message|>I'm in the middle of answering"
|
||||
token_ids = get_encoding().encode(harmony_str, allowed_special="all")
|
||||
reasoning, final_content, _ = parse_chat_output(token_ids)
|
||||
assert reasoning is None
|
||||
assert final_content == "I'm in the middle of answering"
|
||||
|
||||
def test_parse_chat_output_interrupted_reasoning_first_message(self) -> None:
|
||||
harmony_str = "<|channel|>analysis<|message|>I'm in the middle of thinking"
|
||||
token_ids = get_encoding().encode(harmony_str, allowed_special="all")
|
||||
reasoning, final_content, _ = parse_chat_output(token_ids)
|
||||
assert reasoning == "I'm in the middle of thinking"
|
||||
assert final_content is None
|
||||
|
||||
def test_parse_chat_output_complete_reasoning_interrupted_content(self) -> None:
|
||||
harmony_str = (
|
||||
"<|channel|>analysis<|message|>I'm thinking.<|end|>"
|
||||
"<|start|>assistant<|channel|>final"
|
||||
"<|message|>I'm in the middle of answering"
|
||||
)
|
||||
token_ids = get_encoding().encode(harmony_str, allowed_special="all")
|
||||
reasoning, final_content, _ = parse_chat_output(token_ids)
|
||||
assert reasoning == "I'm thinking."
|
||||
assert final_content == "I'm in the middle of answering"
|
||||
|
||||
def test_parse_chat_output_complete_content(self) -> None:
|
||||
harmony_str = "<|channel|>final<|message|>The answer is 4.<|end|>"
|
||||
token_ids = get_encoding().encode(harmony_str, allowed_special="all")
|
||||
reasoning, final_content, _ = parse_chat_output(token_ids)
|
||||
assert reasoning is None
|
||||
assert final_content == "The answer is 4."
|
||||
|
||||
def test_parse_chat_output_complete_commentary(self) -> None:
|
||||
harmony_str = (
|
||||
"<|channel|>commentary<|message|>I need to call some tools.<|end|>"
|
||||
)
|
||||
token_ids = get_encoding().encode(harmony_str, allowed_special="all")
|
||||
reasoning, final_content, _ = parse_chat_output(token_ids)
|
||||
assert reasoning is None
|
||||
assert final_content == "I need to call some tools."
|
||||
|
||||
def test_parse_chat_output_complete_reasoning(self) -> None:
|
||||
harmony_str = (
|
||||
"<|channel|>analysis<|message|>I've thought hard about this.<|end|>"
|
||||
)
|
||||
token_ids = get_encoding().encode(harmony_str, allowed_special="all")
|
||||
reasoning, final_content, _ = parse_chat_output(token_ids)
|
||||
assert reasoning == "I've thought hard about this."
|
||||
assert final_content is None
|
||||
|
||||
def test_parse_chat_output_complete_reasoning_and_content(self) -> None:
|
||||
harmony_str = (
|
||||
"<|channel|>analysis<|message|>I've thought hard about this.<|end|>"
|
||||
"<|start|>assistant<|channel|>final<|message|>The answer is 4.<|end|>"
|
||||
)
|
||||
token_ids = get_encoding().encode(harmony_str, allowed_special="all")
|
||||
reasoning, final_content, _ = parse_chat_output(token_ids)
|
||||
assert reasoning == "I've thought hard about this."
|
||||
assert final_content == "The answer is 4."
|
||||
|
||||
def test_parse_chat_output_commentary_with_recipient_excluded(self) -> None:
|
||||
"""Commentary with a recipient (tool call) should not appear in
|
||||
final_content — those are handled separately by the tool parser.
|
||||
|
||||
The first message is a preamble (visible), the second is a tool
|
||||
call (excluded). Only the preamble should appear in final_content.
|
||||
"""
|
||||
harmony_str = (
|
||||
"<|channel|>commentary"
|
||||
"<|message|>Let me check the weather.<|end|>"
|
||||
"<|start|>assistant to=functions.get_weather"
|
||||
"<|channel|>commentary"
|
||||
'<|message|>{"location": "SF"}<|end|>'
|
||||
)
|
||||
token_ids = get_encoding().encode(harmony_str, allowed_special="all")
|
||||
reasoning, final_content, _ = parse_chat_output(token_ids)
|
||||
assert reasoning is None
|
||||
assert final_content == "Let me check the weather."
|
||||
|
||||
def test_parse_chat_output_interrupted_preamble(self) -> None:
|
||||
"""Partial/interrupted preamble (commentary without recipient) should
|
||||
appear in final_content, not reasoning."""
|
||||
harmony_str = "<|channel|>commentary<|message|>I'll search for that"
|
||||
token_ids = get_encoding().encode(harmony_str, allowed_special="all")
|
||||
reasoning, final_content, _ = parse_chat_output(token_ids)
|
||||
assert reasoning is None
|
||||
assert final_content == "I'll search for that"
|
||||
|
||||
def test_parse_chat_output_preamble_then_final(self) -> None:
|
||||
"""Preamble followed by a final message should both appear in
|
||||
final_content, joined by newline."""
|
||||
harmony_str = (
|
||||
"<|channel|>commentary"
|
||||
"<|message|>Let me look that up.<|end|>"
|
||||
"<|start|>assistant<|channel|>final"
|
||||
"<|message|>The answer is 42.<|end|>"
|
||||
)
|
||||
token_ids = get_encoding().encode(harmony_str, allowed_special="all")
|
||||
reasoning, final_content, _ = parse_chat_output(token_ids)
|
||||
assert reasoning is None
|
||||
assert final_content == "Let me look that up.\nThe answer is 42."
|
||||
|
||||
|
||||
def test_has_custom_tools() -> None:
|
||||
assert not has_custom_tools(set())
|
||||
assert not has_custom_tools({"web_search_preview", "code_interpreter", "container"})
|
||||
|
||||
Reference in New Issue
Block a user