[Refactor] Chat Completions Harmony Refactor, non-streaming path. (#45171)
Signed-off-by: Yifan Zong <yzong@redhat.com>
This commit is contained in:
@@ -38,12 +38,12 @@ from vllm.entrypoints.serve.render.serving import OpenAIServingRender
|
||||
from vllm.exceptions import VLLMValidationError
|
||||
from vllm.inputs import TokensPrompt
|
||||
from vllm.outputs import CompletionOutput, RequestOutput
|
||||
from vllm.parser import HarmonyParser
|
||||
from vllm.renderers.hf import HfRenderer
|
||||
from vllm.renderers.mistral import MistralRenderer
|
||||
from vllm.tokenizers import get_tokenizer
|
||||
from vllm.tokenizers.mistral import MistralTokenizer
|
||||
from vllm.tokenizers.registry import cached_tokenizer_from_config
|
||||
from vllm.tool_parsers import ToolParserManager
|
||||
from vllm.v1.engine.async_llm import AsyncLLM
|
||||
|
||||
GPT_OSS_MODEL_NAME = "openai/gpt-oss-20b"
|
||||
@@ -575,7 +575,13 @@ def _build_serving_render(
|
||||
)
|
||||
|
||||
|
||||
def _build_serving_chat(engine: AsyncLLM) -> OpenAIServingChat:
|
||||
def _build_serving_chat(
|
||||
engine: AsyncLLM,
|
||||
*,
|
||||
reasoning_parser: str = "",
|
||||
tool_parser: str | None = None,
|
||||
enable_auto_tools: bool = False,
|
||||
) -> OpenAIServingChat:
|
||||
models = OpenAIServingModels(
|
||||
engine_client=engine,
|
||||
base_model_paths=BASE_MODEL_PATHS,
|
||||
@@ -590,6 +596,9 @@ def _build_serving_chat(engine: AsyncLLM) -> OpenAIServingChat:
|
||||
chat_template=CHAT_TEMPLATE,
|
||||
chat_template_content_format="auto",
|
||||
request_logger=None,
|
||||
reasoning_parser=reasoning_parser,
|
||||
tool_parser=tool_parser,
|
||||
enable_auto_tools=enable_auto_tools,
|
||||
)
|
||||
|
||||
return serving_chat
|
||||
@@ -637,7 +646,7 @@ async def test_serving_chat_returns_correct_model_name():
|
||||
serving_chat = _build_serving_chat(mock_engine)
|
||||
messages = [{"role": "user", "content": "what is 1+1?"}]
|
||||
|
||||
async def return_model_name(*args):
|
||||
async def return_model_name(*args, **kwargs):
|
||||
return args[3]
|
||||
|
||||
serving_chat.chat_completion_full_generator = return_model_name
|
||||
@@ -1210,15 +1219,21 @@ class TestServingChatWithHarmony:
|
||||
mock_engine = MagicMock(spec=AsyncLLM)
|
||||
mock_engine.errored = False
|
||||
mock_engine.model_config = MockModelConfig()
|
||||
mock_engine.model_config.hf_config = MockHFConfig(model_type="gpt_oss")
|
||||
mock_engine.model_config.hf_text_config = MockHFConfig(model_type="gpt_oss")
|
||||
mock_engine.input_processor = MagicMock()
|
||||
mock_engine.renderer = _build_renderer(mock_engine.model_config)
|
||||
return mock_engine
|
||||
|
||||
@pytest.fixture()
|
||||
def serving_chat(self, mock_engine) -> OpenAIServingChat:
|
||||
chat = _build_serving_chat(mock_engine)
|
||||
chat.use_harmony = True
|
||||
chat.tool_parser = ToolParserManager.get_tool_parser("openai")
|
||||
chat = _build_serving_chat(
|
||||
mock_engine,
|
||||
reasoning_parser="openai_gptoss",
|
||||
tool_parser="openai",
|
||||
enable_auto_tools=True,
|
||||
)
|
||||
assert chat.parser_cls is HarmonyParser
|
||||
return chat
|
||||
|
||||
def mock_request_output_from_req_and_token_ids(
|
||||
@@ -1277,6 +1292,7 @@ class TestServingChatWithHarmony:
|
||||
stream: bool = False,
|
||||
) -> ChatCompletionResponse:
|
||||
harmony_token_ids = get_encoding().encode(harmony_str, allowed_special="all")
|
||||
tokenizer = get_tokenizer(GPT_OSS_MODEL_NAME)
|
||||
|
||||
async def result_generator():
|
||||
if stream:
|
||||
@@ -1304,11 +1320,12 @@ class TestServingChatWithHarmony:
|
||||
request_id=req.request_id,
|
||||
model_name=req.model,
|
||||
conversation=[],
|
||||
tokenizer=get_tokenizer(req.model),
|
||||
tokenizer=tokenizer,
|
||||
request_metadata=RequestResponseMetadata(
|
||||
request_id=req.request_id,
|
||||
model_name=req.model,
|
||||
),
|
||||
chat_template_kwargs=serving_chat._effective_chat_template_kwargs(req),
|
||||
)
|
||||
|
||||
if stream:
|
||||
@@ -1316,11 +1333,18 @@ class TestServingChatWithHarmony:
|
||||
return await result
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_simple_chat(self, serving_chat, stream):
|
||||
@pytest.mark.parametrize(
|
||||
"include_reasoning", [True, False], ids=["with_reasoning", "no_reasoning"]
|
||||
)
|
||||
async def test_simple_chat(self, serving_chat, stream, include_reasoning):
|
||||
messages = [{"role": "user", "content": "what is 1+1?"}]
|
||||
|
||||
# Test the Harmony messages for the first turn's input
|
||||
req = ChatCompletionRequest(model=MODEL_NAME, messages=messages)
|
||||
req = ChatCompletionRequest(
|
||||
model=MODEL_NAME,
|
||||
messages=messages,
|
||||
include_reasoning=include_reasoning,
|
||||
)
|
||||
input_messages, _ = (
|
||||
serving_chat.openai_serving_render._make_request_with_harmony(req)
|
||||
)
|
||||
@@ -1342,7 +1366,11 @@ class TestServingChatWithHarmony:
|
||||
response = await self.generate_response_from_harmony_str(
|
||||
serving_chat, req, response_str, stream=stream
|
||||
)
|
||||
verify_chat_response(response, content=final_str, reasoning=reasoning_str)
|
||||
verify_chat_response(
|
||||
response,
|
||||
content=final_str,
|
||||
reasoning=reasoning_str if include_reasoning else None,
|
||||
)
|
||||
|
||||
# Add the output messages from the first turn as input to the second turn
|
||||
for choice in response.choices:
|
||||
|
||||
Reference in New Issue
Block a user