[Bugfix][Frontend] Apply Harmony adjust_request in batched chat completions (#58958)
Signed-off-by: sfeng33 <4florafeng@gmail.com>
This commit is contained in:
@@ -3,10 +3,20 @@
|
||||
|
||||
import json
|
||||
from collections.abc import AsyncGenerator
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
|
||||
from tests.entrypoints.openai.chat_completion.test_serving_chat import (
|
||||
BASE_MODEL_PATHS,
|
||||
MockHFConfig,
|
||||
MockModelConfig,
|
||||
_build_renderer,
|
||||
)
|
||||
from tests.entrypoints.openai.chat_completion.test_serving_chat import (
|
||||
MODEL_NAME as MOCK_MODEL_NAME,
|
||||
)
|
||||
from tests.utils import RemoteOpenAIServer
|
||||
from vllm.entrypoints.generate.base.protocol import RequestResponseMetadata
|
||||
from vllm.entrypoints.openai.chat_completion.batch_serving import (
|
||||
@@ -15,7 +25,10 @@ from vllm.entrypoints.openai.chat_completion.batch_serving import (
|
||||
from vllm.entrypoints.openai.chat_completion.protocol import (
|
||||
BatchChatCompletionRequest,
|
||||
)
|
||||
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
|
||||
from vllm.outputs import CompletionOutput, RequestOutput
|
||||
from vllm.renderers.online_renderer import OnlineRenderer
|
||||
from vllm.v1.engine.async_llm import AsyncLLM
|
||||
|
||||
# any model with a chat template defined in tokenizer_config should work here
|
||||
MODEL_NAME = "Qwen/Qwen2.5-1.5B-Instruct"
|
||||
@@ -322,3 +335,53 @@ async def test_batched_echo_prepends_matching_assistant_prefix() -> None:
|
||||
)
|
||||
|
||||
assert response.choices[0].message.content == "PREFIX ASSISTANT ANSWER"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.skip_global_cleanup
|
||||
async def test_batched_harmony_response_format_uses_structural_tag() -> None:
|
||||
"""Harmony must constrain only the final channel of every conversation."""
|
||||
mock_engine = MagicMock(spec=AsyncLLM)
|
||||
mock_engine.model_config = MockModelConfig()
|
||||
mock_engine.errored = False
|
||||
mock_engine.input_processor = MagicMock()
|
||||
mock_engine.model_config.hf_config = MockHFConfig(model_type="gpt_oss")
|
||||
mock_engine.renderer = _build_renderer(mock_engine.model_config)
|
||||
mock_engine.generate.side_effect = lambda *args, **kwargs: _generator(0, "ok")
|
||||
|
||||
models = OpenAIServingModels(mock_engine, BASE_MODEL_PATHS)
|
||||
online_renderer = OnlineRenderer(
|
||||
model_config=mock_engine.model_config,
|
||||
renderer=mock_engine.renderer,
|
||||
request_logger=None,
|
||||
chat_template=None,
|
||||
chat_template_content_format="auto",
|
||||
reasoning_parser="openai_gptoss",
|
||||
)
|
||||
serving = OpenAIServingChatBatch(
|
||||
mock_engine,
|
||||
models,
|
||||
response_role="assistant",
|
||||
online_renderer=online_renderer,
|
||||
chat_template=None,
|
||||
chat_template_content_format="auto",
|
||||
request_logger=None,
|
||||
)
|
||||
|
||||
request = BatchChatCompletionRequest(
|
||||
model=MOCK_MODEL_NAME,
|
||||
messages=[
|
||||
[{"role": "user", "content": "France?"}],
|
||||
[{"role": "user", "content": "Japan?"}],
|
||||
],
|
||||
response_format={
|
||||
"type": "json_schema",
|
||||
"json_schema": {"name": "city", "schema": {"type": "object"}},
|
||||
},
|
||||
)
|
||||
await serving.create_batch_chat_completion(request)
|
||||
|
||||
calls = mock_engine.generate.call_args_list
|
||||
assert len(calls) == 2
|
||||
for call in calls:
|
||||
assert call.args[1].structured_outputs.structural_tag is not None
|
||||
|
||||
Reference in New Issue
Block a user