[Frontend] Make finish_reason and Responses item status truthful (#59324)
Signed-off-by: Flora Feng <4florafeng@gmail.com> Signed-off-by: sfeng33 <4florafeng@gmail.com> Co-authored-by: Chauncey <chaunceyjiang@gmail.com>
This commit is contained in:
@@ -42,6 +42,7 @@ from vllm.entrypoints.openai.models.serving import (
|
||||
)
|
||||
from vllm.entrypoints.openai.parser.harmony_utils import get_encoding
|
||||
from vllm.entrypoints.serve.engine.protocol import ErrorResponse
|
||||
from vllm.entrypoints.serve.utils.tool_calls_utils import resolve_finish_reason
|
||||
from vllm.exceptions import QueueOverflowError, VLLMValidationError
|
||||
from vllm.inputs import TokensPrompt
|
||||
from vllm.logprobs import Logprob
|
||||
@@ -3051,3 +3052,137 @@ def test_chat_kv_transfer_prompt_token_ids_allows_text_only_parts():
|
||||
kv_transfer_params={"prompt_token_ids": [10, 20, 30]},
|
||||
)
|
||||
assert request.kv_transfer_params == {"prompt_token_ids": [10, 20, 30]}
|
||||
|
||||
|
||||
_TOOL_FINISH_REASON_TOOLS = [
|
||||
{
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "get_weather",
|
||||
"description": "Get weather",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {"city": {"type": "string"}},
|
||||
"required": ["city"],
|
||||
},
|
||||
},
|
||||
}
|
||||
]
|
||||
_HERMES_TOOL_CALL_TEXT = (
|
||||
'<tool_call>\n{"name": "get_weather", "arguments": {"city": "Tokyo"}}\n</tool_call>'
|
||||
)
|
||||
|
||||
|
||||
_NAMED_WEATHER_TOOL_CHOICE = {"type": "function", "function": {"name": "get_weather"}}
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("tool_choice", "engine_finish_reason", "tool_calls_made", "expected"),
|
||||
[
|
||||
("auto", "stop", True, "tool_calls"),
|
||||
("auto", "length", True, "length"),
|
||||
("auto", "stop", False, "stop"),
|
||||
("required", "stop", True, "tool_calls"),
|
||||
("required", "stop", False, "stop"),
|
||||
("required", "length", True, "length"),
|
||||
(_NAMED_WEATHER_TOOL_CHOICE, "stop", True, "stop"),
|
||||
(_NAMED_WEATHER_TOOL_CHOICE, "length", True, "length"),
|
||||
("none", "stop", False, "stop"),
|
||||
(None, None, False, "stop"),
|
||||
],
|
||||
)
|
||||
def test_resolve_finish_reason(
|
||||
tool_choice, engine_finish_reason, tool_calls_made, expected
|
||||
):
|
||||
"""``tool_calls`` needs a real call, a natural stop and no forced tool;
|
||||
``length`` is never hidden behind a parsed call."""
|
||||
request = ChatCompletionRequest(
|
||||
model=MODEL_NAME,
|
||||
messages=[{"role": "user", "content": "test"}],
|
||||
tools=_TOOL_FINISH_REASON_TOOLS if tool_choice else None,
|
||||
tool_choice=tool_choice,
|
||||
)
|
||||
assert (
|
||||
resolve_finish_reason(engine_finish_reason, request, tool_calls_made)
|
||||
== expected
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
("tool_choice", "text", "engine_finish_reason", "expected_finish_reason"),
|
||||
[
|
||||
("auto", _HERMES_TOOL_CALL_TEXT, "stop", "tool_calls"),
|
||||
("auto", _HERMES_TOOL_CALL_TEXT, "length", "length"),
|
||||
("required", "Hello, how are you?", "stop", "stop"),
|
||||
(_NAMED_WEATHER_TOOL_CHOICE, _HERMES_TOOL_CALL_TEXT, "stop", "stop"),
|
||||
],
|
||||
)
|
||||
async def test_non_streaming_finish_reason_matches_tool_calls(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
tool_choice,
|
||||
text: str,
|
||||
engine_finish_reason: str,
|
||||
expected_finish_reason: str,
|
||||
):
|
||||
"""Non-streaming ``finish_reason`` must agree with ``message.tool_calls``
|
||||
and never hide a ``length`` truncation (#53255, #58228)."""
|
||||
serving_chat = _build_serving_chat(
|
||||
_build_mock_engine(), tool_parser="hermes", enable_auto_tools=True
|
||||
)
|
||||
# Parse required/named calls from the native <tool_call> format regardless
|
||||
# of VLLM_ENFORCE_STRICT_TOOL_CALLING, so one text serves every case.
|
||||
monkeypatch.setattr(
|
||||
serving_chat.parser_cls.tool_parser_cls, "supports_required_and_named", False
|
||||
)
|
||||
|
||||
tokenizer = get_tokenizer(MODEL_NAME)
|
||||
request = ChatCompletionRequest(
|
||||
model=MODEL_NAME,
|
||||
messages=[{"role": "user", "content": "test"}],
|
||||
tools=_TOOL_FINISH_REASON_TOOLS,
|
||||
tool_choice=tool_choice,
|
||||
)
|
||||
parser = serving_chat.parser_cls(
|
||||
tokenizer,
|
||||
request.tools,
|
||||
chat_template_kwargs=serving_chat._effective_chat_template_kwargs(request),
|
||||
model_config=serving_chat.model_config,
|
||||
)
|
||||
request_output = RequestOutput(
|
||||
request_id="test-req",
|
||||
prompt="test",
|
||||
prompt_token_ids=[1, 2, 3],
|
||||
prompt_logprobs=None,
|
||||
outputs=[
|
||||
CompletionOutput(
|
||||
index=0,
|
||||
text=text,
|
||||
token_ids=tokenizer.encode(text, add_special_tokens=False),
|
||||
cumulative_logprob=0.0,
|
||||
logprobs=None,
|
||||
finish_reason=engine_finish_reason,
|
||||
)
|
||||
],
|
||||
finished=True,
|
||||
)
|
||||
|
||||
response = await serving_chat.chat_completion_full_generator(
|
||||
request=request,
|
||||
result_generator=_single_request_output(request_output),
|
||||
request_id="test-req",
|
||||
model_name=MODEL_NAME,
|
||||
conversation=[],
|
||||
tokenizer=tokenizer,
|
||||
request_metadata=RequestResponseMetadata(
|
||||
request_id="test-req", model_name=MODEL_NAME
|
||||
),
|
||||
parser=parser,
|
||||
)
|
||||
|
||||
assert isinstance(response, ChatCompletionResponse)
|
||||
choice = response.choices[0]
|
||||
assert choice.finish_reason == expected_finish_reason
|
||||
assert bool(choice.message.tool_calls) == (text != "Hello, how are you?"), (
|
||||
"finish_reason must reflect whether tool calls were emitted"
|
||||
)
|
||||
|
||||
@@ -456,3 +456,27 @@ def test_num_init_messages_offset():
|
||||
items = ctx.make_response_output_items()
|
||||
assert len(items) == 1
|
||||
assert items[0].type == "message"
|
||||
|
||||
|
||||
def test_length_truncation_marks_last_item_incomplete():
|
||||
"""A turn cut by max_output_tokens ends in an ``incomplete`` item, both
|
||||
with a parser (tool call) and without one (plain message)."""
|
||||
request = _make_request(
|
||||
tool_choice="auto",
|
||||
tools=[
|
||||
{
|
||||
"type": "function",
|
||||
"name": "get_weather",
|
||||
"parameters": {"type": "object", "properties": {}},
|
||||
}
|
||||
],
|
||||
)
|
||||
ctx = _make_context(_ToolCallingParser, request=request, enable_auto_tools=True)
|
||||
ctx.append_output(_make_request_output(text="calling tool", finish_reason="length"))
|
||||
assert ctx.response_messages[-1].type == "function_call"
|
||||
assert ctx.response_messages[-1].status == "incomplete"
|
||||
|
||||
ctx = _make_context(None)
|
||||
ctx.append_output(_make_request_output(text="cut mid sen", finish_reason="length"))
|
||||
assert ctx.response_messages[-1].type == "message"
|
||||
assert ctx.response_messages[-1].status == "incomplete"
|
||||
|
||||
@@ -991,3 +991,28 @@ class TestConstructInputMessagesInstructionsLeak:
|
||||
assert len(msgs) == 2
|
||||
assert msgs[0] == {"role": "system", "content": "be helpful"}
|
||||
assert msgs[1] == {"role": "user", "content": "hello"}
|
||||
|
||||
|
||||
class TestBuildResponseOutputItemsStatus:
|
||||
"""Item ``status`` must follow the finish reason of the generation
|
||||
(#57998): a ``length`` cut marks the item that was cut ``incomplete``."""
|
||||
|
||||
_CALLS = [
|
||||
FunctionCall(name="get_weather", arguments='{"city": "Paris"}'),
|
||||
FunctionCall(name="get_weather", arguments='{"city": "Tok'),
|
||||
]
|
||||
|
||||
def test_length_marks_truncated_message_incomplete(self):
|
||||
(item,) = build_response_output_items(
|
||||
reasoning=None, content="cut mid sen", tool_calls=[], finish_reason="length"
|
||||
)
|
||||
assert item.type == "message"
|
||||
assert item.status == "incomplete"
|
||||
|
||||
def test_length_marks_only_last_tool_call_incomplete(self):
|
||||
message, first, last = build_response_output_items(
|
||||
reasoning=None, content="hi", tool_calls=self._CALLS, finish_reason="length"
|
||||
)
|
||||
assert message.status == "completed"
|
||||
assert first.status == "completed"
|
||||
assert last.status == "incomplete"
|
||||
|
||||
@@ -1,6 +1,8 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import asyncio
|
||||
from collections import deque
|
||||
from contextlib import AsyncExitStack, asynccontextmanager
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import AsyncMock, MagicMock
|
||||
@@ -51,6 +53,7 @@ from vllm.entrypoints.openai.responses.context import (
|
||||
from vllm.entrypoints.openai.responses.protocol import (
|
||||
ResponseCompletedEvent,
|
||||
ResponseCreatedEvent,
|
||||
ResponseIncompleteEvent,
|
||||
ResponseRawMessageAndToken,
|
||||
ResponsesRequest,
|
||||
ResponsesResponse,
|
||||
@@ -1414,6 +1417,7 @@ def _make_request_output(
|
||||
text,
|
||||
token_ids,
|
||||
metrics: RequestStateStats | None = None,
|
||||
finish_reason: str | None = None,
|
||||
):
|
||||
completion = CompletionOutput(
|
||||
index=0,
|
||||
@@ -1421,7 +1425,7 @@ def _make_request_output(
|
||||
token_ids=token_ids,
|
||||
cumulative_logprob=0.0,
|
||||
logprobs=None,
|
||||
finish_reason=None,
|
||||
finish_reason=finish_reason,
|
||||
stop_reason=None,
|
||||
)
|
||||
return RequestOutput(
|
||||
@@ -1441,10 +1445,11 @@ def _make_simple_context_with_output(
|
||||
token_ids,
|
||||
response_parser=None,
|
||||
metrics: RequestStateStats | None = None,
|
||||
finish_reason: str | None = None,
|
||||
):
|
||||
"""Create a SimpleContext with a RequestOutput containing the given text."""
|
||||
ctx = SimpleContext(response_parser=response_parser)
|
||||
req_output = _make_request_output(text, token_ids, metrics)
|
||||
req_output = _make_request_output(text, token_ids, metrics, finish_reason)
|
||||
ctx.append_output(req_output)
|
||||
ctx.request_metrics = metrics
|
||||
return ctx
|
||||
@@ -1561,6 +1566,66 @@ async def test_responses_streaming_metrics_only_on_completed_event():
|
||||
assert events[-1].response.metrics.time_to_first_token_ms == pytest.approx(500.0)
|
||||
|
||||
|
||||
async def _stream_truncated_by_length(serving: OpenAIServingResponses):
|
||||
request = ResponsesRequest(input="hi", tools=[], stream=True, store=False)
|
||||
context = _make_simple_context_with_output(
|
||||
"cut mid sen", [10, 20], finish_reason="length"
|
||||
)
|
||||
|
||||
async def one_context():
|
||||
yield context
|
||||
|
||||
return [
|
||||
event
|
||||
async for event in serving.responses_stream_generator(
|
||||
request=request,
|
||||
sampling_params=SamplingParams(max_tokens=2),
|
||||
result_generator=one_context(),
|
||||
context=context,
|
||||
model_name="test-model",
|
||||
tokenizer=MagicMock(),
|
||||
request_metadata=RequestResponseMetadata(request_id="req"),
|
||||
)
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_responses_stream_truncated_by_length_reports_incomplete():
|
||||
"""A max_output_tokens cut must end the stream with ``response.incomplete``
|
||||
and mark the cut item ``incomplete`` in both ``output_item.done`` and the
|
||||
final response (#57998)."""
|
||||
events = await _stream_truncated_by_length(_make_serving_instance())
|
||||
|
||||
item_done = [e for e in events if e.type == "response.output_item.done"]
|
||||
assert [e.item.status for e in item_done] == ["incomplete"]
|
||||
final = events[-1]
|
||||
assert isinstance(final, ResponseIncompleteEvent)
|
||||
assert final.response.status == "incomplete"
|
||||
assert final.response.incomplete_details.reason == "max_output_tokens"
|
||||
assert [item.status for item in final.response.output] == ["incomplete"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_background_stream_replay_stops_on_response_incomplete():
|
||||
"""Replaying a stored background stream must end at ``response.incomplete``
|
||||
as it does at ``response.completed``, instead of waiting for more events."""
|
||||
serving = _make_serving_instance()
|
||||
events = await _stream_truncated_by_length(serving)
|
||||
assert isinstance(events[-1], ResponseIncompleteEvent)
|
||||
serving.event_store["resp_bg"] = (deque(events), asyncio.Event())
|
||||
|
||||
async def replay(starting_after=None):
|
||||
return [
|
||||
event
|
||||
async for event in serving.responses_background_stream_generator(
|
||||
"resp_bg", starting_after=starting_after
|
||||
)
|
||||
]
|
||||
|
||||
assert await asyncio.wait_for(replay(), timeout=5) == events
|
||||
assert await asyncio.wait_for(replay(starting_after=0), timeout=5) == events[1:]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_responses_metrics_suppressed_for_multiple_generation_turns():
|
||||
response = await _make_full_metrics_response(
|
||||
|
||||
@@ -3,11 +3,18 @@
|
||||
|
||||
"""Tests for the /derender endpoints (postprocessing counterpart to /render)."""
|
||||
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
import pytest_asyncio
|
||||
|
||||
from tests.utils import RemoteLaunchRenderServer
|
||||
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
|
||||
from vllm.entrypoints.scale_out.token_in_token_out.protocol import (
|
||||
GenerateTokensResponse,
|
||||
)
|
||||
from vllm.renderers.online_derenderer import OnlineDerenderer
|
||||
from vllm.tokenizers import get_tokenizer
|
||||
|
||||
MODEL_NAME = "hmellor/tiny-random-LlamaForCausalLM"
|
||||
@@ -1432,3 +1439,55 @@ async def test_e2e_harmony_reasoning(harmony_client, harmony_tokenizer):
|
||||
assert msg["reasoning"] is not None
|
||||
assert reasoning_text in msg["reasoning"]
|
||||
assert answer_text in (msg["content"] or "")
|
||||
|
||||
|
||||
_HERMES_TOOL_CALL_TEXT = (
|
||||
'<tool_call>\n{"name": "get_weather", "arguments": {"city": "Paris"}}\n</tool_call>'
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("tool_choice", "text", "engine_finish_reason", "expected_finish_reason"),
|
||||
[
|
||||
("auto", _HERMES_TOOL_CALL_TEXT, "stop", "tool_calls"),
|
||||
("auto", _HERMES_TOOL_CALL_TEXT, "length", "length"),
|
||||
("required", "Hello there", "stop", "stop"),
|
||||
],
|
||||
)
|
||||
def test_derender_chat_finish_reason_matches_chat_completion(
|
||||
tool_choice, text, engine_finish_reason, expected_finish_reason
|
||||
):
|
||||
"""/derender applies the same finish_reason rule as the coupled chat
|
||||
path: ``tool_calls`` only for a real call at a natural stop."""
|
||||
tokenizer = get_tokenizer(MODEL_NAME)
|
||||
renderer = MagicMock()
|
||||
renderer.get_tokenizer.return_value = tokenizer
|
||||
model_config = MagicMock()
|
||||
model_config.model = MODEL_NAME
|
||||
model_config.hf_config.model_type = "llama"
|
||||
derenderer = OnlineDerenderer(
|
||||
model_config,
|
||||
renderer,
|
||||
request_logger=None,
|
||||
chat_template="{{ messages }}",
|
||||
chat_template_content_format="auto",
|
||||
enable_auto_tools=True,
|
||||
tool_parser="hermes",
|
||||
)
|
||||
request = ChatCompletionRequest(
|
||||
model=MODEL_NAME,
|
||||
messages=[{"role": "user", "content": "Weather in Paris?"}],
|
||||
tools=_E2E_TOOLS,
|
||||
tool_choice=tool_choice,
|
||||
)
|
||||
response = GenerateTokensResponse.model_validate(
|
||||
_make_generate_response(
|
||||
tokenizer.encode(text, add_special_tokens=False),
|
||||
finish_reason=engine_finish_reason,
|
||||
)
|
||||
)
|
||||
|
||||
(choice,) = derenderer._derender_chat(response, request)
|
||||
|
||||
assert choice.finish_reason == expected_finish_reason
|
||||
assert bool(choice.message.tool_calls) == (text == _HERMES_TOOL_CALL_TEXT)
|
||||
|
||||
@@ -1289,7 +1289,15 @@ class TestDerenderChatStreamParsed:
|
||||
assert state.last_tool_call_ids == [first_id]
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_finish_reason_rewritten_to_tool_calls(self, parsed_derenderer):
|
||||
@pytest.mark.parametrize(
|
||||
("engine_finish_reason", "expected_finish_reason"),
|
||||
[("stop", "tool_calls"), ("length", "length")],
|
||||
)
|
||||
async def test_finish_reason_rewritten_to_tool_calls(
|
||||
self, parsed_derenderer, engine_finish_reason, expected_finish_reason
|
||||
):
|
||||
"""A streamed tool call is reported as ``tool_calls`` only at a
|
||||
natural stop; a ``length`` cut must stay visible."""
|
||||
chat_request = _chat_request(
|
||||
tools=[{"type": "function", "function": {"name": "get_weather"}}],
|
||||
tool_choice="auto",
|
||||
@@ -1297,11 +1305,12 @@ class TestDerenderChatStreamParsed:
|
||||
chunk, _ = await parsed_derenderer.derender_chat_stream(
|
||||
model=MODEL_NAME,
|
||||
generate_chunk=_make_stream_chunk(
|
||||
[_FakeParser.TOOL_START, _FakeParser.TOOL_ARG], finish_reason="stop"
|
||||
[_FakeParser.TOOL_START, _FakeParser.TOOL_ARG],
|
||||
finish_reason=engine_finish_reason,
|
||||
),
|
||||
chat_request=chat_request,
|
||||
)
|
||||
assert chunk.choices[0].finish_reason == "tool_calls"
|
||||
assert chunk.choices[0].finish_reason == expected_finish_reason
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_finish_reason_stop_for_named_tool_choice(self, parsed_derenderer):
|
||||
|
||||
Reference in New Issue
Block a user