[Frontend] Make finish_reason and Responses item status truthful (#59324)
Signed-off-by: Flora Feng <4florafeng@gmail.com> Signed-off-by: sfeng33 <4florafeng@gmail.com> Co-authored-by: Chauncey <chaunceyjiang@gmail.com>
This commit is contained in:
@@ -42,6 +42,7 @@ from vllm.entrypoints.openai.models.serving import (
|
|||||||
)
|
)
|
||||||
from vllm.entrypoints.openai.parser.harmony_utils import get_encoding
|
from vllm.entrypoints.openai.parser.harmony_utils import get_encoding
|
||||||
from vllm.entrypoints.serve.engine.protocol import ErrorResponse
|
from vllm.entrypoints.serve.engine.protocol import ErrorResponse
|
||||||
|
from vllm.entrypoints.serve.utils.tool_calls_utils import resolve_finish_reason
|
||||||
from vllm.exceptions import QueueOverflowError, VLLMValidationError
|
from vllm.exceptions import QueueOverflowError, VLLMValidationError
|
||||||
from vllm.inputs import TokensPrompt
|
from vllm.inputs import TokensPrompt
|
||||||
from vllm.logprobs import Logprob
|
from vllm.logprobs import Logprob
|
||||||
@@ -3051,3 +3052,137 @@ def test_chat_kv_transfer_prompt_token_ids_allows_text_only_parts():
|
|||||||
kv_transfer_params={"prompt_token_ids": [10, 20, 30]},
|
kv_transfer_params={"prompt_token_ids": [10, 20, 30]},
|
||||||
)
|
)
|
||||||
assert request.kv_transfer_params == {"prompt_token_ids": [10, 20, 30]}
|
assert request.kv_transfer_params == {"prompt_token_ids": [10, 20, 30]}
|
||||||
|
|
||||||
|
|
||||||
|
_TOOL_FINISH_REASON_TOOLS = [
|
||||||
|
{
|
||||||
|
"type": "function",
|
||||||
|
"function": {
|
||||||
|
"name": "get_weather",
|
||||||
|
"description": "Get weather",
|
||||||
|
"parameters": {
|
||||||
|
"type": "object",
|
||||||
|
"properties": {"city": {"type": "string"}},
|
||||||
|
"required": ["city"],
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
]
|
||||||
|
_HERMES_TOOL_CALL_TEXT = (
|
||||||
|
'<tool_call>\n{"name": "get_weather", "arguments": {"city": "Tokyo"}}\n</tool_call>'
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
_NAMED_WEATHER_TOOL_CHOICE = {"type": "function", "function": {"name": "get_weather"}}
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
("tool_choice", "engine_finish_reason", "tool_calls_made", "expected"),
|
||||||
|
[
|
||||||
|
("auto", "stop", True, "tool_calls"),
|
||||||
|
("auto", "length", True, "length"),
|
||||||
|
("auto", "stop", False, "stop"),
|
||||||
|
("required", "stop", True, "tool_calls"),
|
||||||
|
("required", "stop", False, "stop"),
|
||||||
|
("required", "length", True, "length"),
|
||||||
|
(_NAMED_WEATHER_TOOL_CHOICE, "stop", True, "stop"),
|
||||||
|
(_NAMED_WEATHER_TOOL_CHOICE, "length", True, "length"),
|
||||||
|
("none", "stop", False, "stop"),
|
||||||
|
(None, None, False, "stop"),
|
||||||
|
],
|
||||||
|
)
|
||||||
|
def test_resolve_finish_reason(
|
||||||
|
tool_choice, engine_finish_reason, tool_calls_made, expected
|
||||||
|
):
|
||||||
|
"""``tool_calls`` needs a real call, a natural stop and no forced tool;
|
||||||
|
``length`` is never hidden behind a parsed call."""
|
||||||
|
request = ChatCompletionRequest(
|
||||||
|
model=MODEL_NAME,
|
||||||
|
messages=[{"role": "user", "content": "test"}],
|
||||||
|
tools=_TOOL_FINISH_REASON_TOOLS if tool_choice else None,
|
||||||
|
tool_choice=tool_choice,
|
||||||
|
)
|
||||||
|
assert (
|
||||||
|
resolve_finish_reason(engine_finish_reason, request, tool_calls_made)
|
||||||
|
== expected
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
("tool_choice", "text", "engine_finish_reason", "expected_finish_reason"),
|
||||||
|
[
|
||||||
|
("auto", _HERMES_TOOL_CALL_TEXT, "stop", "tool_calls"),
|
||||||
|
("auto", _HERMES_TOOL_CALL_TEXT, "length", "length"),
|
||||||
|
("required", "Hello, how are you?", "stop", "stop"),
|
||||||
|
(_NAMED_WEATHER_TOOL_CHOICE, _HERMES_TOOL_CALL_TEXT, "stop", "stop"),
|
||||||
|
],
|
||||||
|
)
|
||||||
|
async def test_non_streaming_finish_reason_matches_tool_calls(
|
||||||
|
monkeypatch: pytest.MonkeyPatch,
|
||||||
|
tool_choice,
|
||||||
|
text: str,
|
||||||
|
engine_finish_reason: str,
|
||||||
|
expected_finish_reason: str,
|
||||||
|
):
|
||||||
|
"""Non-streaming ``finish_reason`` must agree with ``message.tool_calls``
|
||||||
|
and never hide a ``length`` truncation (#53255, #58228)."""
|
||||||
|
serving_chat = _build_serving_chat(
|
||||||
|
_build_mock_engine(), tool_parser="hermes", enable_auto_tools=True
|
||||||
|
)
|
||||||
|
# Parse required/named calls from the native <tool_call> format regardless
|
||||||
|
# of VLLM_ENFORCE_STRICT_TOOL_CALLING, so one text serves every case.
|
||||||
|
monkeypatch.setattr(
|
||||||
|
serving_chat.parser_cls.tool_parser_cls, "supports_required_and_named", False
|
||||||
|
)
|
||||||
|
|
||||||
|
tokenizer = get_tokenizer(MODEL_NAME)
|
||||||
|
request = ChatCompletionRequest(
|
||||||
|
model=MODEL_NAME,
|
||||||
|
messages=[{"role": "user", "content": "test"}],
|
||||||
|
tools=_TOOL_FINISH_REASON_TOOLS,
|
||||||
|
tool_choice=tool_choice,
|
||||||
|
)
|
||||||
|
parser = serving_chat.parser_cls(
|
||||||
|
tokenizer,
|
||||||
|
request.tools,
|
||||||
|
chat_template_kwargs=serving_chat._effective_chat_template_kwargs(request),
|
||||||
|
model_config=serving_chat.model_config,
|
||||||
|
)
|
||||||
|
request_output = RequestOutput(
|
||||||
|
request_id="test-req",
|
||||||
|
prompt="test",
|
||||||
|
prompt_token_ids=[1, 2, 3],
|
||||||
|
prompt_logprobs=None,
|
||||||
|
outputs=[
|
||||||
|
CompletionOutput(
|
||||||
|
index=0,
|
||||||
|
text=text,
|
||||||
|
token_ids=tokenizer.encode(text, add_special_tokens=False),
|
||||||
|
cumulative_logprob=0.0,
|
||||||
|
logprobs=None,
|
||||||
|
finish_reason=engine_finish_reason,
|
||||||
|
)
|
||||||
|
],
|
||||||
|
finished=True,
|
||||||
|
)
|
||||||
|
|
||||||
|
response = await serving_chat.chat_completion_full_generator(
|
||||||
|
request=request,
|
||||||
|
result_generator=_single_request_output(request_output),
|
||||||
|
request_id="test-req",
|
||||||
|
model_name=MODEL_NAME,
|
||||||
|
conversation=[],
|
||||||
|
tokenizer=tokenizer,
|
||||||
|
request_metadata=RequestResponseMetadata(
|
||||||
|
request_id="test-req", model_name=MODEL_NAME
|
||||||
|
),
|
||||||
|
parser=parser,
|
||||||
|
)
|
||||||
|
|
||||||
|
assert isinstance(response, ChatCompletionResponse)
|
||||||
|
choice = response.choices[0]
|
||||||
|
assert choice.finish_reason == expected_finish_reason
|
||||||
|
assert bool(choice.message.tool_calls) == (text != "Hello, how are you?"), (
|
||||||
|
"finish_reason must reflect whether tool calls were emitted"
|
||||||
|
)
|
||||||
|
|||||||
@@ -456,3 +456,27 @@ def test_num_init_messages_offset():
|
|||||||
items = ctx.make_response_output_items()
|
items = ctx.make_response_output_items()
|
||||||
assert len(items) == 1
|
assert len(items) == 1
|
||||||
assert items[0].type == "message"
|
assert items[0].type == "message"
|
||||||
|
|
||||||
|
|
||||||
|
def test_length_truncation_marks_last_item_incomplete():
|
||||||
|
"""A turn cut by max_output_tokens ends in an ``incomplete`` item, both
|
||||||
|
with a parser (tool call) and without one (plain message)."""
|
||||||
|
request = _make_request(
|
||||||
|
tool_choice="auto",
|
||||||
|
tools=[
|
||||||
|
{
|
||||||
|
"type": "function",
|
||||||
|
"name": "get_weather",
|
||||||
|
"parameters": {"type": "object", "properties": {}},
|
||||||
|
}
|
||||||
|
],
|
||||||
|
)
|
||||||
|
ctx = _make_context(_ToolCallingParser, request=request, enable_auto_tools=True)
|
||||||
|
ctx.append_output(_make_request_output(text="calling tool", finish_reason="length"))
|
||||||
|
assert ctx.response_messages[-1].type == "function_call"
|
||||||
|
assert ctx.response_messages[-1].status == "incomplete"
|
||||||
|
|
||||||
|
ctx = _make_context(None)
|
||||||
|
ctx.append_output(_make_request_output(text="cut mid sen", finish_reason="length"))
|
||||||
|
assert ctx.response_messages[-1].type == "message"
|
||||||
|
assert ctx.response_messages[-1].status == "incomplete"
|
||||||
|
|||||||
@@ -991,3 +991,28 @@ class TestConstructInputMessagesInstructionsLeak:
|
|||||||
assert len(msgs) == 2
|
assert len(msgs) == 2
|
||||||
assert msgs[0] == {"role": "system", "content": "be helpful"}
|
assert msgs[0] == {"role": "system", "content": "be helpful"}
|
||||||
assert msgs[1] == {"role": "user", "content": "hello"}
|
assert msgs[1] == {"role": "user", "content": "hello"}
|
||||||
|
|
||||||
|
|
||||||
|
class TestBuildResponseOutputItemsStatus:
|
||||||
|
"""Item ``status`` must follow the finish reason of the generation
|
||||||
|
(#57998): a ``length`` cut marks the item that was cut ``incomplete``."""
|
||||||
|
|
||||||
|
_CALLS = [
|
||||||
|
FunctionCall(name="get_weather", arguments='{"city": "Paris"}'),
|
||||||
|
FunctionCall(name="get_weather", arguments='{"city": "Tok'),
|
||||||
|
]
|
||||||
|
|
||||||
|
def test_length_marks_truncated_message_incomplete(self):
|
||||||
|
(item,) = build_response_output_items(
|
||||||
|
reasoning=None, content="cut mid sen", tool_calls=[], finish_reason="length"
|
||||||
|
)
|
||||||
|
assert item.type == "message"
|
||||||
|
assert item.status == "incomplete"
|
||||||
|
|
||||||
|
def test_length_marks_only_last_tool_call_incomplete(self):
|
||||||
|
message, first, last = build_response_output_items(
|
||||||
|
reasoning=None, content="hi", tool_calls=self._CALLS, finish_reason="length"
|
||||||
|
)
|
||||||
|
assert message.status == "completed"
|
||||||
|
assert first.status == "completed"
|
||||||
|
assert last.status == "incomplete"
|
||||||
|
|||||||
@@ -1,6 +1,8 @@
|
|||||||
# SPDX-License-Identifier: Apache-2.0
|
# SPDX-License-Identifier: Apache-2.0
|
||||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||||
|
|
||||||
|
import asyncio
|
||||||
|
from collections import deque
|
||||||
from contextlib import AsyncExitStack, asynccontextmanager
|
from contextlib import AsyncExitStack, asynccontextmanager
|
||||||
from types import SimpleNamespace
|
from types import SimpleNamespace
|
||||||
from unittest.mock import AsyncMock, MagicMock
|
from unittest.mock import AsyncMock, MagicMock
|
||||||
@@ -51,6 +53,7 @@ from vllm.entrypoints.openai.responses.context import (
|
|||||||
from vllm.entrypoints.openai.responses.protocol import (
|
from vllm.entrypoints.openai.responses.protocol import (
|
||||||
ResponseCompletedEvent,
|
ResponseCompletedEvent,
|
||||||
ResponseCreatedEvent,
|
ResponseCreatedEvent,
|
||||||
|
ResponseIncompleteEvent,
|
||||||
ResponseRawMessageAndToken,
|
ResponseRawMessageAndToken,
|
||||||
ResponsesRequest,
|
ResponsesRequest,
|
||||||
ResponsesResponse,
|
ResponsesResponse,
|
||||||
@@ -1414,6 +1417,7 @@ def _make_request_output(
|
|||||||
text,
|
text,
|
||||||
token_ids,
|
token_ids,
|
||||||
metrics: RequestStateStats | None = None,
|
metrics: RequestStateStats | None = None,
|
||||||
|
finish_reason: str | None = None,
|
||||||
):
|
):
|
||||||
completion = CompletionOutput(
|
completion = CompletionOutput(
|
||||||
index=0,
|
index=0,
|
||||||
@@ -1421,7 +1425,7 @@ def _make_request_output(
|
|||||||
token_ids=token_ids,
|
token_ids=token_ids,
|
||||||
cumulative_logprob=0.0,
|
cumulative_logprob=0.0,
|
||||||
logprobs=None,
|
logprobs=None,
|
||||||
finish_reason=None,
|
finish_reason=finish_reason,
|
||||||
stop_reason=None,
|
stop_reason=None,
|
||||||
)
|
)
|
||||||
return RequestOutput(
|
return RequestOutput(
|
||||||
@@ -1441,10 +1445,11 @@ def _make_simple_context_with_output(
|
|||||||
token_ids,
|
token_ids,
|
||||||
response_parser=None,
|
response_parser=None,
|
||||||
metrics: RequestStateStats | None = None,
|
metrics: RequestStateStats | None = None,
|
||||||
|
finish_reason: str | None = None,
|
||||||
):
|
):
|
||||||
"""Create a SimpleContext with a RequestOutput containing the given text."""
|
"""Create a SimpleContext with a RequestOutput containing the given text."""
|
||||||
ctx = SimpleContext(response_parser=response_parser)
|
ctx = SimpleContext(response_parser=response_parser)
|
||||||
req_output = _make_request_output(text, token_ids, metrics)
|
req_output = _make_request_output(text, token_ids, metrics, finish_reason)
|
||||||
ctx.append_output(req_output)
|
ctx.append_output(req_output)
|
||||||
ctx.request_metrics = metrics
|
ctx.request_metrics = metrics
|
||||||
return ctx
|
return ctx
|
||||||
@@ -1561,6 +1566,66 @@ async def test_responses_streaming_metrics_only_on_completed_event():
|
|||||||
assert events[-1].response.metrics.time_to_first_token_ms == pytest.approx(500.0)
|
assert events[-1].response.metrics.time_to_first_token_ms == pytest.approx(500.0)
|
||||||
|
|
||||||
|
|
||||||
|
async def _stream_truncated_by_length(serving: OpenAIServingResponses):
|
||||||
|
request = ResponsesRequest(input="hi", tools=[], stream=True, store=False)
|
||||||
|
context = _make_simple_context_with_output(
|
||||||
|
"cut mid sen", [10, 20], finish_reason="length"
|
||||||
|
)
|
||||||
|
|
||||||
|
async def one_context():
|
||||||
|
yield context
|
||||||
|
|
||||||
|
return [
|
||||||
|
event
|
||||||
|
async for event in serving.responses_stream_generator(
|
||||||
|
request=request,
|
||||||
|
sampling_params=SamplingParams(max_tokens=2),
|
||||||
|
result_generator=one_context(),
|
||||||
|
context=context,
|
||||||
|
model_name="test-model",
|
||||||
|
tokenizer=MagicMock(),
|
||||||
|
request_metadata=RequestResponseMetadata(request_id="req"),
|
||||||
|
)
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_responses_stream_truncated_by_length_reports_incomplete():
|
||||||
|
"""A max_output_tokens cut must end the stream with ``response.incomplete``
|
||||||
|
and mark the cut item ``incomplete`` in both ``output_item.done`` and the
|
||||||
|
final response (#57998)."""
|
||||||
|
events = await _stream_truncated_by_length(_make_serving_instance())
|
||||||
|
|
||||||
|
item_done = [e for e in events if e.type == "response.output_item.done"]
|
||||||
|
assert [e.item.status for e in item_done] == ["incomplete"]
|
||||||
|
final = events[-1]
|
||||||
|
assert isinstance(final, ResponseIncompleteEvent)
|
||||||
|
assert final.response.status == "incomplete"
|
||||||
|
assert final.response.incomplete_details.reason == "max_output_tokens"
|
||||||
|
assert [item.status for item in final.response.output] == ["incomplete"]
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_background_stream_replay_stops_on_response_incomplete():
|
||||||
|
"""Replaying a stored background stream must end at ``response.incomplete``
|
||||||
|
as it does at ``response.completed``, instead of waiting for more events."""
|
||||||
|
serving = _make_serving_instance()
|
||||||
|
events = await _stream_truncated_by_length(serving)
|
||||||
|
assert isinstance(events[-1], ResponseIncompleteEvent)
|
||||||
|
serving.event_store["resp_bg"] = (deque(events), asyncio.Event())
|
||||||
|
|
||||||
|
async def replay(starting_after=None):
|
||||||
|
return [
|
||||||
|
event
|
||||||
|
async for event in serving.responses_background_stream_generator(
|
||||||
|
"resp_bg", starting_after=starting_after
|
||||||
|
)
|
||||||
|
]
|
||||||
|
|
||||||
|
assert await asyncio.wait_for(replay(), timeout=5) == events
|
||||||
|
assert await asyncio.wait_for(replay(starting_after=0), timeout=5) == events[1:]
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
@pytest.mark.asyncio
|
||||||
async def test_responses_metrics_suppressed_for_multiple_generation_turns():
|
async def test_responses_metrics_suppressed_for_multiple_generation_turns():
|
||||||
response = await _make_full_metrics_response(
|
response = await _make_full_metrics_response(
|
||||||
|
|||||||
@@ -3,11 +3,18 @@
|
|||||||
|
|
||||||
"""Tests for the /derender endpoints (postprocessing counterpart to /render)."""
|
"""Tests for the /derender endpoints (postprocessing counterpart to /render)."""
|
||||||
|
|
||||||
|
from unittest.mock import MagicMock
|
||||||
|
|
||||||
import httpx
|
import httpx
|
||||||
import pytest
|
import pytest
|
||||||
import pytest_asyncio
|
import pytest_asyncio
|
||||||
|
|
||||||
from tests.utils import RemoteLaunchRenderServer
|
from tests.utils import RemoteLaunchRenderServer
|
||||||
|
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
|
||||||
|
from vllm.entrypoints.scale_out.token_in_token_out.protocol import (
|
||||||
|
GenerateTokensResponse,
|
||||||
|
)
|
||||||
|
from vllm.renderers.online_derenderer import OnlineDerenderer
|
||||||
from vllm.tokenizers import get_tokenizer
|
from vllm.tokenizers import get_tokenizer
|
||||||
|
|
||||||
MODEL_NAME = "hmellor/tiny-random-LlamaForCausalLM"
|
MODEL_NAME = "hmellor/tiny-random-LlamaForCausalLM"
|
||||||
@@ -1432,3 +1439,55 @@ async def test_e2e_harmony_reasoning(harmony_client, harmony_tokenizer):
|
|||||||
assert msg["reasoning"] is not None
|
assert msg["reasoning"] is not None
|
||||||
assert reasoning_text in msg["reasoning"]
|
assert reasoning_text in msg["reasoning"]
|
||||||
assert answer_text in (msg["content"] or "")
|
assert answer_text in (msg["content"] or "")
|
||||||
|
|
||||||
|
|
||||||
|
_HERMES_TOOL_CALL_TEXT = (
|
||||||
|
'<tool_call>\n{"name": "get_weather", "arguments": {"city": "Paris"}}\n</tool_call>'
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
("tool_choice", "text", "engine_finish_reason", "expected_finish_reason"),
|
||||||
|
[
|
||||||
|
("auto", _HERMES_TOOL_CALL_TEXT, "stop", "tool_calls"),
|
||||||
|
("auto", _HERMES_TOOL_CALL_TEXT, "length", "length"),
|
||||||
|
("required", "Hello there", "stop", "stop"),
|
||||||
|
],
|
||||||
|
)
|
||||||
|
def test_derender_chat_finish_reason_matches_chat_completion(
|
||||||
|
tool_choice, text, engine_finish_reason, expected_finish_reason
|
||||||
|
):
|
||||||
|
"""/derender applies the same finish_reason rule as the coupled chat
|
||||||
|
path: ``tool_calls`` only for a real call at a natural stop."""
|
||||||
|
tokenizer = get_tokenizer(MODEL_NAME)
|
||||||
|
renderer = MagicMock()
|
||||||
|
renderer.get_tokenizer.return_value = tokenizer
|
||||||
|
model_config = MagicMock()
|
||||||
|
model_config.model = MODEL_NAME
|
||||||
|
model_config.hf_config.model_type = "llama"
|
||||||
|
derenderer = OnlineDerenderer(
|
||||||
|
model_config,
|
||||||
|
renderer,
|
||||||
|
request_logger=None,
|
||||||
|
chat_template="{{ messages }}",
|
||||||
|
chat_template_content_format="auto",
|
||||||
|
enable_auto_tools=True,
|
||||||
|
tool_parser="hermes",
|
||||||
|
)
|
||||||
|
request = ChatCompletionRequest(
|
||||||
|
model=MODEL_NAME,
|
||||||
|
messages=[{"role": "user", "content": "Weather in Paris?"}],
|
||||||
|
tools=_E2E_TOOLS,
|
||||||
|
tool_choice=tool_choice,
|
||||||
|
)
|
||||||
|
response = GenerateTokensResponse.model_validate(
|
||||||
|
_make_generate_response(
|
||||||
|
tokenizer.encode(text, add_special_tokens=False),
|
||||||
|
finish_reason=engine_finish_reason,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
(choice,) = derenderer._derender_chat(response, request)
|
||||||
|
|
||||||
|
assert choice.finish_reason == expected_finish_reason
|
||||||
|
assert bool(choice.message.tool_calls) == (text == _HERMES_TOOL_CALL_TEXT)
|
||||||
|
|||||||
@@ -1289,7 +1289,15 @@ class TestDerenderChatStreamParsed:
|
|||||||
assert state.last_tool_call_ids == [first_id]
|
assert state.last_tool_call_ids == [first_id]
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
@pytest.mark.asyncio
|
||||||
async def test_finish_reason_rewritten_to_tool_calls(self, parsed_derenderer):
|
@pytest.mark.parametrize(
|
||||||
|
("engine_finish_reason", "expected_finish_reason"),
|
||||||
|
[("stop", "tool_calls"), ("length", "length")],
|
||||||
|
)
|
||||||
|
async def test_finish_reason_rewritten_to_tool_calls(
|
||||||
|
self, parsed_derenderer, engine_finish_reason, expected_finish_reason
|
||||||
|
):
|
||||||
|
"""A streamed tool call is reported as ``tool_calls`` only at a
|
||||||
|
natural stop; a ``length`` cut must stay visible."""
|
||||||
chat_request = _chat_request(
|
chat_request = _chat_request(
|
||||||
tools=[{"type": "function", "function": {"name": "get_weather"}}],
|
tools=[{"type": "function", "function": {"name": "get_weather"}}],
|
||||||
tool_choice="auto",
|
tool_choice="auto",
|
||||||
@@ -1297,11 +1305,12 @@ class TestDerenderChatStreamParsed:
|
|||||||
chunk, _ = await parsed_derenderer.derender_chat_stream(
|
chunk, _ = await parsed_derenderer.derender_chat_stream(
|
||||||
model=MODEL_NAME,
|
model=MODEL_NAME,
|
||||||
generate_chunk=_make_stream_chunk(
|
generate_chunk=_make_stream_chunk(
|
||||||
[_FakeParser.TOOL_START, _FakeParser.TOOL_ARG], finish_reason="stop"
|
[_FakeParser.TOOL_START, _FakeParser.TOOL_ARG],
|
||||||
|
finish_reason=engine_finish_reason,
|
||||||
),
|
),
|
||||||
chat_request=chat_request,
|
chat_request=chat_request,
|
||||||
)
|
)
|
||||||
assert chunk.choices[0].finish_reason == "tool_calls"
|
assert chunk.choices[0].finish_reason == expected_finish_reason
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
@pytest.mark.asyncio
|
||||||
async def test_finish_reason_stop_for_named_tool_choice(self, parsed_derenderer):
|
async def test_finish_reason_stop_for_named_tool_choice(self, parsed_derenderer):
|
||||||
|
|||||||
@@ -48,6 +48,7 @@ from vllm.entrypoints.serve.utils.api_utils import get_max_tokens, should_includ
|
|||||||
from vllm.entrypoints.serve.utils.request_logger import RequestLogger
|
from vllm.entrypoints.serve.utils.request_logger import RequestLogger
|
||||||
from vllm.entrypoints.serve.utils.tool_calls_utils import (
|
from vllm.entrypoints.serve.utils.tool_calls_utils import (
|
||||||
maybe_filter_parallel_tool_calls,
|
maybe_filter_parallel_tool_calls,
|
||||||
|
resolve_finish_reason,
|
||||||
)
|
)
|
||||||
from vllm.exceptions import GenerationError
|
from vllm.exceptions import GenerationError
|
||||||
from vllm.inputs import EngineInput, MultiModalPlaceholders
|
from vllm.inputs import EngineInput, MultiModalPlaceholders
|
||||||
@@ -491,11 +492,6 @@ class OpenAIServingChat(GenerateBaseServing):
|
|||||||
num_cache_creation_tokens = None
|
num_cache_creation_tokens = None
|
||||||
tools_streamed = [False] * num_choices
|
tools_streamed = [False] * num_choices
|
||||||
|
|
||||||
if isinstance(request.tool_choice, ChatCompletionNamedToolChoiceParam):
|
|
||||||
tool_choice_function_name = request.tool_choice.function.name
|
|
||||||
else:
|
|
||||||
tool_choice_function_name = None
|
|
||||||
|
|
||||||
previous_texts = [""] * num_choices
|
previous_texts = [""] * num_choices
|
||||||
|
|
||||||
try:
|
try:
|
||||||
@@ -764,25 +760,13 @@ class OpenAIServingChat(GenerateBaseServing):
|
|||||||
# if the model is finished generating
|
# if the model is finished generating
|
||||||
else:
|
else:
|
||||||
# Send the finish response for each request.n only once
|
# Send the finish response for each request.n only once
|
||||||
# In OpenAI's API, when a tool is called, the
|
|
||||||
# finish_reason is:
|
|
||||||
# "tool_calls" for "auto" or "required" tool calls,
|
|
||||||
# and "stop" for named tool calls.
|
|
||||||
if (
|
|
||||||
tools_streamed[i]
|
|
||||||
and not tool_choice_function_name
|
|
||||||
and output.finish_reason == "stop"
|
|
||||||
):
|
|
||||||
finish_reason_ = "tool_calls"
|
|
||||||
else:
|
|
||||||
finish_reason_ = (
|
|
||||||
output.finish_reason if output.finish_reason else "stop"
|
|
||||||
)
|
|
||||||
choice_data = ChatCompletionResponseStreamChoice(
|
choice_data = ChatCompletionResponseStreamChoice(
|
||||||
index=i,
|
index=i,
|
||||||
delta=delta_message,
|
delta=delta_message,
|
||||||
logprobs=logprobs,
|
logprobs=logprobs,
|
||||||
finish_reason=finish_reason_,
|
finish_reason=resolve_finish_reason(
|
||||||
|
output.finish_reason, request, tools_streamed[i]
|
||||||
|
),
|
||||||
stop_reason=output.stop_reason,
|
stop_reason=output.stop_reason,
|
||||||
token_ids=(
|
token_ids=(
|
||||||
as_list(output.token_ids) if include_token_ids else None
|
as_list(output.token_ids) if include_token_ids else None
|
||||||
@@ -1013,7 +997,6 @@ class OpenAIServingChat(GenerateBaseServing):
|
|||||||
tool_calls = []
|
tool_calls = []
|
||||||
suppress_metadata = False
|
suppress_metadata = False
|
||||||
|
|
||||||
auto_tools_called = False
|
|
||||||
is_named_tool_choice = (
|
is_named_tool_choice = (
|
||||||
request.tool_choice is not None
|
request.tool_choice is not None
|
||||||
and type(request.tool_choice) is ChatCompletionNamedToolChoiceParam
|
and type(request.tool_choice) is ChatCompletionNamedToolChoiceParam
|
||||||
@@ -1056,7 +1039,6 @@ class OpenAIServingChat(GenerateBaseServing):
|
|||||||
and self.enable_auto_tools
|
and self.enable_auto_tools
|
||||||
and tool_parser_cls
|
and tool_parser_cls
|
||||||
):
|
):
|
||||||
auto_tools_called = tool_calls is not None and len(tool_calls) > 0
|
|
||||||
if tool_calls:
|
if tool_calls:
|
||||||
message = self._create_chat_message(
|
message = self._create_chat_message(
|
||||||
role=role,
|
role=role,
|
||||||
@@ -1093,15 +1075,6 @@ class OpenAIServingChat(GenerateBaseServing):
|
|||||||
# metadata cached on the reasoning parser.
|
# metadata cached on the reasoning parser.
|
||||||
message = self._finalize_response_message(message, parser=parser)
|
message = self._finalize_response_message(message, parser=parser)
|
||||||
|
|
||||||
# In OpenAI's API, when a tool is called, the finish_reason is:
|
|
||||||
# "tool_calls" for "auto" or "required" tool calls,
|
|
||||||
# and "stop" for named tool calls.
|
|
||||||
is_finish_reason_tool_calls = auto_tools_called or (
|
|
||||||
request.tool_choice
|
|
||||||
and request.tool_choice == "required"
|
|
||||||
and output.finish_reason == "stop"
|
|
||||||
)
|
|
||||||
|
|
||||||
routed_experts_b64 = (
|
routed_experts_b64 = (
|
||||||
numpy2base64(output.routed_experts)
|
numpy2base64(output.routed_experts)
|
||||||
if output.routed_experts is not None
|
if output.routed_experts is not None
|
||||||
@@ -1112,11 +1085,9 @@ class OpenAIServingChat(GenerateBaseServing):
|
|||||||
index=output.index,
|
index=output.index,
|
||||||
message=message,
|
message=message,
|
||||||
logprobs=logprobs,
|
logprobs=logprobs,
|
||||||
finish_reason="tool_calls"
|
finish_reason=resolve_finish_reason(
|
||||||
if is_finish_reason_tool_calls
|
output.finish_reason, request, bool(message.tool_calls)
|
||||||
else output.finish_reason
|
),
|
||||||
if output.finish_reason
|
|
||||||
else "stop",
|
|
||||||
stop_reason=output.stop_reason,
|
stop_reason=output.stop_reason,
|
||||||
token_ids=(
|
token_ids=(
|
||||||
as_list(output.token_ids)
|
as_list(output.token_ids)
|
||||||
|
|||||||
@@ -32,6 +32,7 @@ from vllm.entrypoints.openai.responses.protocol import (
|
|||||||
from vllm.entrypoints.openai.responses.utils import (
|
from vllm.entrypoints.openai.responses.utils import (
|
||||||
build_response_output_items,
|
build_response_output_items,
|
||||||
construct_tool_dicts,
|
construct_tool_dicts,
|
||||||
|
response_item_status,
|
||||||
)
|
)
|
||||||
from vllm.entrypoints.serve.utils.tool_calls_utils import (
|
from vllm.entrypoints.serve.utils.tool_calls_utils import (
|
||||||
maybe_filter_parallel_tool_calls,
|
maybe_filter_parallel_tool_calls,
|
||||||
@@ -380,6 +381,7 @@ class ParsableContext(ConversationContext):
|
|||||||
tool_calls or [], self.request
|
tool_calls or [], self.request
|
||||||
),
|
),
|
||||||
tools=self.request.tools,
|
tools=self.request.tools,
|
||||||
|
finish_reason=completion.finish_reason,
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
elif completion.text:
|
elif completion.text:
|
||||||
@@ -387,7 +389,7 @@ class ParsableContext(ConversationContext):
|
|||||||
ResponseOutputMessage(
|
ResponseOutputMessage(
|
||||||
type="message",
|
type="message",
|
||||||
id=f"msg_{random_uuid()}",
|
id=f"msg_{random_uuid()}",
|
||||||
status="completed",
|
status=response_item_status(completion.finish_reason),
|
||||||
role="assistant",
|
role="assistant",
|
||||||
content=[
|
content=[
|
||||||
ResponseOutputText(
|
ResponseOutputText(
|
||||||
|
|||||||
@@ -40,6 +40,9 @@ from openai.types.responses import (
|
|||||||
ResponseCompletedEvent as OpenAIResponseCompletedEvent,
|
ResponseCompletedEvent as OpenAIResponseCompletedEvent,
|
||||||
)
|
)
|
||||||
from openai.types.responses import ResponseCreatedEvent as OpenAIResponseCreatedEvent
|
from openai.types.responses import ResponseCreatedEvent as OpenAIResponseCreatedEvent
|
||||||
|
from openai.types.responses import (
|
||||||
|
ResponseIncompleteEvent as OpenAIResponseIncompleteEvent,
|
||||||
|
)
|
||||||
from openai.types.responses import (
|
from openai.types.responses import (
|
||||||
ResponseInProgressEvent as OpenAIResponseInProgressEvent,
|
ResponseInProgressEvent as OpenAIResponseInProgressEvent,
|
||||||
)
|
)
|
||||||
@@ -927,6 +930,10 @@ class ResponseCompletedEvent(OpenAIResponseCompletedEvent):
|
|||||||
response: ResponsesResponse # type: ignore[override]
|
response: ResponsesResponse # type: ignore[override]
|
||||||
|
|
||||||
|
|
||||||
|
class ResponseIncompleteEvent(OpenAIResponseIncompleteEvent):
|
||||||
|
response: ResponsesResponse # type: ignore[override]
|
||||||
|
|
||||||
|
|
||||||
class ResponseCreatedEvent(OpenAIResponseCreatedEvent):
|
class ResponseCreatedEvent(OpenAIResponseCreatedEvent):
|
||||||
response: ResponsesResponse # type: ignore[override]
|
response: ResponsesResponse # type: ignore[override]
|
||||||
|
|
||||||
@@ -939,6 +946,7 @@ StreamingResponsesResponse: TypeAlias = (
|
|||||||
ResponseCreatedEvent
|
ResponseCreatedEvent
|
||||||
| ResponseInProgressEvent
|
| ResponseInProgressEvent
|
||||||
| ResponseCompletedEvent
|
| ResponseCompletedEvent
|
||||||
|
| ResponseIncompleteEvent
|
||||||
| ResponseOutputItemAddedEvent
|
| ResponseOutputItemAddedEvent
|
||||||
| ResponseOutputItemDoneEvent
|
| ResponseOutputItemDoneEvent
|
||||||
| ResponseContentPartAddedEvent
|
| ResponseContentPartAddedEvent
|
||||||
|
|||||||
@@ -45,6 +45,7 @@ from vllm.entrypoints.openai.responses.protocol import (
|
|||||||
OutputTokensDetails,
|
OutputTokensDetails,
|
||||||
ResponseCompletedEvent,
|
ResponseCompletedEvent,
|
||||||
ResponseCreatedEvent,
|
ResponseCreatedEvent,
|
||||||
|
ResponseIncompleteEvent,
|
||||||
ResponseInProgressEvent,
|
ResponseInProgressEvent,
|
||||||
ResponseInputOutputItem,
|
ResponseInputOutputItem,
|
||||||
ResponseInputOutputMessage,
|
ResponseInputOutputMessage,
|
||||||
@@ -66,6 +67,7 @@ from vllm.entrypoints.openai.responses.utils import (
|
|||||||
build_response_output_items,
|
build_response_output_items,
|
||||||
extract_function_tool_names,
|
extract_function_tool_names,
|
||||||
extract_tool_types,
|
extract_tool_types,
|
||||||
|
response_item_status,
|
||||||
reuse_streamed_item_ids,
|
reuse_streamed_item_ids,
|
||||||
)
|
)
|
||||||
from vllm.entrypoints.serve.engine.protocol import ErrorResponse
|
from vllm.entrypoints.serve.engine.protocol import ErrorResponse
|
||||||
@@ -1045,6 +1047,7 @@ class OpenAIServingResponses(GenerateBaseServing):
|
|||||||
tool_calls=maybe_filter_parallel_tool_calls(tool_calls or [], request),
|
tool_calls=maybe_filter_parallel_tool_calls(tool_calls or [], request),
|
||||||
logprobs=logprobs,
|
logprobs=logprobs,
|
||||||
tools=request.tools,
|
tools=request.tools,
|
||||||
|
finish_reason=final_output.finish_reason,
|
||||||
)
|
)
|
||||||
|
|
||||||
# Fallback when no parser is configured
|
# Fallback when no parser is configured
|
||||||
@@ -1062,7 +1065,7 @@ class OpenAIServingResponses(GenerateBaseServing):
|
|||||||
if final_output.text
|
if final_output.text
|
||||||
else [],
|
else [],
|
||||||
role="assistant",
|
role="assistant",
|
||||||
status="completed",
|
status=response_item_status(final_output.finish_reason),
|
||||||
type="message",
|
type="message",
|
||||||
)
|
)
|
||||||
]
|
]
|
||||||
@@ -1124,7 +1127,10 @@ class OpenAIServingResponses(GenerateBaseServing):
|
|||||||
while current_index < len(event_deque):
|
while current_index < len(event_deque):
|
||||||
event = event_deque[current_index]
|
event = event_deque[current_index]
|
||||||
yield event
|
yield event
|
||||||
if getattr(event, "type", "unknown") == "response.completed":
|
if getattr(event, "type", "unknown") in (
|
||||||
|
"response.completed",
|
||||||
|
"response.incomplete",
|
||||||
|
):
|
||||||
return
|
return
|
||||||
current_index += 1
|
current_index += 1
|
||||||
|
|
||||||
@@ -1222,6 +1228,7 @@ class OpenAIServingResponses(GenerateBaseServing):
|
|||||||
top_logprobs=request.top_logprobs,
|
top_logprobs=request.top_logprobs,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
finish_reason: str | None = None
|
||||||
async for ctx in result_generator:
|
async for ctx in result_generator:
|
||||||
assert isinstance(ctx, SimpleContext)
|
assert isinstance(ctx, SimpleContext)
|
||||||
if ctx.last_output is None or not ctx.last_output.outputs:
|
if ctx.last_output is None or not ctx.last_output.outputs:
|
||||||
@@ -1229,6 +1236,7 @@ class OpenAIServingResponses(GenerateBaseServing):
|
|||||||
|
|
||||||
output = ctx.last_output.outputs[0]
|
output = ctx.last_output.outputs[0]
|
||||||
self._raise_if_error(output.finish_reason, request.request_id)
|
self._raise_if_error(output.finish_reason, request.request_id)
|
||||||
|
finish_reason = output.finish_reason
|
||||||
delta_text = output.text
|
delta_text = output.text
|
||||||
delta_token_ids = as_list(output.token_ids)
|
delta_token_ids = as_list(output.token_ids)
|
||||||
|
|
||||||
@@ -1261,7 +1269,7 @@ class OpenAIServingResponses(GenerateBaseServing):
|
|||||||
for event in processor.emit_delta(dm, output, _get_logprobs):
|
for event in processor.emit_delta(dm, output, _get_logprobs):
|
||||||
yield _increment_sequence_number_and_return(event)
|
yield _increment_sequence_number_and_return(event)
|
||||||
|
|
||||||
for event in processor.close_current():
|
for event in processor.close_current(response_item_status(finish_reason)):
|
||||||
yield _increment_sequence_number_and_return(event)
|
yield _increment_sequence_number_and_return(event)
|
||||||
|
|
||||||
assert isinstance(context, SimpleContext)
|
assert isinstance(context, SimpleContext)
|
||||||
@@ -1415,10 +1423,20 @@ class OpenAIServingResponses(GenerateBaseServing):
|
|||||||
request_metadata,
|
request_metadata,
|
||||||
created_time=created_time,
|
created_time=created_time,
|
||||||
)
|
)
|
||||||
yield _increment_sequence_number_and_return(
|
terminal_event: StreamingResponsesResponse
|
||||||
ResponseCompletedEvent(
|
if (
|
||||||
|
isinstance(final_response, ResponsesResponse)
|
||||||
|
and final_response.status == "incomplete"
|
||||||
|
):
|
||||||
|
terminal_event = ResponseIncompleteEvent(
|
||||||
|
type="response.incomplete",
|
||||||
|
sequence_number=-1,
|
||||||
|
response=final_response,
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
terminal_event = ResponseCompletedEvent(
|
||||||
type="response.completed",
|
type="response.completed",
|
||||||
sequence_number=-1,
|
sequence_number=-1,
|
||||||
response=final_response,
|
response=final_response,
|
||||||
)
|
)
|
||||||
)
|
yield _increment_sequence_number_and_return(terminal_event)
|
||||||
|
|||||||
@@ -18,7 +18,7 @@ import json
|
|||||||
from collections.abc import Callable
|
from collections.abc import Callable
|
||||||
from dataclasses import dataclass, field
|
from dataclasses import dataclass, field
|
||||||
from enum import Enum, auto
|
from enum import Enum, auto
|
||||||
from typing import Any, ClassVar, Final, NamedTuple
|
from typing import Any, ClassVar, Final, Literal, NamedTuple
|
||||||
|
|
||||||
from openai.types.responses import (
|
from openai.types.responses import (
|
||||||
ResponseCodeInterpreterCallCodeDeltaEvent,
|
ResponseCodeInterpreterCallCodeDeltaEvent,
|
||||||
@@ -910,6 +910,7 @@ def emit_simple_content_delta(
|
|||||||
|
|
||||||
def emit_simple_content_done(
|
def emit_simple_content_done(
|
||||||
state: SimpleStreamingState,
|
state: SimpleStreamingState,
|
||||||
|
status: Literal["completed", "incomplete"] = "completed",
|
||||||
) -> list[StreamingResponsesResponse]:
|
) -> list[StreamingResponsesResponse]:
|
||||||
part = ResponseOutputText(
|
part = ResponseOutputText(
|
||||||
type="output_text",
|
type="output_text",
|
||||||
@@ -944,7 +945,7 @@ def emit_simple_content_done(
|
|||||||
type="message",
|
type="message",
|
||||||
role="assistant",
|
role="assistant",
|
||||||
content=[part] if state.accumulated_text else [],
|
content=[part] if state.accumulated_text else [],
|
||||||
status="completed",
|
status=status,
|
||||||
),
|
),
|
||||||
),
|
),
|
||||||
]
|
]
|
||||||
@@ -1002,6 +1003,7 @@ def emit_simple_reasoning_delta(
|
|||||||
|
|
||||||
def emit_simple_reasoning_done(
|
def emit_simple_reasoning_done(
|
||||||
state: SimpleStreamingState,
|
state: SimpleStreamingState,
|
||||||
|
status: Literal["completed", "incomplete"] = "completed",
|
||||||
) -> list[StreamingResponsesResponse]:
|
) -> list[StreamingResponsesResponse]:
|
||||||
part = ResponseReasoningTextContent(
|
part = ResponseReasoningTextContent(
|
||||||
text=state.accumulated_text,
|
text=state.accumulated_text,
|
||||||
@@ -1031,7 +1033,7 @@ def emit_simple_reasoning_done(
|
|||||||
item=ResponseReasoningItem(
|
item=ResponseReasoningItem(
|
||||||
type="reasoning",
|
type="reasoning",
|
||||||
content=[part],
|
content=[part],
|
||||||
status="completed",
|
status=status,
|
||||||
id=state.current_item_id,
|
id=state.current_item_id,
|
||||||
summary=[],
|
summary=[],
|
||||||
),
|
),
|
||||||
@@ -1134,6 +1136,7 @@ def emit_simple_tool_call_delta(
|
|||||||
|
|
||||||
def emit_simple_tool_call_done(
|
def emit_simple_tool_call_done(
|
||||||
state: SimpleStreamingState,
|
state: SimpleStreamingState,
|
||||||
|
status: Literal["completed", "incomplete"] = "completed",
|
||||||
) -> list[StreamingResponsesResponse]:
|
) -> list[StreamingResponsesResponse]:
|
||||||
events: list[StreamingResponsesResponse]
|
events: list[StreamingResponsesResponse]
|
||||||
item: ResponseOutputItem
|
item: ResponseOutputItem
|
||||||
@@ -1172,7 +1175,7 @@ def emit_simple_tool_call_done(
|
|||||||
name=state.tool_call_name,
|
name=state.tool_call_name,
|
||||||
namespace=state.tool_call_namespace,
|
namespace=state.tool_call_namespace,
|
||||||
arguments=state.accumulated_text,
|
arguments=state.accumulated_text,
|
||||||
status="completed",
|
status=status,
|
||||||
id=state.current_item_id,
|
id=state.current_item_id,
|
||||||
call_id=state.tool_call_id,
|
call_id=state.tool_call_id,
|
||||||
)
|
)
|
||||||
@@ -1312,12 +1315,14 @@ class SimpleStreamingEventProcessor:
|
|||||||
and self.state.tool_call_index != tool_call.index
|
and self.state.tool_call_index != tool_call.index
|
||||||
)
|
)
|
||||||
|
|
||||||
def close_current(self) -> list[StreamingResponsesResponse]:
|
def close_current(
|
||||||
|
self, status: Literal["completed", "incomplete"] = "completed"
|
||||||
|
) -> list[StreamingResponsesResponse]:
|
||||||
"""Close the current state and emit its 'done' event sequence."""
|
"""Close the current state and emit its 'done' event sequence."""
|
||||||
handlers = self._STATE_HANDLERS.get(self.state.current_state)
|
handlers = self._STATE_HANDLERS.get(self.state.current_state)
|
||||||
if handlers is None:
|
if handlers is None:
|
||||||
return []
|
return []
|
||||||
events = handlers.done_fn(self.state)
|
events = handlers.done_fn(self.state, status)
|
||||||
self.output_items.extend(
|
self.output_items.extend(
|
||||||
event.item
|
event.item
|
||||||
for event in events
|
for event in events
|
||||||
|
|||||||
@@ -3,7 +3,7 @@
|
|||||||
|
|
||||||
import json
|
import json
|
||||||
from collections.abc import Iterable
|
from collections.abc import Iterable
|
||||||
from typing import Any
|
from typing import Any, Literal
|
||||||
|
|
||||||
import partial_json_parser
|
import partial_json_parser
|
||||||
from openai.types.chat import (
|
from openai.types.chat import (
|
||||||
@@ -100,16 +100,30 @@ def _item_dict(item: ResponseInputOutputItem) -> dict[str, Any]:
|
|||||||
return item if isinstance(item, dict) else item.model_dump()
|
return item if isinstance(item, dict) else item.model_dump()
|
||||||
|
|
||||||
|
|
||||||
|
def response_item_status(
|
||||||
|
finish_reason: str | None,
|
||||||
|
) -> Literal["completed", "incomplete"]:
|
||||||
|
"""Status of the output item that ended a generation with ``finish_reason``."""
|
||||||
|
return "incomplete" if finish_reason == "length" else "completed"
|
||||||
|
|
||||||
|
|
||||||
def build_response_output_items(
|
def build_response_output_items(
|
||||||
reasoning: str | None,
|
reasoning: str | None,
|
||||||
content: str | None,
|
content: str | None,
|
||||||
tool_calls: list[FunctionCall] | None,
|
tool_calls: list[FunctionCall] | None,
|
||||||
logprobs: list[Logprob] | None = None,
|
logprobs: list[Logprob] | None = None,
|
||||||
tools: list[Tool] | None = None,
|
tools: list[Tool] | None = None,
|
||||||
|
finish_reason: str | None = None,
|
||||||
) -> list[ResponseOutputItem]:
|
) -> list[ResponseOutputItem]:
|
||||||
|
"""Build the output items of one generation.
|
||||||
|
|
||||||
|
Only the last item can have been cut short, so ``finish_reason`` decides
|
||||||
|
that item's status and every earlier item is ``completed``.
|
||||||
|
"""
|
||||||
outputs: list[ResponseOutputItem] = []
|
outputs: list[ResponseOutputItem] = []
|
||||||
tool_call_name_map = build_responses_tool_call_name_map(tools)
|
tool_call_name_map = build_responses_tool_call_name_map(tools)
|
||||||
custom_names = custom_tool_names(tools)
|
custom_names = custom_tool_names(tools)
|
||||||
|
last_item_status = response_item_status(finish_reason)
|
||||||
|
|
||||||
if reasoning:
|
if reasoning:
|
||||||
outputs.append(
|
outputs.append(
|
||||||
@@ -137,13 +151,14 @@ def build_response_output_items(
|
|||||||
)
|
)
|
||||||
],
|
],
|
||||||
role="assistant",
|
role="assistant",
|
||||||
status="completed",
|
status="completed" if tool_calls else last_item_status,
|
||||||
type="message",
|
type="message",
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
|
|
||||||
if tool_calls:
|
if tool_calls:
|
||||||
for idx, tool_call in enumerate(tool_calls):
|
for idx, tool_call in enumerate(tool_calls):
|
||||||
|
is_last = idx == len(tool_calls) - 1
|
||||||
call_id = tool_call.id or make_tool_call_id(
|
call_id = tool_call.id or make_tool_call_id(
|
||||||
func_name=tool_call.name, idx=idx
|
func_name=tool_call.name, idx=idx
|
||||||
)
|
)
|
||||||
@@ -166,7 +181,7 @@ def build_response_output_items(
|
|||||||
id=f"fc_{random_uuid()}",
|
id=f"fc_{random_uuid()}",
|
||||||
call_id=call_id,
|
call_id=call_id,
|
||||||
type="function_call",
|
type="function_call",
|
||||||
status="completed",
|
status=last_item_status if is_last else "completed",
|
||||||
name=call_name.name,
|
name=call_name.name,
|
||||||
namespace=call_name.namespace,
|
namespace=call_name.namespace,
|
||||||
arguments=tool_call.arguments,
|
arguments=tool_call.arguments,
|
||||||
|
|||||||
@@ -4,6 +4,7 @@ from typing import TypeVar
|
|||||||
|
|
||||||
from vllm.entrypoints.generate.base.protocol import DeltaMessage, FunctionCall
|
from vllm.entrypoints.generate.base.protocol import DeltaMessage, FunctionCall
|
||||||
from vllm.entrypoints.openai.chat_completion.protocol import (
|
from vllm.entrypoints.openai.chat_completion.protocol import (
|
||||||
|
ChatCompletionNamedToolChoiceParam,
|
||||||
ChatCompletionRequest,
|
ChatCompletionRequest,
|
||||||
ChatCompletionResponseChoice,
|
ChatCompletionResponseChoice,
|
||||||
ChatCompletionResponseStreamChoice,
|
ChatCompletionResponseStreamChoice,
|
||||||
@@ -41,3 +42,24 @@ def maybe_filter_parallel_tool_calls(
|
|||||||
tool_call for tool_call in delta.tool_calls if tool_call.index == 0
|
tool_call for tool_call in delta.tool_calls if tool_call.index == 0
|
||||||
]
|
]
|
||||||
return item
|
return item
|
||||||
|
|
||||||
|
|
||||||
|
def resolve_finish_reason(
|
||||||
|
engine_finish_reason: str | None,
|
||||||
|
request: ChatCompletionRequest | None,
|
||||||
|
tool_calls_made: bool,
|
||||||
|
) -> str:
|
||||||
|
"""Map the engine finish reason to the OpenAI chat ``finish_reason``.
|
||||||
|
|
||||||
|
``tool_calls`` is reported only when the choice actually carries tool
|
||||||
|
calls, the engine stopped on its own (a ``length`` cut stays visible so
|
||||||
|
clients can tell a truncated call from a complete one) and the caller did
|
||||||
|
not force a single named tool, for which OpenAI reports ``stop``.
|
||||||
|
"""
|
||||||
|
finish_reason = engine_finish_reason or "stop"
|
||||||
|
forced_named_tool = request is not None and isinstance(
|
||||||
|
request.tool_choice, ChatCompletionNamedToolChoiceParam
|
||||||
|
)
|
||||||
|
if tool_calls_made and finish_reason == "stop" and not forced_named_tool:
|
||||||
|
return "tool_calls"
|
||||||
|
return finish_reason
|
||||||
|
|||||||
@@ -37,6 +37,7 @@ from vllm.entrypoints.serve.engine.protocol import UsageInfo
|
|||||||
from vllm.entrypoints.serve.utils.request_logger import RequestLogger
|
from vllm.entrypoints.serve.utils.request_logger import RequestLogger
|
||||||
from vllm.entrypoints.serve.utils.tool_calls_utils import (
|
from vllm.entrypoints.serve.utils.tool_calls_utils import (
|
||||||
maybe_filter_parallel_tool_calls,
|
maybe_filter_parallel_tool_calls,
|
||||||
|
resolve_finish_reason,
|
||||||
)
|
)
|
||||||
from vllm.logger import init_logger
|
from vllm.logger import init_logger
|
||||||
from vllm.parser import Parser, ParserManager
|
from vllm.parser import Parser, ParserManager
|
||||||
@@ -205,7 +206,9 @@ class OnlineDerenderer:
|
|||||||
index=choice.index,
|
index=choice.index,
|
||||||
message=message,
|
message=message,
|
||||||
logprobs=resolved_logprobs,
|
logprobs=resolved_logprobs,
|
||||||
finish_reason=choice.finish_reason,
|
finish_reason=resolve_finish_reason(
|
||||||
|
choice.finish_reason, chat_request, bool(message.tool_calls)
|
||||||
|
),
|
||||||
stop_reason=choice.stop_reason,
|
stop_reason=choice.stop_reason,
|
||||||
token_ids=choice.token_ids if return_token_ids else None,
|
token_ids=choice.token_ids if return_token_ids else None,
|
||||||
)
|
)
|
||||||
@@ -664,11 +667,9 @@ class OnlineDerenderer:
|
|||||||
|
|
||||||
finish_reason = choice.finish_reason
|
finish_reason = choice.finish_reason
|
||||||
if finish_reason is not None:
|
if finish_reason is not None:
|
||||||
is_named_tool_choice = (
|
finish_reason = resolve_finish_reason(
|
||||||
type(chat_request.tool_choice) is ChatCompletionNamedToolChoiceParam
|
finish_reason, chat_request, tools_streamed
|
||||||
)
|
)
|
||||||
if tools_streamed and not is_named_tool_choice:
|
|
||||||
finish_reason = "tool_calls"
|
|
||||||
|
|
||||||
stream_choice = ChatCompletionResponseStreamChoice(
|
stream_choice = ChatCompletionResponseStreamChoice(
|
||||||
index=choice.index,
|
index=choice.index,
|
||||||
|
|||||||
Reference in New Issue
Block a user