diff --git a/tests/entrypoints/openai/chat_completion/test_serving_chat.py b/tests/entrypoints/openai/chat_completion/test_serving_chat.py
index d22f581f67..747b84a938 100644
--- a/tests/entrypoints/openai/chat_completion/test_serving_chat.py
+++ b/tests/entrypoints/openai/chat_completion/test_serving_chat.py
@@ -42,6 +42,7 @@ from vllm.entrypoints.openai.models.serving import (
)
from vllm.entrypoints.openai.parser.harmony_utils import get_encoding
from vllm.entrypoints.serve.engine.protocol import ErrorResponse
+from vllm.entrypoints.serve.utils.tool_calls_utils import resolve_finish_reason
from vllm.exceptions import QueueOverflowError, VLLMValidationError
from vllm.inputs import TokensPrompt
from vllm.logprobs import Logprob
@@ -3051,3 +3052,137 @@ def test_chat_kv_transfer_prompt_token_ids_allows_text_only_parts():
kv_transfer_params={"prompt_token_ids": [10, 20, 30]},
)
assert request.kv_transfer_params == {"prompt_token_ids": [10, 20, 30]}
+
+
+_TOOL_FINISH_REASON_TOOLS = [
+ {
+ "type": "function",
+ "function": {
+ "name": "get_weather",
+ "description": "Get weather",
+ "parameters": {
+ "type": "object",
+ "properties": {"city": {"type": "string"}},
+ "required": ["city"],
+ },
+ },
+ }
+]
+_HERMES_TOOL_CALL_TEXT = (
+ '\n{"name": "get_weather", "arguments": {"city": "Tokyo"}}\n'
+)
+
+
+_NAMED_WEATHER_TOOL_CHOICE = {"type": "function", "function": {"name": "get_weather"}}
+
+
+@pytest.mark.parametrize(
+ ("tool_choice", "engine_finish_reason", "tool_calls_made", "expected"),
+ [
+ ("auto", "stop", True, "tool_calls"),
+ ("auto", "length", True, "length"),
+ ("auto", "stop", False, "stop"),
+ ("required", "stop", True, "tool_calls"),
+ ("required", "stop", False, "stop"),
+ ("required", "length", True, "length"),
+ (_NAMED_WEATHER_TOOL_CHOICE, "stop", True, "stop"),
+ (_NAMED_WEATHER_TOOL_CHOICE, "length", True, "length"),
+ ("none", "stop", False, "stop"),
+ (None, None, False, "stop"),
+ ],
+)
+def test_resolve_finish_reason(
+ tool_choice, engine_finish_reason, tool_calls_made, expected
+):
+ """``tool_calls`` needs a real call, a natural stop and no forced tool;
+ ``length`` is never hidden behind a parsed call."""
+ request = ChatCompletionRequest(
+ model=MODEL_NAME,
+ messages=[{"role": "user", "content": "test"}],
+ tools=_TOOL_FINISH_REASON_TOOLS if tool_choice else None,
+ tool_choice=tool_choice,
+ )
+ assert (
+ resolve_finish_reason(engine_finish_reason, request, tool_calls_made)
+ == expected
+ )
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize(
+ ("tool_choice", "text", "engine_finish_reason", "expected_finish_reason"),
+ [
+ ("auto", _HERMES_TOOL_CALL_TEXT, "stop", "tool_calls"),
+ ("auto", _HERMES_TOOL_CALL_TEXT, "length", "length"),
+ ("required", "Hello, how are you?", "stop", "stop"),
+ (_NAMED_WEATHER_TOOL_CHOICE, _HERMES_TOOL_CALL_TEXT, "stop", "stop"),
+ ],
+)
+async def test_non_streaming_finish_reason_matches_tool_calls(
+ monkeypatch: pytest.MonkeyPatch,
+ tool_choice,
+ text: str,
+ engine_finish_reason: str,
+ expected_finish_reason: str,
+):
+ """Non-streaming ``finish_reason`` must agree with ``message.tool_calls``
+ and never hide a ``length`` truncation (#53255, #58228)."""
+ serving_chat = _build_serving_chat(
+ _build_mock_engine(), tool_parser="hermes", enable_auto_tools=True
+ )
+ # Parse required/named calls from the native format regardless
+ # of VLLM_ENFORCE_STRICT_TOOL_CALLING, so one text serves every case.
+ monkeypatch.setattr(
+ serving_chat.parser_cls.tool_parser_cls, "supports_required_and_named", False
+ )
+
+ tokenizer = get_tokenizer(MODEL_NAME)
+ request = ChatCompletionRequest(
+ model=MODEL_NAME,
+ messages=[{"role": "user", "content": "test"}],
+ tools=_TOOL_FINISH_REASON_TOOLS,
+ tool_choice=tool_choice,
+ )
+ parser = serving_chat.parser_cls(
+ tokenizer,
+ request.tools,
+ chat_template_kwargs=serving_chat._effective_chat_template_kwargs(request),
+ model_config=serving_chat.model_config,
+ )
+ request_output = RequestOutput(
+ request_id="test-req",
+ prompt="test",
+ prompt_token_ids=[1, 2, 3],
+ prompt_logprobs=None,
+ outputs=[
+ CompletionOutput(
+ index=0,
+ text=text,
+ token_ids=tokenizer.encode(text, add_special_tokens=False),
+ cumulative_logprob=0.0,
+ logprobs=None,
+ finish_reason=engine_finish_reason,
+ )
+ ],
+ finished=True,
+ )
+
+ response = await serving_chat.chat_completion_full_generator(
+ request=request,
+ result_generator=_single_request_output(request_output),
+ request_id="test-req",
+ model_name=MODEL_NAME,
+ conversation=[],
+ tokenizer=tokenizer,
+ request_metadata=RequestResponseMetadata(
+ request_id="test-req", model_name=MODEL_NAME
+ ),
+ parser=parser,
+ )
+
+ assert isinstance(response, ChatCompletionResponse)
+ choice = response.choices[0]
+ assert choice.finish_reason == expected_finish_reason
+ assert bool(choice.message.tool_calls) == (text != "Hello, how are you?"), (
+ "finish_reason must reflect whether tool calls were emitted"
+ )
diff --git a/tests/entrypoints/openai/responses/test_parsable_context_unit.py b/tests/entrypoints/openai/responses/test_parsable_context_unit.py
index f3927c9d71..db11443ad1 100644
--- a/tests/entrypoints/openai/responses/test_parsable_context_unit.py
+++ b/tests/entrypoints/openai/responses/test_parsable_context_unit.py
@@ -456,3 +456,27 @@ def test_num_init_messages_offset():
items = ctx.make_response_output_items()
assert len(items) == 1
assert items[0].type == "message"
+
+
+def test_length_truncation_marks_last_item_incomplete():
+ """A turn cut by max_output_tokens ends in an ``incomplete`` item, both
+ with a parser (tool call) and without one (plain message)."""
+ request = _make_request(
+ tool_choice="auto",
+ tools=[
+ {
+ "type": "function",
+ "name": "get_weather",
+ "parameters": {"type": "object", "properties": {}},
+ }
+ ],
+ )
+ ctx = _make_context(_ToolCallingParser, request=request, enable_auto_tools=True)
+ ctx.append_output(_make_request_output(text="calling tool", finish_reason="length"))
+ assert ctx.response_messages[-1].type == "function_call"
+ assert ctx.response_messages[-1].status == "incomplete"
+
+ ctx = _make_context(None)
+ ctx.append_output(_make_request_output(text="cut mid sen", finish_reason="length"))
+ assert ctx.response_messages[-1].type == "message"
+ assert ctx.response_messages[-1].status == "incomplete"
diff --git a/tests/entrypoints/openai/responses/test_responses_utils.py b/tests/entrypoints/openai/responses/test_responses_utils.py
index 6789ddc08b..13da0a1e09 100644
--- a/tests/entrypoints/openai/responses/test_responses_utils.py
+++ b/tests/entrypoints/openai/responses/test_responses_utils.py
@@ -991,3 +991,28 @@ class TestConstructInputMessagesInstructionsLeak:
assert len(msgs) == 2
assert msgs[0] == {"role": "system", "content": "be helpful"}
assert msgs[1] == {"role": "user", "content": "hello"}
+
+
+class TestBuildResponseOutputItemsStatus:
+ """Item ``status`` must follow the finish reason of the generation
+ (#57998): a ``length`` cut marks the item that was cut ``incomplete``."""
+
+ _CALLS = [
+ FunctionCall(name="get_weather", arguments='{"city": "Paris"}'),
+ FunctionCall(name="get_weather", arguments='{"city": "Tok'),
+ ]
+
+ def test_length_marks_truncated_message_incomplete(self):
+ (item,) = build_response_output_items(
+ reasoning=None, content="cut mid sen", tool_calls=[], finish_reason="length"
+ )
+ assert item.type == "message"
+ assert item.status == "incomplete"
+
+ def test_length_marks_only_last_tool_call_incomplete(self):
+ message, first, last = build_response_output_items(
+ reasoning=None, content="hi", tool_calls=self._CALLS, finish_reason="length"
+ )
+ assert message.status == "completed"
+ assert first.status == "completed"
+ assert last.status == "incomplete"
diff --git a/tests/entrypoints/openai/responses/test_serving_responses.py b/tests/entrypoints/openai/responses/test_serving_responses.py
index 1622d0e8f8..67b9dd4e4f 100644
--- a/tests/entrypoints/openai/responses/test_serving_responses.py
+++ b/tests/entrypoints/openai/responses/test_serving_responses.py
@@ -1,6 +1,8 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+import asyncio
+from collections import deque
from contextlib import AsyncExitStack, asynccontextmanager
from types import SimpleNamespace
from unittest.mock import AsyncMock, MagicMock
@@ -51,6 +53,7 @@ from vllm.entrypoints.openai.responses.context import (
from vllm.entrypoints.openai.responses.protocol import (
ResponseCompletedEvent,
ResponseCreatedEvent,
+ ResponseIncompleteEvent,
ResponseRawMessageAndToken,
ResponsesRequest,
ResponsesResponse,
@@ -1414,6 +1417,7 @@ def _make_request_output(
text,
token_ids,
metrics: RequestStateStats | None = None,
+ finish_reason: str | None = None,
):
completion = CompletionOutput(
index=0,
@@ -1421,7 +1425,7 @@ def _make_request_output(
token_ids=token_ids,
cumulative_logprob=0.0,
logprobs=None,
- finish_reason=None,
+ finish_reason=finish_reason,
stop_reason=None,
)
return RequestOutput(
@@ -1441,10 +1445,11 @@ def _make_simple_context_with_output(
token_ids,
response_parser=None,
metrics: RequestStateStats | None = None,
+ finish_reason: str | None = None,
):
"""Create a SimpleContext with a RequestOutput containing the given text."""
ctx = SimpleContext(response_parser=response_parser)
- req_output = _make_request_output(text, token_ids, metrics)
+ req_output = _make_request_output(text, token_ids, metrics, finish_reason)
ctx.append_output(req_output)
ctx.request_metrics = metrics
return ctx
@@ -1561,6 +1566,66 @@ async def test_responses_streaming_metrics_only_on_completed_event():
assert events[-1].response.metrics.time_to_first_token_ms == pytest.approx(500.0)
+async def _stream_truncated_by_length(serving: OpenAIServingResponses):
+ request = ResponsesRequest(input="hi", tools=[], stream=True, store=False)
+ context = _make_simple_context_with_output(
+ "cut mid sen", [10, 20], finish_reason="length"
+ )
+
+ async def one_context():
+ yield context
+
+ return [
+ event
+ async for event in serving.responses_stream_generator(
+ request=request,
+ sampling_params=SamplingParams(max_tokens=2),
+ result_generator=one_context(),
+ context=context,
+ model_name="test-model",
+ tokenizer=MagicMock(),
+ request_metadata=RequestResponseMetadata(request_id="req"),
+ )
+ ]
+
+
+@pytest.mark.asyncio
+async def test_responses_stream_truncated_by_length_reports_incomplete():
+ """A max_output_tokens cut must end the stream with ``response.incomplete``
+ and mark the cut item ``incomplete`` in both ``output_item.done`` and the
+ final response (#57998)."""
+ events = await _stream_truncated_by_length(_make_serving_instance())
+
+ item_done = [e for e in events if e.type == "response.output_item.done"]
+ assert [e.item.status for e in item_done] == ["incomplete"]
+ final = events[-1]
+ assert isinstance(final, ResponseIncompleteEvent)
+ assert final.response.status == "incomplete"
+ assert final.response.incomplete_details.reason == "max_output_tokens"
+ assert [item.status for item in final.response.output] == ["incomplete"]
+
+
+@pytest.mark.asyncio
+async def test_background_stream_replay_stops_on_response_incomplete():
+ """Replaying a stored background stream must end at ``response.incomplete``
+ as it does at ``response.completed``, instead of waiting for more events."""
+ serving = _make_serving_instance()
+ events = await _stream_truncated_by_length(serving)
+ assert isinstance(events[-1], ResponseIncompleteEvent)
+ serving.event_store["resp_bg"] = (deque(events), asyncio.Event())
+
+ async def replay(starting_after=None):
+ return [
+ event
+ async for event in serving.responses_background_stream_generator(
+ "resp_bg", starting_after=starting_after
+ )
+ ]
+
+ assert await asyncio.wait_for(replay(), timeout=5) == events
+ assert await asyncio.wait_for(replay(starting_after=0), timeout=5) == events[1:]
+
+
@pytest.mark.asyncio
async def test_responses_metrics_suppressed_for_multiple_generation_turns():
response = await _make_full_metrics_response(
diff --git a/tests/entrypoints/scale_out/derender/test_derender.py b/tests/entrypoints/scale_out/derender/test_derender.py
index 30d64ed39b..771213309f 100644
--- a/tests/entrypoints/scale_out/derender/test_derender.py
+++ b/tests/entrypoints/scale_out/derender/test_derender.py
@@ -3,11 +3,18 @@
"""Tests for the /derender endpoints (postprocessing counterpart to /render)."""
+from unittest.mock import MagicMock
+
import httpx
import pytest
import pytest_asyncio
from tests.utils import RemoteLaunchRenderServer
+from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
+from vllm.entrypoints.scale_out.token_in_token_out.protocol import (
+ GenerateTokensResponse,
+)
+from vllm.renderers.online_derenderer import OnlineDerenderer
from vllm.tokenizers import get_tokenizer
MODEL_NAME = "hmellor/tiny-random-LlamaForCausalLM"
@@ -1432,3 +1439,55 @@ async def test_e2e_harmony_reasoning(harmony_client, harmony_tokenizer):
assert msg["reasoning"] is not None
assert reasoning_text in msg["reasoning"]
assert answer_text in (msg["content"] or "")
+
+
+_HERMES_TOOL_CALL_TEXT = (
+ '\n{"name": "get_weather", "arguments": {"city": "Paris"}}\n'
+)
+
+
+@pytest.mark.parametrize(
+ ("tool_choice", "text", "engine_finish_reason", "expected_finish_reason"),
+ [
+ ("auto", _HERMES_TOOL_CALL_TEXT, "stop", "tool_calls"),
+ ("auto", _HERMES_TOOL_CALL_TEXT, "length", "length"),
+ ("required", "Hello there", "stop", "stop"),
+ ],
+)
+def test_derender_chat_finish_reason_matches_chat_completion(
+ tool_choice, text, engine_finish_reason, expected_finish_reason
+):
+ """/derender applies the same finish_reason rule as the coupled chat
+ path: ``tool_calls`` only for a real call at a natural stop."""
+ tokenizer = get_tokenizer(MODEL_NAME)
+ renderer = MagicMock()
+ renderer.get_tokenizer.return_value = tokenizer
+ model_config = MagicMock()
+ model_config.model = MODEL_NAME
+ model_config.hf_config.model_type = "llama"
+ derenderer = OnlineDerenderer(
+ model_config,
+ renderer,
+ request_logger=None,
+ chat_template="{{ messages }}",
+ chat_template_content_format="auto",
+ enable_auto_tools=True,
+ tool_parser="hermes",
+ )
+ request = ChatCompletionRequest(
+ model=MODEL_NAME,
+ messages=[{"role": "user", "content": "Weather in Paris?"}],
+ tools=_E2E_TOOLS,
+ tool_choice=tool_choice,
+ )
+ response = GenerateTokensResponse.model_validate(
+ _make_generate_response(
+ tokenizer.encode(text, add_special_tokens=False),
+ finish_reason=engine_finish_reason,
+ )
+ )
+
+ (choice,) = derenderer._derender_chat(response, request)
+
+ assert choice.finish_reason == expected_finish_reason
+ assert bool(choice.message.tool_calls) == (text == _HERMES_TOOL_CALL_TEXT)
diff --git a/tests/entrypoints/scale_out/derender/test_derender_stream.py b/tests/entrypoints/scale_out/derender/test_derender_stream.py
index 55b0789843..a1c4e2a330 100644
--- a/tests/entrypoints/scale_out/derender/test_derender_stream.py
+++ b/tests/entrypoints/scale_out/derender/test_derender_stream.py
@@ -1289,7 +1289,15 @@ class TestDerenderChatStreamParsed:
assert state.last_tool_call_ids == [first_id]
@pytest.mark.asyncio
- async def test_finish_reason_rewritten_to_tool_calls(self, parsed_derenderer):
+ @pytest.mark.parametrize(
+ ("engine_finish_reason", "expected_finish_reason"),
+ [("stop", "tool_calls"), ("length", "length")],
+ )
+ async def test_finish_reason_rewritten_to_tool_calls(
+ self, parsed_derenderer, engine_finish_reason, expected_finish_reason
+ ):
+ """A streamed tool call is reported as ``tool_calls`` only at a
+ natural stop; a ``length`` cut must stay visible."""
chat_request = _chat_request(
tools=[{"type": "function", "function": {"name": "get_weather"}}],
tool_choice="auto",
@@ -1297,11 +1305,12 @@ class TestDerenderChatStreamParsed:
chunk, _ = await parsed_derenderer.derender_chat_stream(
model=MODEL_NAME,
generate_chunk=_make_stream_chunk(
- [_FakeParser.TOOL_START, _FakeParser.TOOL_ARG], finish_reason="stop"
+ [_FakeParser.TOOL_START, _FakeParser.TOOL_ARG],
+ finish_reason=engine_finish_reason,
),
chat_request=chat_request,
)
- assert chunk.choices[0].finish_reason == "tool_calls"
+ assert chunk.choices[0].finish_reason == expected_finish_reason
@pytest.mark.asyncio
async def test_finish_reason_stop_for_named_tool_choice(self, parsed_derenderer):
diff --git a/vllm/entrypoints/openai/chat_completion/serving.py b/vllm/entrypoints/openai/chat_completion/serving.py
index 84061a8822..007c961ca8 100644
--- a/vllm/entrypoints/openai/chat_completion/serving.py
+++ b/vllm/entrypoints/openai/chat_completion/serving.py
@@ -48,6 +48,7 @@ from vllm.entrypoints.serve.utils.api_utils import get_max_tokens, should_includ
from vllm.entrypoints.serve.utils.request_logger import RequestLogger
from vllm.entrypoints.serve.utils.tool_calls_utils import (
maybe_filter_parallel_tool_calls,
+ resolve_finish_reason,
)
from vllm.exceptions import GenerationError
from vllm.inputs import EngineInput, MultiModalPlaceholders
@@ -491,11 +492,6 @@ class OpenAIServingChat(GenerateBaseServing):
num_cache_creation_tokens = None
tools_streamed = [False] * num_choices
- if isinstance(request.tool_choice, ChatCompletionNamedToolChoiceParam):
- tool_choice_function_name = request.tool_choice.function.name
- else:
- tool_choice_function_name = None
-
previous_texts = [""] * num_choices
try:
@@ -764,25 +760,13 @@ class OpenAIServingChat(GenerateBaseServing):
# if the model is finished generating
else:
# Send the finish response for each request.n only once
- # In OpenAI's API, when a tool is called, the
- # finish_reason is:
- # "tool_calls" for "auto" or "required" tool calls,
- # and "stop" for named tool calls.
- if (
- tools_streamed[i]
- and not tool_choice_function_name
- and output.finish_reason == "stop"
- ):
- finish_reason_ = "tool_calls"
- else:
- finish_reason_ = (
- output.finish_reason if output.finish_reason else "stop"
- )
choice_data = ChatCompletionResponseStreamChoice(
index=i,
delta=delta_message,
logprobs=logprobs,
- finish_reason=finish_reason_,
+ finish_reason=resolve_finish_reason(
+ output.finish_reason, request, tools_streamed[i]
+ ),
stop_reason=output.stop_reason,
token_ids=(
as_list(output.token_ids) if include_token_ids else None
@@ -1013,7 +997,6 @@ class OpenAIServingChat(GenerateBaseServing):
tool_calls = []
suppress_metadata = False
- auto_tools_called = False
is_named_tool_choice = (
request.tool_choice is not None
and type(request.tool_choice) is ChatCompletionNamedToolChoiceParam
@@ -1056,7 +1039,6 @@ class OpenAIServingChat(GenerateBaseServing):
and self.enable_auto_tools
and tool_parser_cls
):
- auto_tools_called = tool_calls is not None and len(tool_calls) > 0
if tool_calls:
message = self._create_chat_message(
role=role,
@@ -1093,15 +1075,6 @@ class OpenAIServingChat(GenerateBaseServing):
# metadata cached on the reasoning parser.
message = self._finalize_response_message(message, parser=parser)
- # In OpenAI's API, when a tool is called, the finish_reason is:
- # "tool_calls" for "auto" or "required" tool calls,
- # and "stop" for named tool calls.
- is_finish_reason_tool_calls = auto_tools_called or (
- request.tool_choice
- and request.tool_choice == "required"
- and output.finish_reason == "stop"
- )
-
routed_experts_b64 = (
numpy2base64(output.routed_experts)
if output.routed_experts is not None
@@ -1112,11 +1085,9 @@ class OpenAIServingChat(GenerateBaseServing):
index=output.index,
message=message,
logprobs=logprobs,
- finish_reason="tool_calls"
- if is_finish_reason_tool_calls
- else output.finish_reason
- if output.finish_reason
- else "stop",
+ finish_reason=resolve_finish_reason(
+ output.finish_reason, request, bool(message.tool_calls)
+ ),
stop_reason=output.stop_reason,
token_ids=(
as_list(output.token_ids)
diff --git a/vllm/entrypoints/openai/responses/context.py b/vllm/entrypoints/openai/responses/context.py
index 9f5a281163..6aaae51910 100644
--- a/vllm/entrypoints/openai/responses/context.py
+++ b/vllm/entrypoints/openai/responses/context.py
@@ -32,6 +32,7 @@ from vllm.entrypoints.openai.responses.protocol import (
from vllm.entrypoints.openai.responses.utils import (
build_response_output_items,
construct_tool_dicts,
+ response_item_status,
)
from vllm.entrypoints.serve.utils.tool_calls_utils import (
maybe_filter_parallel_tool_calls,
@@ -380,6 +381,7 @@ class ParsableContext(ConversationContext):
tool_calls or [], self.request
),
tools=self.request.tools,
+ finish_reason=completion.finish_reason,
)
)
elif completion.text:
@@ -387,7 +389,7 @@ class ParsableContext(ConversationContext):
ResponseOutputMessage(
type="message",
id=f"msg_{random_uuid()}",
- status="completed",
+ status=response_item_status(completion.finish_reason),
role="assistant",
content=[
ResponseOutputText(
diff --git a/vllm/entrypoints/openai/responses/protocol.py b/vllm/entrypoints/openai/responses/protocol.py
index 200054a4ef..0e94bca51f 100644
--- a/vllm/entrypoints/openai/responses/protocol.py
+++ b/vllm/entrypoints/openai/responses/protocol.py
@@ -40,6 +40,9 @@ from openai.types.responses import (
ResponseCompletedEvent as OpenAIResponseCompletedEvent,
)
from openai.types.responses import ResponseCreatedEvent as OpenAIResponseCreatedEvent
+from openai.types.responses import (
+ ResponseIncompleteEvent as OpenAIResponseIncompleteEvent,
+)
from openai.types.responses import (
ResponseInProgressEvent as OpenAIResponseInProgressEvent,
)
@@ -927,6 +930,10 @@ class ResponseCompletedEvent(OpenAIResponseCompletedEvent):
response: ResponsesResponse # type: ignore[override]
+class ResponseIncompleteEvent(OpenAIResponseIncompleteEvent):
+ response: ResponsesResponse # type: ignore[override]
+
+
class ResponseCreatedEvent(OpenAIResponseCreatedEvent):
response: ResponsesResponse # type: ignore[override]
@@ -939,6 +946,7 @@ StreamingResponsesResponse: TypeAlias = (
ResponseCreatedEvent
| ResponseInProgressEvent
| ResponseCompletedEvent
+ | ResponseIncompleteEvent
| ResponseOutputItemAddedEvent
| ResponseOutputItemDoneEvent
| ResponseContentPartAddedEvent
diff --git a/vllm/entrypoints/openai/responses/serving.py b/vllm/entrypoints/openai/responses/serving.py
index 0d46855c94..dd154f968b 100644
--- a/vllm/entrypoints/openai/responses/serving.py
+++ b/vllm/entrypoints/openai/responses/serving.py
@@ -45,6 +45,7 @@ from vllm.entrypoints.openai.responses.protocol import (
OutputTokensDetails,
ResponseCompletedEvent,
ResponseCreatedEvent,
+ ResponseIncompleteEvent,
ResponseInProgressEvent,
ResponseInputOutputItem,
ResponseInputOutputMessage,
@@ -66,6 +67,7 @@ from vllm.entrypoints.openai.responses.utils import (
build_response_output_items,
extract_function_tool_names,
extract_tool_types,
+ response_item_status,
reuse_streamed_item_ids,
)
from vllm.entrypoints.serve.engine.protocol import ErrorResponse
@@ -1045,6 +1047,7 @@ class OpenAIServingResponses(GenerateBaseServing):
tool_calls=maybe_filter_parallel_tool_calls(tool_calls or [], request),
logprobs=logprobs,
tools=request.tools,
+ finish_reason=final_output.finish_reason,
)
# Fallback when no parser is configured
@@ -1062,7 +1065,7 @@ class OpenAIServingResponses(GenerateBaseServing):
if final_output.text
else [],
role="assistant",
- status="completed",
+ status=response_item_status(final_output.finish_reason),
type="message",
)
]
@@ -1124,7 +1127,10 @@ class OpenAIServingResponses(GenerateBaseServing):
while current_index < len(event_deque):
event = event_deque[current_index]
yield event
- if getattr(event, "type", "unknown") == "response.completed":
+ if getattr(event, "type", "unknown") in (
+ "response.completed",
+ "response.incomplete",
+ ):
return
current_index += 1
@@ -1222,6 +1228,7 @@ class OpenAIServingResponses(GenerateBaseServing):
top_logprobs=request.top_logprobs,
)
+ finish_reason: str | None = None
async for ctx in result_generator:
assert isinstance(ctx, SimpleContext)
if ctx.last_output is None or not ctx.last_output.outputs:
@@ -1229,6 +1236,7 @@ class OpenAIServingResponses(GenerateBaseServing):
output = ctx.last_output.outputs[0]
self._raise_if_error(output.finish_reason, request.request_id)
+ finish_reason = output.finish_reason
delta_text = output.text
delta_token_ids = as_list(output.token_ids)
@@ -1261,7 +1269,7 @@ class OpenAIServingResponses(GenerateBaseServing):
for event in processor.emit_delta(dm, output, _get_logprobs):
yield _increment_sequence_number_and_return(event)
- for event in processor.close_current():
+ for event in processor.close_current(response_item_status(finish_reason)):
yield _increment_sequence_number_and_return(event)
assert isinstance(context, SimpleContext)
@@ -1415,10 +1423,20 @@ class OpenAIServingResponses(GenerateBaseServing):
request_metadata,
created_time=created_time,
)
- yield _increment_sequence_number_and_return(
- ResponseCompletedEvent(
+ terminal_event: StreamingResponsesResponse
+ if (
+ isinstance(final_response, ResponsesResponse)
+ and final_response.status == "incomplete"
+ ):
+ terminal_event = ResponseIncompleteEvent(
+ type="response.incomplete",
+ sequence_number=-1,
+ response=final_response,
+ )
+ else:
+ terminal_event = ResponseCompletedEvent(
type="response.completed",
sequence_number=-1,
response=final_response,
)
- )
+ yield _increment_sequence_number_and_return(terminal_event)
diff --git a/vllm/entrypoints/openai/responses/streaming_events.py b/vllm/entrypoints/openai/responses/streaming_events.py
index 49107661a7..b14379cf12 100644
--- a/vllm/entrypoints/openai/responses/streaming_events.py
+++ b/vllm/entrypoints/openai/responses/streaming_events.py
@@ -18,7 +18,7 @@ import json
from collections.abc import Callable
from dataclasses import dataclass, field
from enum import Enum, auto
-from typing import Any, ClassVar, Final, NamedTuple
+from typing import Any, ClassVar, Final, Literal, NamedTuple
from openai.types.responses import (
ResponseCodeInterpreterCallCodeDeltaEvent,
@@ -910,6 +910,7 @@ def emit_simple_content_delta(
def emit_simple_content_done(
state: SimpleStreamingState,
+ status: Literal["completed", "incomplete"] = "completed",
) -> list[StreamingResponsesResponse]:
part = ResponseOutputText(
type="output_text",
@@ -944,7 +945,7 @@ def emit_simple_content_done(
type="message",
role="assistant",
content=[part] if state.accumulated_text else [],
- status="completed",
+ status=status,
),
),
]
@@ -1002,6 +1003,7 @@ def emit_simple_reasoning_delta(
def emit_simple_reasoning_done(
state: SimpleStreamingState,
+ status: Literal["completed", "incomplete"] = "completed",
) -> list[StreamingResponsesResponse]:
part = ResponseReasoningTextContent(
text=state.accumulated_text,
@@ -1031,7 +1033,7 @@ def emit_simple_reasoning_done(
item=ResponseReasoningItem(
type="reasoning",
content=[part],
- status="completed",
+ status=status,
id=state.current_item_id,
summary=[],
),
@@ -1134,6 +1136,7 @@ def emit_simple_tool_call_delta(
def emit_simple_tool_call_done(
state: SimpleStreamingState,
+ status: Literal["completed", "incomplete"] = "completed",
) -> list[StreamingResponsesResponse]:
events: list[StreamingResponsesResponse]
item: ResponseOutputItem
@@ -1172,7 +1175,7 @@ def emit_simple_tool_call_done(
name=state.tool_call_name,
namespace=state.tool_call_namespace,
arguments=state.accumulated_text,
- status="completed",
+ status=status,
id=state.current_item_id,
call_id=state.tool_call_id,
)
@@ -1312,12 +1315,14 @@ class SimpleStreamingEventProcessor:
and self.state.tool_call_index != tool_call.index
)
- def close_current(self) -> list[StreamingResponsesResponse]:
+ def close_current(
+ self, status: Literal["completed", "incomplete"] = "completed"
+ ) -> list[StreamingResponsesResponse]:
"""Close the current state and emit its 'done' event sequence."""
handlers = self._STATE_HANDLERS.get(self.state.current_state)
if handlers is None:
return []
- events = handlers.done_fn(self.state)
+ events = handlers.done_fn(self.state, status)
self.output_items.extend(
event.item
for event in events
diff --git a/vllm/entrypoints/openai/responses/utils.py b/vllm/entrypoints/openai/responses/utils.py
index d9b71d6849..d3e3d698c4 100644
--- a/vllm/entrypoints/openai/responses/utils.py
+++ b/vllm/entrypoints/openai/responses/utils.py
@@ -3,7 +3,7 @@
import json
from collections.abc import Iterable
-from typing import Any
+from typing import Any, Literal
import partial_json_parser
from openai.types.chat import (
@@ -100,16 +100,30 @@ def _item_dict(item: ResponseInputOutputItem) -> dict[str, Any]:
return item if isinstance(item, dict) else item.model_dump()
+def response_item_status(
+ finish_reason: str | None,
+) -> Literal["completed", "incomplete"]:
+ """Status of the output item that ended a generation with ``finish_reason``."""
+ return "incomplete" if finish_reason == "length" else "completed"
+
+
def build_response_output_items(
reasoning: str | None,
content: str | None,
tool_calls: list[FunctionCall] | None,
logprobs: list[Logprob] | None = None,
tools: list[Tool] | None = None,
+ finish_reason: str | None = None,
) -> list[ResponseOutputItem]:
+ """Build the output items of one generation.
+
+ Only the last item can have been cut short, so ``finish_reason`` decides
+ that item's status and every earlier item is ``completed``.
+ """
outputs: list[ResponseOutputItem] = []
tool_call_name_map = build_responses_tool_call_name_map(tools)
custom_names = custom_tool_names(tools)
+ last_item_status = response_item_status(finish_reason)
if reasoning:
outputs.append(
@@ -137,13 +151,14 @@ def build_response_output_items(
)
],
role="assistant",
- status="completed",
+ status="completed" if tool_calls else last_item_status,
type="message",
)
)
if tool_calls:
for idx, tool_call in enumerate(tool_calls):
+ is_last = idx == len(tool_calls) - 1
call_id = tool_call.id or make_tool_call_id(
func_name=tool_call.name, idx=idx
)
@@ -166,7 +181,7 @@ def build_response_output_items(
id=f"fc_{random_uuid()}",
call_id=call_id,
type="function_call",
- status="completed",
+ status=last_item_status if is_last else "completed",
name=call_name.name,
namespace=call_name.namespace,
arguments=tool_call.arguments,
diff --git a/vllm/entrypoints/serve/utils/tool_calls_utils.py b/vllm/entrypoints/serve/utils/tool_calls_utils.py
index 1252218127..07ed15b897 100644
--- a/vllm/entrypoints/serve/utils/tool_calls_utils.py
+++ b/vllm/entrypoints/serve/utils/tool_calls_utils.py
@@ -4,6 +4,7 @@ from typing import TypeVar
from vllm.entrypoints.generate.base.protocol import DeltaMessage, FunctionCall
from vllm.entrypoints.openai.chat_completion.protocol import (
+ ChatCompletionNamedToolChoiceParam,
ChatCompletionRequest,
ChatCompletionResponseChoice,
ChatCompletionResponseStreamChoice,
@@ -41,3 +42,24 @@ def maybe_filter_parallel_tool_calls(
tool_call for tool_call in delta.tool_calls if tool_call.index == 0
]
return item
+
+
+def resolve_finish_reason(
+ engine_finish_reason: str | None,
+ request: ChatCompletionRequest | None,
+ tool_calls_made: bool,
+) -> str:
+ """Map the engine finish reason to the OpenAI chat ``finish_reason``.
+
+ ``tool_calls`` is reported only when the choice actually carries tool
+ calls, the engine stopped on its own (a ``length`` cut stays visible so
+ clients can tell a truncated call from a complete one) and the caller did
+ not force a single named tool, for which OpenAI reports ``stop``.
+ """
+ finish_reason = engine_finish_reason or "stop"
+ forced_named_tool = request is not None and isinstance(
+ request.tool_choice, ChatCompletionNamedToolChoiceParam
+ )
+ if tool_calls_made and finish_reason == "stop" and not forced_named_tool:
+ return "tool_calls"
+ return finish_reason
diff --git a/vllm/renderers/online_derenderer.py b/vllm/renderers/online_derenderer.py
index f7e04fd1c7..09ea33857c 100644
--- a/vllm/renderers/online_derenderer.py
+++ b/vllm/renderers/online_derenderer.py
@@ -37,6 +37,7 @@ from vllm.entrypoints.serve.engine.protocol import UsageInfo
from vllm.entrypoints.serve.utils.request_logger import RequestLogger
from vllm.entrypoints.serve.utils.tool_calls_utils import (
maybe_filter_parallel_tool_calls,
+ resolve_finish_reason,
)
from vllm.logger import init_logger
from vllm.parser import Parser, ParserManager
@@ -205,7 +206,9 @@ class OnlineDerenderer:
index=choice.index,
message=message,
logprobs=resolved_logprobs,
- finish_reason=choice.finish_reason,
+ finish_reason=resolve_finish_reason(
+ choice.finish_reason, chat_request, bool(message.tool_calls)
+ ),
stop_reason=choice.stop_reason,
token_ids=choice.token_ids if return_token_ids else None,
)
@@ -664,11 +667,9 @@ class OnlineDerenderer:
finish_reason = choice.finish_reason
if finish_reason is not None:
- is_named_tool_choice = (
- type(chat_request.tool_choice) is ChatCompletionNamedToolChoiceParam
+ finish_reason = resolve_finish_reason(
+ finish_reason, chat_request, tools_streamed
)
- if tools_streamed and not is_named_tool_choice:
- finish_reason = "tool_calls"
stream_choice = ChatCompletionResponseStreamChoice(
index=choice.index,