[responseAPI] support partial message generation (#32100)

Signed-off-by: Andrew Xia <axia@fb.com>
Signed-off-by: Andrew Xia <mitandrewxia@gmail.com>
Signed-off-by: Lu Fang <30275821+houseroad@users.noreply.github.com>
Co-authored-by: Andrew Xia <axia@fb.com>
Co-authored-by: Lu Fang <30275821+houseroad@users.noreply.github.com>
This commit is contained in:
Andrew Xia
2026-01-13 10:41:26 -08:00
committed by GitHub
co-authored by Andrew Xia Lu Fang
parent 6beef12b9b
commit af54d2e2d0
3 changed files with 337 additions and 0 deletions
@@ -111,6 +111,7 @@ from vllm.entrypoints.responses_utils import (
construct_input_messages,
construct_tool_dicts,
extract_tool_types,
should_continue_final_message,
)
from vllm.entrypoints.tool_server import ToolServer
from vllm.inputs.data import TokensPrompt
@@ -590,6 +591,10 @@ class OpenAIServingResponses(OpenAIServing):
prev_response_output=prev_response.output if prev_response else None,
)
# Check if we should continue the final message (partial completion)
# This enables Anthropic-style partial message completion where the
# user provides an incomplete assistant message to continue from.
continue_final = should_continue_final_message(request.input)
chat_template_kwargs = dict(
reasoning_effort=None
if request.reasoning is None
@@ -604,6 +609,11 @@ class OpenAIServingResponses(OpenAIServing):
tool_parser=self.tool_parser,
chat_template=self.chat_template,
chat_template_content_format=self.chat_template_content_format,
# When continuing a partial message, we set continue_final_message=True
# and add_generation_prompt=False so the model continues the message
# rather than starting a new one.
add_generation_prompt=not continue_final,
continue_final_message=continue_final,
chat_template_kwargs=chat_template_kwargs,
)
return messages, engine_prompts
+47
View File
@@ -28,6 +28,53 @@ from vllm.entrypoints.openai.engine.protocol import (
)
def should_continue_final_message(
request_input: str | list[ResponseInputOutputItem],
) -> bool:
"""
Determine if the last input message is a partial assistant message
that should be continued rather than starting a new generation.
This enables partial message completion similar to Anthropic's Messages API,
where users can provide an incomplete assistant message and have the model
continue from where it left off.
A message is considered partial if:
1. It's a ResponseOutputMessage or ResponseReasoningItem
2. Its status is "in_progress" or "incomplete"
Args:
request_input: The input to the Responses API request
Returns:
True if the final message should be continued, False otherwise
"""
if isinstance(request_input, str):
# Simple string input is always a user message
return False
if not request_input:
return False
last_item = request_input[-1]
# Check if the last item is a partial assistant message
if isinstance(last_item, ResponseOutputMessage):
return last_item.status in ("in_progress", "incomplete")
# Check if the last item is a partial reasoning item
if isinstance(last_item, ResponseReasoningItem):
return last_item.status in ("in_progress", "incomplete")
if isinstance(last_item, dict):
# only support partial completion for messages for now
if last_item.get("type", "message") not in ("message", "reasoning"):
return False
return last_item.get("status") in ("in_progress", "incomplete")
return False
def construct_input_messages(
*,
request_instructions: str | None = None,