[responseAPI] support partial message generation (#32100)
Signed-off-by: Andrew Xia <axia@fb.com> Signed-off-by: Andrew Xia <mitandrewxia@gmail.com> Signed-off-by: Lu Fang <30275821+houseroad@users.noreply.github.com> Co-authored-by: Andrew Xia <axia@fb.com> Co-authored-by: Lu Fang <30275821+houseroad@users.noreply.github.com>
This commit is contained in:
co-authored by
Andrew Xia
Lu Fang
parent
6beef12b9b
commit
af54d2e2d0
@@ -111,6 +111,7 @@ from vllm.entrypoints.responses_utils import (
|
||||
construct_input_messages,
|
||||
construct_tool_dicts,
|
||||
extract_tool_types,
|
||||
should_continue_final_message,
|
||||
)
|
||||
from vllm.entrypoints.tool_server import ToolServer
|
||||
from vllm.inputs.data import TokensPrompt
|
||||
@@ -590,6 +591,10 @@ class OpenAIServingResponses(OpenAIServing):
|
||||
prev_response_output=prev_response.output if prev_response else None,
|
||||
)
|
||||
|
||||
# Check if we should continue the final message (partial completion)
|
||||
# This enables Anthropic-style partial message completion where the
|
||||
# user provides an incomplete assistant message to continue from.
|
||||
continue_final = should_continue_final_message(request.input)
|
||||
chat_template_kwargs = dict(
|
||||
reasoning_effort=None
|
||||
if request.reasoning is None
|
||||
@@ -604,6 +609,11 @@ class OpenAIServingResponses(OpenAIServing):
|
||||
tool_parser=self.tool_parser,
|
||||
chat_template=self.chat_template,
|
||||
chat_template_content_format=self.chat_template_content_format,
|
||||
# When continuing a partial message, we set continue_final_message=True
|
||||
# and add_generation_prompt=False so the model continues the message
|
||||
# rather than starting a new one.
|
||||
add_generation_prompt=not continue_final,
|
||||
continue_final_message=continue_final,
|
||||
chat_template_kwargs=chat_template_kwargs,
|
||||
)
|
||||
return messages, engine_prompts
|
||||
|
||||
@@ -28,6 +28,53 @@ from vllm.entrypoints.openai.engine.protocol import (
|
||||
)
|
||||
|
||||
|
||||
def should_continue_final_message(
|
||||
request_input: str | list[ResponseInputOutputItem],
|
||||
) -> bool:
|
||||
"""
|
||||
Determine if the last input message is a partial assistant message
|
||||
that should be continued rather than starting a new generation.
|
||||
|
||||
This enables partial message completion similar to Anthropic's Messages API,
|
||||
where users can provide an incomplete assistant message and have the model
|
||||
continue from where it left off.
|
||||
|
||||
A message is considered partial if:
|
||||
1. It's a ResponseOutputMessage or ResponseReasoningItem
|
||||
2. Its status is "in_progress" or "incomplete"
|
||||
|
||||
Args:
|
||||
request_input: The input to the Responses API request
|
||||
|
||||
Returns:
|
||||
True if the final message should be continued, False otherwise
|
||||
"""
|
||||
if isinstance(request_input, str):
|
||||
# Simple string input is always a user message
|
||||
return False
|
||||
|
||||
if not request_input:
|
||||
return False
|
||||
|
||||
last_item = request_input[-1]
|
||||
|
||||
# Check if the last item is a partial assistant message
|
||||
if isinstance(last_item, ResponseOutputMessage):
|
||||
return last_item.status in ("in_progress", "incomplete")
|
||||
|
||||
# Check if the last item is a partial reasoning item
|
||||
if isinstance(last_item, ResponseReasoningItem):
|
||||
return last_item.status in ("in_progress", "incomplete")
|
||||
|
||||
if isinstance(last_item, dict):
|
||||
# only support partial completion for messages for now
|
||||
if last_item.get("type", "message") not in ("message", "reasoning"):
|
||||
return False
|
||||
return last_item.get("status") in ("in_progress", "incomplete")
|
||||
|
||||
return False
|
||||
|
||||
|
||||
def construct_input_messages(
|
||||
*,
|
||||
request_instructions: str | None = None,
|
||||
|
||||
Reference in New Issue
Block a user