[Refactor] Responses API parser state into conversation context (#46030)
Signed-off-by: chaunceyjiang <chaunceyjiang@gmail.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
04c2a8deac
commit
3ce5823762
@@ -386,8 +386,15 @@ async def test_reasoning_tokens_counted_for_text_reasoning_model(monkeypatch):
|
||||
reasoning_parser="qwen3",
|
||||
)
|
||||
|
||||
request = ResponsesRequest(input="hi", tools=[], stream=False)
|
||||
response_parser = serving._make_response_parser(
|
||||
request,
|
||||
tokenizer,
|
||||
serving._effective_chat_template_kwargs(request),
|
||||
)
|
||||
|
||||
# Build a SimpleContext with thinking tokens in the output.
|
||||
context = SimpleContext()
|
||||
context = SimpleContext(response_parser=response_parser)
|
||||
token_ids = [1, 10, 2, 20] # <think> 10 </think> 20 -> reasoning token count = 1
|
||||
completion = CompletionOutput(
|
||||
index=0,
|
||||
@@ -412,7 +419,6 @@ async def test_reasoning_tokens_counted_for_text_reasoning_model(monkeypatch):
|
||||
async def dummy_result_generator():
|
||||
yield None
|
||||
|
||||
request = ResponsesRequest(input="hi", tools=[], stream=False)
|
||||
sampling_params = SamplingParams(max_tokens=16)
|
||||
metadata = RequestResponseMetadata(request_id="req")
|
||||
|
||||
@@ -636,9 +642,9 @@ class TestHarmonyPreambleStreaming:
|
||||
assert "response.output_text.done" not in type_names
|
||||
|
||||
|
||||
def _make_simple_context_with_output(text, token_ids):
|
||||
def _make_simple_context_with_output(text, token_ids, response_parser=None):
|
||||
"""Create a SimpleContext with a RequestOutput containing the given text."""
|
||||
ctx = SimpleContext()
|
||||
ctx = SimpleContext(response_parser=response_parser)
|
||||
completion = CompletionOutput(
|
||||
index=0,
|
||||
text=text,
|
||||
@@ -719,6 +725,7 @@ def _mock_parser_with_reasoning(serving, delta_sequence: list[DeltaMessage]):
|
||||
mock_parser_instance.parse_delta = mock_parse_delta
|
||||
mock_parser_instance.is_reasoning_end = MagicMock(return_value=False)
|
||||
serving.parser = MagicMock(return_value=mock_parser_instance)
|
||||
return mock_parser_instance
|
||||
|
||||
|
||||
class TestStreamingReasoningToContentTransition:
|
||||
@@ -745,12 +752,12 @@ class TestStreamingReasoningToContentTransition:
|
||||
DeltaMessage(reasoning=" end", content="hello"), # mixed delta
|
||||
DeltaMessage(content=" world"),
|
||||
]
|
||||
_mock_parser_with_reasoning(serving, delta_sequence)
|
||||
response_parser = _mock_parser_with_reasoning(serving, delta_sequence)
|
||||
# Create contexts for each streaming chunk
|
||||
contexts = [
|
||||
_make_simple_context_with_output("chunk1", [10]),
|
||||
_make_simple_context_with_output("chunk2", [20]),
|
||||
_make_simple_context_with_output("chunk3", [30]),
|
||||
_make_simple_context_with_output("chunk1", [10], response_parser),
|
||||
_make_simple_context_with_output("chunk2", [20], response_parser),
|
||||
_make_simple_context_with_output("chunk3", [30], response_parser),
|
||||
]
|
||||
|
||||
async def result_generator():
|
||||
@@ -767,7 +774,7 @@ class TestStreamingReasoningToContentTransition:
|
||||
request=request,
|
||||
sampling_params=sampling_params,
|
||||
result_generator=result_generator(),
|
||||
context=SimpleContext(),
|
||||
context=SimpleContext(response_parser=response_parser),
|
||||
model_name="test-model",
|
||||
tokenizer=MagicMock(),
|
||||
request_metadata=metadata,
|
||||
@@ -813,11 +820,11 @@ class TestStreamingReasoningToContentTransition:
|
||||
DeltaMessage(reasoning="thinking"),
|
||||
DeltaMessage(content="answer"),
|
||||
]
|
||||
_mock_parser_with_reasoning(serving, delta_sequence)
|
||||
response_parser = _mock_parser_with_reasoning(serving, delta_sequence)
|
||||
|
||||
contexts = [
|
||||
_make_simple_context_with_output("chunk1", [10]),
|
||||
_make_simple_context_with_output("chunk2", [20]),
|
||||
_make_simple_context_with_output("chunk1", [10], response_parser),
|
||||
_make_simple_context_with_output("chunk2", [20], response_parser),
|
||||
]
|
||||
|
||||
async def result_generator():
|
||||
@@ -834,7 +841,7 @@ class TestStreamingReasoningToContentTransition:
|
||||
request=request,
|
||||
sampling_params=sampling_params,
|
||||
result_generator=result_generator(),
|
||||
context=SimpleContext(),
|
||||
context=SimpleContext(response_parser=response_parser),
|
||||
model_name="test-model",
|
||||
tokenizer=MagicMock(),
|
||||
request_metadata=metadata,
|
||||
@@ -875,11 +882,11 @@ class TestStreamingReasoningToContentTransition:
|
||||
DeltaMessage(reasoning="step 1"),
|
||||
DeltaMessage(reasoning=" step 2"),
|
||||
]
|
||||
_mock_parser_with_reasoning(serving, delta_sequence)
|
||||
response_parser = _mock_parser_with_reasoning(serving, delta_sequence)
|
||||
|
||||
contexts = [
|
||||
_make_simple_context_with_output("chunk1", [10]),
|
||||
_make_simple_context_with_output("chunk2", [20]),
|
||||
_make_simple_context_with_output("chunk1", [10], response_parser),
|
||||
_make_simple_context_with_output("chunk2", [20], response_parser),
|
||||
]
|
||||
|
||||
async def result_generator():
|
||||
@@ -896,7 +903,7 @@ class TestStreamingReasoningToContentTransition:
|
||||
request=request,
|
||||
sampling_params=sampling_params,
|
||||
result_generator=result_generator(),
|
||||
context=SimpleContext(),
|
||||
context=SimpleContext(response_parser=response_parser),
|
||||
model_name="test-model",
|
||||
tokenizer=MagicMock(),
|
||||
request_metadata=metadata,
|
||||
@@ -936,10 +943,10 @@ class TestAutoToolStreaming:
|
||||
@staticmethod
|
||||
async def _collect_events(delta_sequence: list[DeltaMessage]):
|
||||
serving = _make_serving_instance_with_reasoning()
|
||||
_mock_parser_with_reasoning(serving, delta_sequence)
|
||||
response_parser = _mock_parser_with_reasoning(serving, delta_sequence)
|
||||
|
||||
contexts = [
|
||||
_make_simple_context_with_output("chunk", [i])
|
||||
_make_simple_context_with_output("chunk", [i], response_parser)
|
||||
for i in range(len(delta_sequence))
|
||||
]
|
||||
|
||||
@@ -974,7 +981,7 @@ class TestAutoToolStreaming:
|
||||
request=request,
|
||||
sampling_params=sampling_params,
|
||||
result_generator=result_generator(),
|
||||
context=SimpleContext(),
|
||||
context=SimpleContext(response_parser=response_parser),
|
||||
model_name="test-model",
|
||||
tokenizer=MagicMock(),
|
||||
request_metadata=metadata,
|
||||
|
||||
@@ -107,6 +107,8 @@ class TurnMetrics:
|
||||
|
||||
|
||||
class ConversationContext(ABC):
|
||||
response_parser: Parser | None = None
|
||||
|
||||
@abstractmethod
|
||||
def append_output(self, output: RequestOutput) -> None:
|
||||
pass
|
||||
@@ -167,8 +169,25 @@ def _create_json_parse_error_messages(
|
||||
class SimpleContext(ConversationContext):
|
||||
"""This is a context that cannot handle MCP tool calls"""
|
||||
|
||||
def __init__(self):
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
response_parser: Parser | None = None,
|
||||
parser_cls: type[Parser] | None = None,
|
||||
tokenizer: TokenizerLike | None = None,
|
||||
request: ResponsesRequest | None = None,
|
||||
chat_template_kwargs: dict[str, Any] | None = None,
|
||||
):
|
||||
self.last_output = None
|
||||
self.response_parser = response_parser or (
|
||||
parser_cls(
|
||||
tokenizer,
|
||||
request.tools,
|
||||
chat_template_kwargs=chat_template_kwargs,
|
||||
)
|
||||
if parser_cls is not None and tokenizer is not None and request is not None
|
||||
else None
|
||||
)
|
||||
|
||||
# Accumulated final output for streaming mode
|
||||
self._accumulated_text: str = ""
|
||||
@@ -181,7 +200,7 @@ class SimpleContext(ConversationContext):
|
||||
# todo num_reasoning_tokens is not implemented yet.
|
||||
self.num_reasoning_tokens = 0
|
||||
# not implemented yet for SimpleContext
|
||||
self.all_turn_metrics = []
|
||||
self.all_turn_metrics: list[TurnMetrics] = []
|
||||
|
||||
self.input_messages: list[ResponseRawMessageAndToken] = []
|
||||
self.kv_transfer_params: dict[str, Any] | None = None
|
||||
@@ -280,6 +299,7 @@ class ParsableContext(ConversationContext):
|
||||
available_tools: list[str] | None,
|
||||
chat_template: str | None,
|
||||
chat_template_content_format: ChatTemplateContentFormatOption,
|
||||
response_parser: Parser | None = None,
|
||||
enable_auto_tools: bool = False,
|
||||
tool_call_id_type: str = "random",
|
||||
):
|
||||
@@ -296,13 +316,13 @@ class ParsableContext(ConversationContext):
|
||||
self.enable_auto_tools = enable_auto_tools
|
||||
self.tool_call_id_type = tool_call_id_type
|
||||
|
||||
self.parser_instance: Parser | None = None
|
||||
if parser_cls is not None:
|
||||
self.response_parser = response_parser
|
||||
if self.response_parser is None and parser_cls is not None:
|
||||
chat_template_kwargs = request.build_chat_params(
|
||||
default_template=chat_template,
|
||||
default_template_content_format=chat_template_content_format,
|
||||
).chat_template_kwargs
|
||||
self.parser_instance = parser_cls(
|
||||
self.response_parser = parser_cls(
|
||||
tokenizer,
|
||||
tools=request.tools,
|
||||
chat_template_kwargs=chat_template_kwargs,
|
||||
@@ -334,8 +354,8 @@ class ParsableContext(ConversationContext):
|
||||
completion = output.outputs[0]
|
||||
self.finish_reason = completion.finish_reason
|
||||
|
||||
if self.parser_instance is not None:
|
||||
reasoning, content, tool_calls = self.parser_instance.parse(
|
||||
if self.response_parser is not None:
|
||||
reasoning, content, tool_calls = self.response_parser.parse(
|
||||
completion.text,
|
||||
self.request,
|
||||
enable_auto_tools=self.enable_auto_tools,
|
||||
@@ -591,8 +611,10 @@ class HarmonyContext(ConversationContext):
|
||||
messages: list,
|
||||
available_tools: list[str],
|
||||
function_tool_names: frozenset[str] | None = None,
|
||||
response_parser: Parser | None = None,
|
||||
):
|
||||
self._messages = messages
|
||||
self.response_parser = response_parser
|
||||
self.finish_reason: str | None = None
|
||||
self.available_tools = available_tools
|
||||
self.function_tool_names = function_tool_names
|
||||
|
||||
@@ -260,6 +260,20 @@ class OpenAIServingResponses(OpenAIServing):
|
||||
.chat_template_kwargs
|
||||
)
|
||||
|
||||
def _make_response_parser(
|
||||
self,
|
||||
request: ResponsesRequest,
|
||||
tokenizer: TokenizerLike,
|
||||
chat_template_kwargs: dict[str, Any],
|
||||
) -> Parser | None:
|
||||
if self.parser is None:
|
||||
return None
|
||||
return self.parser(
|
||||
tokenizer,
|
||||
request.tools,
|
||||
chat_template_kwargs=chat_template_kwargs,
|
||||
)
|
||||
|
||||
def _validate_generator_input(
|
||||
self,
|
||||
engine_input: EngineInput,
|
||||
@@ -443,16 +457,27 @@ class OpenAIServingResponses(OpenAIServing):
|
||||
else await self._get_trace_headers(raw_request.headers)
|
||||
)
|
||||
|
||||
chat_template_kwargs = self._effective_chat_template_kwargs(request)
|
||||
response_parser = self._make_response_parser(
|
||||
request, tokenizer, chat_template_kwargs
|
||||
)
|
||||
|
||||
context: ConversationContext
|
||||
function_tool_names = extract_function_tool_names(request.tools)
|
||||
if self.use_harmony:
|
||||
if request.stream:
|
||||
context = StreamingHarmonyContext(
|
||||
messages, available_tools, function_tool_names
|
||||
messages,
|
||||
available_tools,
|
||||
function_tool_names,
|
||||
response_parser=response_parser,
|
||||
)
|
||||
else:
|
||||
context = HarmonyContext(
|
||||
messages, available_tools, function_tool_names
|
||||
messages,
|
||||
available_tools,
|
||||
function_tool_names,
|
||||
response_parser=response_parser,
|
||||
)
|
||||
else:
|
||||
if envs.VLLM_USE_EXPERIMENTAL_PARSER_CONTEXT:
|
||||
@@ -463,6 +488,7 @@ class OpenAIServingResponses(OpenAIServing):
|
||||
tokenizer=tokenizer,
|
||||
parser_cls=self.parser,
|
||||
request=request,
|
||||
response_parser=response_parser,
|
||||
available_tools=available_tools,
|
||||
chat_template=self.chat_template,
|
||||
chat_template_content_format=self.chat_template_content_format,
|
||||
@@ -470,17 +496,17 @@ class OpenAIServingResponses(OpenAIServing):
|
||||
tool_call_id_type=self.tool_call_id_type,
|
||||
)
|
||||
else:
|
||||
context = SimpleContext()
|
||||
context = SimpleContext(
|
||||
response_parser=response_parser,
|
||||
)
|
||||
|
||||
if self.parser and self.parser.reasoning_parser_cls is not None:
|
||||
chat_template_kwargs = self._effective_chat_template_kwargs(request)
|
||||
if (
|
||||
context.response_parser is not None
|
||||
and context.response_parser.reasoning_parser is not None
|
||||
):
|
||||
reasoning_parser_kwargs = {
|
||||
"chat_template_kwargs": chat_template_kwargs,
|
||||
}
|
||||
reasoning_parser = self.parser.reasoning_parser_cls(
|
||||
tokenizer,
|
||||
chat_template_kwargs=chat_template_kwargs,
|
||||
)
|
||||
if (
|
||||
isinstance(
|
||||
struct_out := sampling_params.structured_outputs,
|
||||
@@ -490,8 +516,10 @@ class OpenAIServingResponses(OpenAIServing):
|
||||
):
|
||||
sampling_params.structured_outputs = replace(
|
||||
struct_out,
|
||||
structural_tag=reasoning_parser.prepare_structured_tag(
|
||||
struct_out.structural_tag, self.tool_server
|
||||
structural_tag=(
|
||||
context.response_parser.reasoning_parser.prepare_structured_tag(
|
||||
struct_out.structural_tag, self.tool_server
|
||||
)
|
||||
),
|
||||
)
|
||||
generator = self._generate_with_builtin_tools(
|
||||
@@ -833,7 +861,12 @@ class OpenAIServingResponses(OpenAIServing):
|
||||
if final_output.finish_reason == "length":
|
||||
status = "incomplete"
|
||||
|
||||
output = self._make_response_output_items(request, final_output, tokenizer)
|
||||
output = self._make_response_output_items(
|
||||
request,
|
||||
final_output,
|
||||
tokenizer,
|
||||
parser=context.response_parser,
|
||||
)
|
||||
|
||||
if request.enable_response_messages:
|
||||
input_messages = context.input_messages
|
||||
@@ -854,16 +887,16 @@ class OpenAIServingResponses(OpenAIServing):
|
||||
# accumulated output token IDs using the parser if not already set.
|
||||
if (
|
||||
num_reasoning_tokens == 0
|
||||
and self.parser is not None
|
||||
and self.parser.reasoning_parser_cls is not None
|
||||
and isinstance(context, (SimpleContext, ParsableContext))
|
||||
and context.response_parser is not None
|
||||
and context.response_parser.reasoning_parser is not None
|
||||
):
|
||||
reasoning_parser = self.parser.reasoning_parser_cls(
|
||||
tokenizer,
|
||||
chat_template_kwargs=self._effective_chat_template_kwargs(request),
|
||||
)
|
||||
accumulated = getattr(context, "_accumulated_token_ids", []) or []
|
||||
num_reasoning_tokens = reasoning_parser.count_reasoning_tokens(accumulated)
|
||||
num_reasoning_tokens = (
|
||||
context.response_parser.reasoning_parser.count_reasoning_tokens(
|
||||
accumulated
|
||||
)
|
||||
)
|
||||
|
||||
usage = ResponseUsage(
|
||||
input_tokens=num_prompt_tokens,
|
||||
@@ -1005,6 +1038,7 @@ class OpenAIServingResponses(OpenAIServing):
|
||||
request: ResponsesRequest,
|
||||
final_output: CompletionOutput,
|
||||
tokenizer: TokenizerLike,
|
||||
parser: Parser | None = None,
|
||||
) -> list[ResponseOutputItem]:
|
||||
# Log complete response if output logging is enabled
|
||||
if self.enable_log_outputs and self.request_logger:
|
||||
@@ -1028,11 +1062,7 @@ class OpenAIServingResponses(OpenAIServing):
|
||||
)
|
||||
|
||||
# Use parser to extract reasoning, content, and tool calls
|
||||
if self.parser:
|
||||
chat_template_kwargs = self._effective_chat_template_kwargs(request)
|
||||
parser = self.parser(
|
||||
tokenizer, request.tools, chat_template_kwargs=chat_template_kwargs
|
||||
)
|
||||
if parser:
|
||||
reasoning, content, tool_calls = parser.parse(
|
||||
final_output.text,
|
||||
request,
|
||||
@@ -1350,15 +1380,6 @@ class OpenAIServingResponses(OpenAIServing):
|
||||
],
|
||||
) -> AsyncGenerator[StreamingResponsesResponse, None]:
|
||||
processor = SimpleStreamingEventProcessor()
|
||||
parser = (
|
||||
self.parser(
|
||||
tokenizer,
|
||||
request.tools,
|
||||
chat_template_kwargs=self._effective_chat_template_kwargs(request),
|
||||
)
|
||||
if self.parser
|
||||
else None
|
||||
)
|
||||
|
||||
def _get_logprobs(
|
||||
output: CompletionOutput,
|
||||
@@ -1382,8 +1403,8 @@ class OpenAIServingResponses(OpenAIServing):
|
||||
delta_text = output.text
|
||||
delta_token_ids = as_list(output.token_ids)
|
||||
|
||||
if parser:
|
||||
delta_message = parser.parse_delta(
|
||||
if ctx.response_parser:
|
||||
delta_message = ctx.response_parser.parse_delta(
|
||||
delta_text=delta_text,
|
||||
delta_token_ids=delta_token_ids,
|
||||
request=request,
|
||||
|
||||
Reference in New Issue
Block a user