diff --git a/tests/entrypoints/openai/responses/test_serving_responses.py b/tests/entrypoints/openai/responses/test_serving_responses.py index 25b00ff192..b19abdff68 100644 --- a/tests/entrypoints/openai/responses/test_serving_responses.py +++ b/tests/entrypoints/openai/responses/test_serving_responses.py @@ -386,8 +386,15 @@ async def test_reasoning_tokens_counted_for_text_reasoning_model(monkeypatch): reasoning_parser="qwen3", ) + request = ResponsesRequest(input="hi", tools=[], stream=False) + response_parser = serving._make_response_parser( + request, + tokenizer, + serving._effective_chat_template_kwargs(request), + ) + # Build a SimpleContext with thinking tokens in the output. - context = SimpleContext() + context = SimpleContext(response_parser=response_parser) token_ids = [1, 10, 2, 20] # 10 20 -> reasoning token count = 1 completion = CompletionOutput( index=0, @@ -412,7 +419,6 @@ async def test_reasoning_tokens_counted_for_text_reasoning_model(monkeypatch): async def dummy_result_generator(): yield None - request = ResponsesRequest(input="hi", tools=[], stream=False) sampling_params = SamplingParams(max_tokens=16) metadata = RequestResponseMetadata(request_id="req") @@ -636,9 +642,9 @@ class TestHarmonyPreambleStreaming: assert "response.output_text.done" not in type_names -def _make_simple_context_with_output(text, token_ids): +def _make_simple_context_with_output(text, token_ids, response_parser=None): """Create a SimpleContext with a RequestOutput containing the given text.""" - ctx = SimpleContext() + ctx = SimpleContext(response_parser=response_parser) completion = CompletionOutput( index=0, text=text, @@ -719,6 +725,7 @@ def _mock_parser_with_reasoning(serving, delta_sequence: list[DeltaMessage]): mock_parser_instance.parse_delta = mock_parse_delta mock_parser_instance.is_reasoning_end = MagicMock(return_value=False) serving.parser = MagicMock(return_value=mock_parser_instance) + return mock_parser_instance class TestStreamingReasoningToContentTransition: @@ -745,12 +752,12 @@ class TestStreamingReasoningToContentTransition: DeltaMessage(reasoning=" end", content="hello"), # mixed delta DeltaMessage(content=" world"), ] - _mock_parser_with_reasoning(serving, delta_sequence) + response_parser = _mock_parser_with_reasoning(serving, delta_sequence) # Create contexts for each streaming chunk contexts = [ - _make_simple_context_with_output("chunk1", [10]), - _make_simple_context_with_output("chunk2", [20]), - _make_simple_context_with_output("chunk3", [30]), + _make_simple_context_with_output("chunk1", [10], response_parser), + _make_simple_context_with_output("chunk2", [20], response_parser), + _make_simple_context_with_output("chunk3", [30], response_parser), ] async def result_generator(): @@ -767,7 +774,7 @@ class TestStreamingReasoningToContentTransition: request=request, sampling_params=sampling_params, result_generator=result_generator(), - context=SimpleContext(), + context=SimpleContext(response_parser=response_parser), model_name="test-model", tokenizer=MagicMock(), request_metadata=metadata, @@ -813,11 +820,11 @@ class TestStreamingReasoningToContentTransition: DeltaMessage(reasoning="thinking"), DeltaMessage(content="answer"), ] - _mock_parser_with_reasoning(serving, delta_sequence) + response_parser = _mock_parser_with_reasoning(serving, delta_sequence) contexts = [ - _make_simple_context_with_output("chunk1", [10]), - _make_simple_context_with_output("chunk2", [20]), + _make_simple_context_with_output("chunk1", [10], response_parser), + _make_simple_context_with_output("chunk2", [20], response_parser), ] async def result_generator(): @@ -834,7 +841,7 @@ class TestStreamingReasoningToContentTransition: request=request, sampling_params=sampling_params, result_generator=result_generator(), - context=SimpleContext(), + context=SimpleContext(response_parser=response_parser), model_name="test-model", tokenizer=MagicMock(), request_metadata=metadata, @@ -875,11 +882,11 @@ class TestStreamingReasoningToContentTransition: DeltaMessage(reasoning="step 1"), DeltaMessage(reasoning=" step 2"), ] - _mock_parser_with_reasoning(serving, delta_sequence) + response_parser = _mock_parser_with_reasoning(serving, delta_sequence) contexts = [ - _make_simple_context_with_output("chunk1", [10]), - _make_simple_context_with_output("chunk2", [20]), + _make_simple_context_with_output("chunk1", [10], response_parser), + _make_simple_context_with_output("chunk2", [20], response_parser), ] async def result_generator(): @@ -896,7 +903,7 @@ class TestStreamingReasoningToContentTransition: request=request, sampling_params=sampling_params, result_generator=result_generator(), - context=SimpleContext(), + context=SimpleContext(response_parser=response_parser), model_name="test-model", tokenizer=MagicMock(), request_metadata=metadata, @@ -936,10 +943,10 @@ class TestAutoToolStreaming: @staticmethod async def _collect_events(delta_sequence: list[DeltaMessage]): serving = _make_serving_instance_with_reasoning() - _mock_parser_with_reasoning(serving, delta_sequence) + response_parser = _mock_parser_with_reasoning(serving, delta_sequence) contexts = [ - _make_simple_context_with_output("chunk", [i]) + _make_simple_context_with_output("chunk", [i], response_parser) for i in range(len(delta_sequence)) ] @@ -974,7 +981,7 @@ class TestAutoToolStreaming: request=request, sampling_params=sampling_params, result_generator=result_generator(), - context=SimpleContext(), + context=SimpleContext(response_parser=response_parser), model_name="test-model", tokenizer=MagicMock(), request_metadata=metadata, diff --git a/vllm/entrypoints/openai/responses/context.py b/vllm/entrypoints/openai/responses/context.py index 9679b732a7..6b987f449d 100644 --- a/vllm/entrypoints/openai/responses/context.py +++ b/vllm/entrypoints/openai/responses/context.py @@ -107,6 +107,8 @@ class TurnMetrics: class ConversationContext(ABC): + response_parser: Parser | None = None + @abstractmethod def append_output(self, output: RequestOutput) -> None: pass @@ -167,8 +169,25 @@ def _create_json_parse_error_messages( class SimpleContext(ConversationContext): """This is a context that cannot handle MCP tool calls""" - def __init__(self): + def __init__( + self, + *, + response_parser: Parser | None = None, + parser_cls: type[Parser] | None = None, + tokenizer: TokenizerLike | None = None, + request: ResponsesRequest | None = None, + chat_template_kwargs: dict[str, Any] | None = None, + ): self.last_output = None + self.response_parser = response_parser or ( + parser_cls( + tokenizer, + request.tools, + chat_template_kwargs=chat_template_kwargs, + ) + if parser_cls is not None and tokenizer is not None and request is not None + else None + ) # Accumulated final output for streaming mode self._accumulated_text: str = "" @@ -181,7 +200,7 @@ class SimpleContext(ConversationContext): # todo num_reasoning_tokens is not implemented yet. self.num_reasoning_tokens = 0 # not implemented yet for SimpleContext - self.all_turn_metrics = [] + self.all_turn_metrics: list[TurnMetrics] = [] self.input_messages: list[ResponseRawMessageAndToken] = [] self.kv_transfer_params: dict[str, Any] | None = None @@ -280,6 +299,7 @@ class ParsableContext(ConversationContext): available_tools: list[str] | None, chat_template: str | None, chat_template_content_format: ChatTemplateContentFormatOption, + response_parser: Parser | None = None, enable_auto_tools: bool = False, tool_call_id_type: str = "random", ): @@ -296,13 +316,13 @@ class ParsableContext(ConversationContext): self.enable_auto_tools = enable_auto_tools self.tool_call_id_type = tool_call_id_type - self.parser_instance: Parser | None = None - if parser_cls is not None: + self.response_parser = response_parser + if self.response_parser is None and parser_cls is not None: chat_template_kwargs = request.build_chat_params( default_template=chat_template, default_template_content_format=chat_template_content_format, ).chat_template_kwargs - self.parser_instance = parser_cls( + self.response_parser = parser_cls( tokenizer, tools=request.tools, chat_template_kwargs=chat_template_kwargs, @@ -334,8 +354,8 @@ class ParsableContext(ConversationContext): completion = output.outputs[0] self.finish_reason = completion.finish_reason - if self.parser_instance is not None: - reasoning, content, tool_calls = self.parser_instance.parse( + if self.response_parser is not None: + reasoning, content, tool_calls = self.response_parser.parse( completion.text, self.request, enable_auto_tools=self.enable_auto_tools, @@ -591,8 +611,10 @@ class HarmonyContext(ConversationContext): messages: list, available_tools: list[str], function_tool_names: frozenset[str] | None = None, + response_parser: Parser | None = None, ): self._messages = messages + self.response_parser = response_parser self.finish_reason: str | None = None self.available_tools = available_tools self.function_tool_names = function_tool_names diff --git a/vllm/entrypoints/openai/responses/serving.py b/vllm/entrypoints/openai/responses/serving.py index 9d95ccc0cb..048efda9f6 100644 --- a/vllm/entrypoints/openai/responses/serving.py +++ b/vllm/entrypoints/openai/responses/serving.py @@ -260,6 +260,20 @@ class OpenAIServingResponses(OpenAIServing): .chat_template_kwargs ) + def _make_response_parser( + self, + request: ResponsesRequest, + tokenizer: TokenizerLike, + chat_template_kwargs: dict[str, Any], + ) -> Parser | None: + if self.parser is None: + return None + return self.parser( + tokenizer, + request.tools, + chat_template_kwargs=chat_template_kwargs, + ) + def _validate_generator_input( self, engine_input: EngineInput, @@ -443,16 +457,27 @@ class OpenAIServingResponses(OpenAIServing): else await self._get_trace_headers(raw_request.headers) ) + chat_template_kwargs = self._effective_chat_template_kwargs(request) + response_parser = self._make_response_parser( + request, tokenizer, chat_template_kwargs + ) + context: ConversationContext function_tool_names = extract_function_tool_names(request.tools) if self.use_harmony: if request.stream: context = StreamingHarmonyContext( - messages, available_tools, function_tool_names + messages, + available_tools, + function_tool_names, + response_parser=response_parser, ) else: context = HarmonyContext( - messages, available_tools, function_tool_names + messages, + available_tools, + function_tool_names, + response_parser=response_parser, ) else: if envs.VLLM_USE_EXPERIMENTAL_PARSER_CONTEXT: @@ -463,6 +488,7 @@ class OpenAIServingResponses(OpenAIServing): tokenizer=tokenizer, parser_cls=self.parser, request=request, + response_parser=response_parser, available_tools=available_tools, chat_template=self.chat_template, chat_template_content_format=self.chat_template_content_format, @@ -470,17 +496,17 @@ class OpenAIServingResponses(OpenAIServing): tool_call_id_type=self.tool_call_id_type, ) else: - context = SimpleContext() + context = SimpleContext( + response_parser=response_parser, + ) - if self.parser and self.parser.reasoning_parser_cls is not None: - chat_template_kwargs = self._effective_chat_template_kwargs(request) + if ( + context.response_parser is not None + and context.response_parser.reasoning_parser is not None + ): reasoning_parser_kwargs = { "chat_template_kwargs": chat_template_kwargs, } - reasoning_parser = self.parser.reasoning_parser_cls( - tokenizer, - chat_template_kwargs=chat_template_kwargs, - ) if ( isinstance( struct_out := sampling_params.structured_outputs, @@ -490,8 +516,10 @@ class OpenAIServingResponses(OpenAIServing): ): sampling_params.structured_outputs = replace( struct_out, - structural_tag=reasoning_parser.prepare_structured_tag( - struct_out.structural_tag, self.tool_server + structural_tag=( + context.response_parser.reasoning_parser.prepare_structured_tag( + struct_out.structural_tag, self.tool_server + ) ), ) generator = self._generate_with_builtin_tools( @@ -833,7 +861,12 @@ class OpenAIServingResponses(OpenAIServing): if final_output.finish_reason == "length": status = "incomplete" - output = self._make_response_output_items(request, final_output, tokenizer) + output = self._make_response_output_items( + request, + final_output, + tokenizer, + parser=context.response_parser, + ) if request.enable_response_messages: input_messages = context.input_messages @@ -854,16 +887,16 @@ class OpenAIServingResponses(OpenAIServing): # accumulated output token IDs using the parser if not already set. if ( num_reasoning_tokens == 0 - and self.parser is not None - and self.parser.reasoning_parser_cls is not None and isinstance(context, (SimpleContext, ParsableContext)) + and context.response_parser is not None + and context.response_parser.reasoning_parser is not None ): - reasoning_parser = self.parser.reasoning_parser_cls( - tokenizer, - chat_template_kwargs=self._effective_chat_template_kwargs(request), - ) accumulated = getattr(context, "_accumulated_token_ids", []) or [] - num_reasoning_tokens = reasoning_parser.count_reasoning_tokens(accumulated) + num_reasoning_tokens = ( + context.response_parser.reasoning_parser.count_reasoning_tokens( + accumulated + ) + ) usage = ResponseUsage( input_tokens=num_prompt_tokens, @@ -1005,6 +1038,7 @@ class OpenAIServingResponses(OpenAIServing): request: ResponsesRequest, final_output: CompletionOutput, tokenizer: TokenizerLike, + parser: Parser | None = None, ) -> list[ResponseOutputItem]: # Log complete response if output logging is enabled if self.enable_log_outputs and self.request_logger: @@ -1028,11 +1062,7 @@ class OpenAIServingResponses(OpenAIServing): ) # Use parser to extract reasoning, content, and tool calls - if self.parser: - chat_template_kwargs = self._effective_chat_template_kwargs(request) - parser = self.parser( - tokenizer, request.tools, chat_template_kwargs=chat_template_kwargs - ) + if parser: reasoning, content, tool_calls = parser.parse( final_output.text, request, @@ -1350,15 +1380,6 @@ class OpenAIServingResponses(OpenAIServing): ], ) -> AsyncGenerator[StreamingResponsesResponse, None]: processor = SimpleStreamingEventProcessor() - parser = ( - self.parser( - tokenizer, - request.tools, - chat_template_kwargs=self._effective_chat_template_kwargs(request), - ) - if self.parser - else None - ) def _get_logprobs( output: CompletionOutput, @@ -1382,8 +1403,8 @@ class OpenAIServingResponses(OpenAIServing): delta_text = output.text delta_token_ids = as_list(output.token_ids) - if parser: - delta_message = parser.parse_delta( + if ctx.response_parser: + delta_message = ctx.response_parser.parse_delta( delta_text=delta_text, delta_token_ids=delta_token_ids, request=request,