diff --git a/tests/entrypoints/openai/responses/test_serving_responses.py b/tests/entrypoints/openai/responses/test_serving_responses.py index 1a315902e2..429fcf1d0d 100644 --- a/tests/entrypoints/openai/responses/test_serving_responses.py +++ b/tests/entrypoints/openai/responses/test_serving_responses.py @@ -748,6 +748,7 @@ class TestInitializeToolSessions: class ToolCallingHarmonyContext(HarmonyContext): def __init__(self): self._messages = [] + self.request = None self._needs_tool_call = True def append_output(self, output) -> None: @@ -804,6 +805,53 @@ class TestInitializeToolSessions: assert followup_engine_input.get("cache_salt") == "request-salt" assert followup_engine_input["prompt_token_ids"] == [3, 4] + @pytest.mark.asyncio + async def test_harmony_tool_followup_respects_max_output_tokens( + self, serving_responses_instance + ): + class ToolCallingHarmonyContext(HarmonyContext): + def __init__(self, request): + self._messages = [] + self.request = request + self._needs_tool_call = True + + def append_output(self, output) -> None: + pass + + def need_builtin_tool_call(self) -> bool: + return self._needs_tool_call + + async def call_tool(self): + self._needs_tool_call = False + return [] + + def append_tool_output(self, output) -> None: + pass + + max_tokens_per_turn = [] + + async def generate_output(engine_input, sampling_params, *args, **kwargs): + max_tokens_per_turn.append(sampling_params.max_tokens) + yield MagicMock() + + serving_responses_instance.engine_client.generate.side_effect = generate_output + serving_responses_instance.online_renderer.render_responses_harmony_messages = ( + MagicMock(return_value=tokens_input([1, 2, 3])) + ) + context = ToolCallingHarmonyContext( + ResponsesRequest(input="test", max_output_tokens=10) + ) + + async for _ in serving_responses_instance._generate_with_builtin_tools( + request_id="req", + engine_input=tokens_input([1]), + sampling_params=SamplingParams(max_tokens=10), + context=context, + ): + pass + + assert max_tokens_per_turn == [10, 10] + @pytest.mark.asyncio async def test_initialize_tool_sessions( self, serving_responses_instance, mock_context, mock_exit_stack diff --git a/vllm/entrypoints/openai/responses/context.py b/vllm/entrypoints/openai/responses/context.py index 0db47cf367..59b42f96a3 100644 --- a/vllm/entrypoints/openai/responses/context.py +++ b/vllm/entrypoints/openai/responses/context.py @@ -615,12 +615,14 @@ class HarmonyContext(ConversationContext): available_tools: list[str], function_tool_names: frozenset[str], response_parser: Parser | None = None, + request: ResponsesRequest | None = None, ): from vllm.parser.harmony import HarmonyParser, Segment assert isinstance(response_parser, HarmonyParser) self._messages = messages + self.request = request self.response_parser: HarmonyParser = response_parser self.finish_reason: str | None = None self.available_tools = available_tools diff --git a/vllm/entrypoints/openai/responses/serving.py b/vllm/entrypoints/openai/responses/serving.py index 8108c5f775..9fba86b80c 100644 --- a/vllm/entrypoints/openai/responses/serving.py +++ b/vllm/entrypoints/openai/responses/serving.py @@ -443,6 +443,7 @@ class OpenAIServingResponses(GenerateBaseServing): available_tools, function_tool_names, response_parser=response_parser, + request=request, ) else: if envs.VLLM_USE_EXPERIMENTAL_PARSER_CONTEXT: @@ -678,8 +679,12 @@ class OpenAIServingResponses(GenerateBaseServing): tok_params=tok_params, ) - sampling_params.max_tokens = max_model_len - self._extract_prompt_len( - engine_input + sampling_params.max_tokens = get_max_tokens( + max_model_len, + context.request.max_output_tokens if context.request else None, + self._extract_prompt_len(engine_input), + self.default_sampling_params, + self.override_max_tokens, ) elif isinstance(context, ParsableContext): (engine_input,) = await self._render_next_turn(