[Bugfix][Frontend] Respect max_output_tokens in the Harmony tool-call loop (#58551)
Signed-off-by: errmakov <ide404@gmail.com> Co-authored-by: Du Bin <8174807+dubin555@users.noreply.github.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com> Co-authored-by: Flora Feng <4florafeng@gmail.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
Du Bin
Claude Opus 5.5
Flora Feng
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
83db8c8390
commit
7871963fcc
@@ -748,6 +748,7 @@ class TestInitializeToolSessions:
|
|||||||
class ToolCallingHarmonyContext(HarmonyContext):
|
class ToolCallingHarmonyContext(HarmonyContext):
|
||||||
def __init__(self):
|
def __init__(self):
|
||||||
self._messages = []
|
self._messages = []
|
||||||
|
self.request = None
|
||||||
self._needs_tool_call = True
|
self._needs_tool_call = True
|
||||||
|
|
||||||
def append_output(self, output) -> None:
|
def append_output(self, output) -> None:
|
||||||
@@ -804,6 +805,53 @@ class TestInitializeToolSessions:
|
|||||||
assert followup_engine_input.get("cache_salt") == "request-salt"
|
assert followup_engine_input.get("cache_salt") == "request-salt"
|
||||||
assert followup_engine_input["prompt_token_ids"] == [3, 4]
|
assert followup_engine_input["prompt_token_ids"] == [3, 4]
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_harmony_tool_followup_respects_max_output_tokens(
|
||||||
|
self, serving_responses_instance
|
||||||
|
):
|
||||||
|
class ToolCallingHarmonyContext(HarmonyContext):
|
||||||
|
def __init__(self, request):
|
||||||
|
self._messages = []
|
||||||
|
self.request = request
|
||||||
|
self._needs_tool_call = True
|
||||||
|
|
||||||
|
def append_output(self, output) -> None:
|
||||||
|
pass
|
||||||
|
|
||||||
|
def need_builtin_tool_call(self) -> bool:
|
||||||
|
return self._needs_tool_call
|
||||||
|
|
||||||
|
async def call_tool(self):
|
||||||
|
self._needs_tool_call = False
|
||||||
|
return []
|
||||||
|
|
||||||
|
def append_tool_output(self, output) -> None:
|
||||||
|
pass
|
||||||
|
|
||||||
|
max_tokens_per_turn = []
|
||||||
|
|
||||||
|
async def generate_output(engine_input, sampling_params, *args, **kwargs):
|
||||||
|
max_tokens_per_turn.append(sampling_params.max_tokens)
|
||||||
|
yield MagicMock()
|
||||||
|
|
||||||
|
serving_responses_instance.engine_client.generate.side_effect = generate_output
|
||||||
|
serving_responses_instance.online_renderer.render_responses_harmony_messages = (
|
||||||
|
MagicMock(return_value=tokens_input([1, 2, 3]))
|
||||||
|
)
|
||||||
|
context = ToolCallingHarmonyContext(
|
||||||
|
ResponsesRequest(input="test", max_output_tokens=10)
|
||||||
|
)
|
||||||
|
|
||||||
|
async for _ in serving_responses_instance._generate_with_builtin_tools(
|
||||||
|
request_id="req",
|
||||||
|
engine_input=tokens_input([1]),
|
||||||
|
sampling_params=SamplingParams(max_tokens=10),
|
||||||
|
context=context,
|
||||||
|
):
|
||||||
|
pass
|
||||||
|
|
||||||
|
assert max_tokens_per_turn == [10, 10]
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
@pytest.mark.asyncio
|
||||||
async def test_initialize_tool_sessions(
|
async def test_initialize_tool_sessions(
|
||||||
self, serving_responses_instance, mock_context, mock_exit_stack
|
self, serving_responses_instance, mock_context, mock_exit_stack
|
||||||
|
|||||||
@@ -615,12 +615,14 @@ class HarmonyContext(ConversationContext):
|
|||||||
available_tools: list[str],
|
available_tools: list[str],
|
||||||
function_tool_names: frozenset[str],
|
function_tool_names: frozenset[str],
|
||||||
response_parser: Parser | None = None,
|
response_parser: Parser | None = None,
|
||||||
|
request: ResponsesRequest | None = None,
|
||||||
):
|
):
|
||||||
from vllm.parser.harmony import HarmonyParser, Segment
|
from vllm.parser.harmony import HarmonyParser, Segment
|
||||||
|
|
||||||
assert isinstance(response_parser, HarmonyParser)
|
assert isinstance(response_parser, HarmonyParser)
|
||||||
|
|
||||||
self._messages = messages
|
self._messages = messages
|
||||||
|
self.request = request
|
||||||
self.response_parser: HarmonyParser = response_parser
|
self.response_parser: HarmonyParser = response_parser
|
||||||
self.finish_reason: str | None = None
|
self.finish_reason: str | None = None
|
||||||
self.available_tools = available_tools
|
self.available_tools = available_tools
|
||||||
|
|||||||
@@ -443,6 +443,7 @@ class OpenAIServingResponses(GenerateBaseServing):
|
|||||||
available_tools,
|
available_tools,
|
||||||
function_tool_names,
|
function_tool_names,
|
||||||
response_parser=response_parser,
|
response_parser=response_parser,
|
||||||
|
request=request,
|
||||||
)
|
)
|
||||||
else:
|
else:
|
||||||
if envs.VLLM_USE_EXPERIMENTAL_PARSER_CONTEXT:
|
if envs.VLLM_USE_EXPERIMENTAL_PARSER_CONTEXT:
|
||||||
@@ -678,8 +679,12 @@ class OpenAIServingResponses(GenerateBaseServing):
|
|||||||
tok_params=tok_params,
|
tok_params=tok_params,
|
||||||
)
|
)
|
||||||
|
|
||||||
sampling_params.max_tokens = max_model_len - self._extract_prompt_len(
|
sampling_params.max_tokens = get_max_tokens(
|
||||||
engine_input
|
max_model_len,
|
||||||
|
context.request.max_output_tokens if context.request else None,
|
||||||
|
self._extract_prompt_len(engine_input),
|
||||||
|
self.default_sampling_params,
|
||||||
|
self.override_max_tokens,
|
||||||
)
|
)
|
||||||
elif isinstance(context, ParsableContext):
|
elif isinstance(context, ParsableContext):
|
||||||
(engine_input,) = await self._render_next_turn(
|
(engine_input,) = await self._render_next_turn(
|
||||||
|
|||||||
Reference in New Issue
Block a user