[Bugfix][Frontend] Respect max_output_tokens in the Harmony tool-call loop (#58551)
Signed-off-by: errmakov <ide404@gmail.com> Co-authored-by: Du Bin <8174807+dubin555@users.noreply.github.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com> Co-authored-by: Flora Feng <4florafeng@gmail.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
Du Bin
Claude Opus 5.5
Flora Feng
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
83db8c8390
commit
7871963fcc
@@ -748,6 +748,7 @@ class TestInitializeToolSessions:
|
||||
class ToolCallingHarmonyContext(HarmonyContext):
|
||||
def __init__(self):
|
||||
self._messages = []
|
||||
self.request = None
|
||||
self._needs_tool_call = True
|
||||
|
||||
def append_output(self, output) -> None:
|
||||
@@ -804,6 +805,53 @@ class TestInitializeToolSessions:
|
||||
assert followup_engine_input.get("cache_salt") == "request-salt"
|
||||
assert followup_engine_input["prompt_token_ids"] == [3, 4]
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_harmony_tool_followup_respects_max_output_tokens(
|
||||
self, serving_responses_instance
|
||||
):
|
||||
class ToolCallingHarmonyContext(HarmonyContext):
|
||||
def __init__(self, request):
|
||||
self._messages = []
|
||||
self.request = request
|
||||
self._needs_tool_call = True
|
||||
|
||||
def append_output(self, output) -> None:
|
||||
pass
|
||||
|
||||
def need_builtin_tool_call(self) -> bool:
|
||||
return self._needs_tool_call
|
||||
|
||||
async def call_tool(self):
|
||||
self._needs_tool_call = False
|
||||
return []
|
||||
|
||||
def append_tool_output(self, output) -> None:
|
||||
pass
|
||||
|
||||
max_tokens_per_turn = []
|
||||
|
||||
async def generate_output(engine_input, sampling_params, *args, **kwargs):
|
||||
max_tokens_per_turn.append(sampling_params.max_tokens)
|
||||
yield MagicMock()
|
||||
|
||||
serving_responses_instance.engine_client.generate.side_effect = generate_output
|
||||
serving_responses_instance.online_renderer.render_responses_harmony_messages = (
|
||||
MagicMock(return_value=tokens_input([1, 2, 3]))
|
||||
)
|
||||
context = ToolCallingHarmonyContext(
|
||||
ResponsesRequest(input="test", max_output_tokens=10)
|
||||
)
|
||||
|
||||
async for _ in serving_responses_instance._generate_with_builtin_tools(
|
||||
request_id="req",
|
||||
engine_input=tokens_input([1]),
|
||||
sampling_params=SamplingParams(max_tokens=10),
|
||||
context=context,
|
||||
):
|
||||
pass
|
||||
|
||||
assert max_tokens_per_turn == [10, 10]
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_initialize_tool_sessions(
|
||||
self, serving_responses_instance, mock_context, mock_exit_stack
|
||||
|
||||
Reference in New Issue
Block a user