[Bugfix][Frontend] Keep length finish_reason for max_tokens-truncated streaming tool calls (#46303)
Signed-off-by: Ting Sun <suntcrick@gmail.com>
This commit is contained in:
@@ -2239,7 +2239,13 @@ async def test_tool_choice_validation_without_parser():
|
|||||||
|
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
@pytest.mark.asyncio
|
||||||
async def test_streaming_n_gt1_independent_tool_parsers():
|
@pytest.mark.parametrize(
|
||||||
|
("engine_finish_reason", "expected_finish_reason"),
|
||||||
|
[("stop", "tool_calls"), ("length", "length")],
|
||||||
|
)
|
||||||
|
async def test_streaming_n_gt1_independent_tool_parsers(
|
||||||
|
engine_finish_reason: str, expected_finish_reason: str
|
||||||
|
):
|
||||||
"""n>1 streaming must use independent parser instances
|
"""n>1 streaming must use independent parser instances
|
||||||
and token-id histories per choice.
|
and token-id histories per choice.
|
||||||
"""
|
"""
|
||||||
@@ -2344,7 +2350,7 @@ async def test_streaming_n_gt1_independent_tool_parsers():
|
|||||||
token_ids=[],
|
token_ids=[],
|
||||||
cumulative_logprob=0.0,
|
cumulative_logprob=0.0,
|
||||||
logprobs=None,
|
logprobs=None,
|
||||||
finish_reason="stop",
|
finish_reason=engine_finish_reason,
|
||||||
)
|
)
|
||||||
for choice_idx in range(num_choices)
|
for choice_idx in range(num_choices)
|
||||||
],
|
],
|
||||||
@@ -2408,8 +2414,8 @@ async def test_streaming_n_gt1_independent_tool_parsers():
|
|||||||
assert len(reasons) == 1, (
|
assert len(reasons) == 1, (
|
||||||
f"Choice {choice_idx}: expected exactly 1 finish_reason, got {reasons}"
|
f"Choice {choice_idx}: expected exactly 1 finish_reason, got {reasons}"
|
||||||
)
|
)
|
||||||
assert reasons[0] == "tool_calls", (
|
assert reasons[0] == expected_finish_reason, (
|
||||||
f"Choice {choice_idx}: expected finish_reason='tool_calls', "
|
f"Choice {choice_idx}: expected finish_reason={expected_finish_reason!r}, "
|
||||||
f"got '{reasons[0]}'"
|
f"got '{reasons[0]}'"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|||||||
@@ -757,7 +757,11 @@ class OpenAIServingChat(GenerateBaseServing):
|
|||||||
# finish_reason is:
|
# finish_reason is:
|
||||||
# "tool_calls" for "auto" or "required" tool calls,
|
# "tool_calls" for "auto" or "required" tool calls,
|
||||||
# and "stop" for named tool calls.
|
# and "stop" for named tool calls.
|
||||||
if tools_streamed[i] and not tool_choice_function_name:
|
if (
|
||||||
|
tools_streamed[i]
|
||||||
|
and not tool_choice_function_name
|
||||||
|
and output.finish_reason == "stop"
|
||||||
|
):
|
||||||
finish_reason_ = "tool_calls"
|
finish_reason_ = "tool_calls"
|
||||||
else:
|
else:
|
||||||
finish_reason_ = (
|
finish_reason_ = (
|
||||||
|
|||||||
Reference in New Issue
Block a user