[Bugfix][ToolParser] Stream every tool call when one delta completes several (#60354)
Signed-off-by: Junyi Yao <j.yao@wustl.edu> Signed-off-by: sfeng33 <4florafeng@gmail.com> Co-authored-by: sfeng33 <4florafeng@gmail.com>
This commit is contained in:
@@ -295,45 +295,61 @@ def test_structured_outputs_json_without_parameters(
|
||||
)
|
||||
|
||||
|
||||
def _collect_required_tool_streaming_json(output_json: str, delta_len: int) -> str:
|
||||
def _stream_required_tool_calls(
|
||||
output_json: str,
|
||||
deltas: list[str],
|
||||
tool_call_id_type: str = "random",
|
||||
tool_call_idx: int | None = None,
|
||||
) -> list[dict]:
|
||||
"""Feed ``deltas`` (which concatenate to ``output_json``) through the
|
||||
required-tool streaming helper and rebuild the calls per array index,
|
||||
checking that every index gets exactly one id/name chunk first."""
|
||||
assert "".join(deltas) == output_json
|
||||
previous_text = ""
|
||||
function_name_returned = False
|
||||
messages = []
|
||||
for i in range(0, len(output_json), delta_len):
|
||||
delta_text = output_json[i : i + delta_len]
|
||||
calls: dict[int, dict] = {}
|
||||
for delta_text in deltas:
|
||||
current_text = previous_text + delta_text
|
||||
|
||||
delta_message, function_name_returned = extract_required_tool_call_streaming(
|
||||
delta_message, _ = extract_required_tool_call_streaming(
|
||||
previous_text=previous_text,
|
||||
current_text=current_text,
|
||||
delta_text=delta_text,
|
||||
function_name_returned=function_name_returned,
|
||||
tool_call_idx=None,
|
||||
tool_call_id_type="random",
|
||||
tool_call_idx=tool_call_idx,
|
||||
tool_call_id_type=tool_call_id_type,
|
||||
)
|
||||
|
||||
if delta_message:
|
||||
messages.append(delta_message)
|
||||
|
||||
previous_text = current_text
|
||||
if delta_message is None:
|
||||
continue
|
||||
assert delta_message.tool_calls
|
||||
for tc in delta_message.tool_calls:
|
||||
assert tc.function is not None
|
||||
if tc.id is not None:
|
||||
assert tc.index not in calls, "id/name sent twice for one index"
|
||||
assert tc.type == "function"
|
||||
assert tc.function.name
|
||||
calls[tc.index] = {
|
||||
"id": tc.id,
|
||||
"name": tc.function.name,
|
||||
"arguments": tc.function.arguments or "",
|
||||
}
|
||||
else:
|
||||
assert tc.index in calls, "arguments before id/name"
|
||||
assert tc.function.name is None
|
||||
assert tc.function.arguments
|
||||
calls[tc.index]["arguments"] += tc.function.arguments
|
||||
assert list(calls) == list(range(len(calls)))
|
||||
return [calls[i] for i in range(len(calls))]
|
||||
|
||||
assert len(messages) > 0
|
||||
|
||||
combined_messages = "["
|
||||
for message in messages:
|
||||
fn = message.tool_calls[0].function
|
||||
assert fn is not None
|
||||
if fn.name:
|
||||
if len(combined_messages) > 1:
|
||||
combined_messages += "},"
|
||||
def _fixed_len_deltas(text: str, delta_len: int) -> list[str]:
|
||||
return [text[i : i + delta_len] for i in range(0, len(text), delta_len)]
|
||||
|
||||
combined_messages += (
|
||||
'{"name": "' + fn.name + '", "parameters": ' + (fn.arguments or "")
|
||||
)
|
||||
else:
|
||||
combined_messages += fn.arguments or ""
|
||||
combined_messages += "}]"
|
||||
return combined_messages
|
||||
|
||||
def _assert_streams_to(output: list[dict], deltas: list[str]) -> None:
|
||||
calls = _stream_required_tool_calls(json.dumps(output), deltas)
|
||||
assert [
|
||||
{"name": c["name"], "parameters": json.loads(c["arguments"])} for c in calls
|
||||
] == output
|
||||
for call, expected in zip(calls, output):
|
||||
assert call["arguments"] == json.dumps(expected["parameters"])
|
||||
|
||||
|
||||
@pytest.mark.parametrize("output", VALID_TOOLS)
|
||||
@@ -343,11 +359,7 @@ def test_streaming_output_valid(output, empty_params, delta_len):
|
||||
output = deepcopy(output)
|
||||
if empty_params:
|
||||
output = [{"name": o["name"], "parameters": {}} for o in output]
|
||||
output_json = json.dumps(output)
|
||||
|
||||
combined_messages = _collect_required_tool_streaming_json(output_json, delta_len)
|
||||
assert json.loads(combined_messages) == output
|
||||
assert json.dumps(json.loads(combined_messages)) == output_json
|
||||
_assert_streams_to(output, _fixed_len_deltas(json.dumps(output), delta_len))
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
@@ -358,22 +370,78 @@ def test_streaming_output_valid(output, empty_params, delta_len):
|
||||
"a }} b",
|
||||
'a " } b',
|
||||
r"a \ } b",
|
||||
# Text that looks like the end of one call and the start of another.
|
||||
'"}}, {"name": "get_forecast", "parameters": {"city": "x',
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize("delta_len", [1, 2, 3, 8, 9999])
|
||||
def test_streaming_output_valid_with_braces_in_string(city, delta_len):
|
||||
output = [{"name": "get_current_weather", "parameters": {"city": city}}]
|
||||
output_json = json.dumps(output)
|
||||
combined_messages = _collect_required_tool_streaming_json(output_json, delta_len)
|
||||
assert json.loads(combined_messages) == output
|
||||
assert json.dumps(json.loads(combined_messages)) == output_json
|
||||
output = [
|
||||
{"name": "get_current_weather", "parameters": {"city": city}},
|
||||
{"name": "get_forecast", "parameters": {"city": city, "days": 1}},
|
||||
]
|
||||
_assert_streams_to(output, _fixed_len_deltas(json.dumps(output), delta_len))
|
||||
|
||||
|
||||
def test_streaming_output_valid_with_trailing_extra_data():
|
||||
output = [{"name": "get_current_weather", "parameters": {"city": "Vienna"}}]
|
||||
output_json = json.dumps(output) + "\nDONE"
|
||||
combined_messages = _collect_required_tool_streaming_json(output_json, delta_len=3)
|
||||
assert json.loads(combined_messages) == output
|
||||
calls = _stream_required_tool_calls(output_json, _fixed_len_deltas(output_json, 3))
|
||||
assert [
|
||||
{"name": c["name"], "parameters": json.loads(c["arguments"])} for c in calls
|
||||
] == output
|
||||
|
||||
|
||||
TWO_CALLS = [
|
||||
{"name": "get_current_weather", "parameters": {"city": "Dallas"}},
|
||||
{"name": "get_forecast", "parameters": {"city": "Dallas", "days": 3}},
|
||||
]
|
||||
|
||||
|
||||
def _split_at(text: str, cuts: list[int]) -> list[str]:
|
||||
bounds = [0, *cuts, len(text)]
|
||||
return [text[a:b] for a, b in zip(bounds, bounds[1:])]
|
||||
|
||||
|
||||
def test_streaming_multiple_calls_completed_in_one_delta():
|
||||
"""Regression test for #60351: a delta that completes more than one
|
||||
call must stream every call, not only the last one."""
|
||||
text = json.dumps(TWO_CALLS)
|
||||
_assert_streams_to(TWO_CALLS, [text[:1], text[1:]])
|
||||
_assert_streams_to(TWO_CALLS, [text])
|
||||
|
||||
|
||||
def test_streaming_delta_crossing_call_boundary():
|
||||
"""A delta that closes one call and opens the next must attribute the
|
||||
closing argument text to the first call and still announce the next."""
|
||||
text = json.dumps(TWO_CALLS)
|
||||
boundary = text.index("}, {") + 1
|
||||
name_end = text.index('"get_forecast"') + 6
|
||||
for cuts in (
|
||||
[boundary - 1, name_end], # '}, {"name": "get_f' in one delta
|
||||
[boundary - 2, boundary + 2], # '"}}, {' then the rest
|
||||
[boundary + 2], # opening brace of the next call ends a delta
|
||||
):
|
||||
_assert_streams_to(TWO_CALLS, _split_at(text, cuts))
|
||||
|
||||
|
||||
def test_streaming_invalid_name_string_is_never_announced():
|
||||
"""A closed name that is not a valid JSON string must not be streamed
|
||||
(the non-streaming path rejects such an array as well)."""
|
||||
text = '[{"name": "bad\\x", "parameters": {"city": "x"}}]'
|
||||
assert _stream_required_tool_calls(text, _fixed_len_deltas(text, 4)) == []
|
||||
|
||||
|
||||
def test_streaming_tool_call_idx_increments_per_started_call():
|
||||
"""kimi_k2 ids number the calls started in one delta consecutively."""
|
||||
text = json.dumps(TWO_CALLS)
|
||||
calls = _stream_required_tool_calls(
|
||||
text, [text], tool_call_id_type="kimi_k2", tool_call_idx=5
|
||||
)
|
||||
assert [c["id"] for c in calls] == [
|
||||
"functions.get_current_weather:5",
|
||||
"functions.get_forecast:6",
|
||||
]
|
||||
|
||||
|
||||
FUNCTION_TOOL = FunctionTool(
|
||||
|
||||
@@ -741,8 +741,6 @@ class DelegatingParser(Parser):
|
||||
extract_required_tool_call_streaming(
|
||||
previous_text=previous_text,
|
||||
current_text=current_text,
|
||||
delta_text=delta_text,
|
||||
function_name_returned=function_name_returned,
|
||||
tool_call_idx=tool_call_idx,
|
||||
tool_call_id_type=tool_call_id_type,
|
||||
)
|
||||
@@ -952,12 +950,13 @@ class DelegatingParser(Parser):
|
||||
elif not delta_message.reasoning:
|
||||
delta_message.reasoning = reasoning_from_this_batch
|
||||
|
||||
if (
|
||||
delta_message
|
||||
and delta_message.tool_calls
|
||||
and delta_message.tool_calls[0].id is not None
|
||||
):
|
||||
state.history_tool_call_cnt += 1
|
||||
if delta_message and delta_message.tool_calls:
|
||||
# A delta may start several tool calls at once (e.g. the
|
||||
# "required" path when one delta completes more than one
|
||||
# call), so count every id rather than only the first.
|
||||
state.history_tool_call_cnt += sum(
|
||||
tc.id is not None for tc in delta_message.tool_calls
|
||||
)
|
||||
|
||||
# No phase active: pass through as content.
|
||||
# Skip when reasoning just ended in this delta — the engine already
|
||||
|
||||
@@ -640,12 +640,6 @@ class OnlineDerenderer:
|
||||
if tc.index < len(last_tool_call_ids):
|
||||
# Pin: reuse the ID already recorded for this index
|
||||
# rather than one a from scratch replay regenerated.
|
||||
# Real trigger not just defensive with
|
||||
# tool_choice="required",
|
||||
# extract_required_tool_call_streaming resets
|
||||
# function_name_returned to False whenever the
|
||||
# partial JSON transiently fails to parse which
|
||||
# re-emits id+name for the same index on replay.
|
||||
tc.id = last_tool_call_ids[tc.index]
|
||||
else:
|
||||
last_tool_call_ids.append(tc.id)
|
||||
|
||||
+221
-146
@@ -2,19 +2,15 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import json
|
||||
from dataclasses import dataclass
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import partial_json_parser
|
||||
import regex as re
|
||||
from partial_json_parser.core.options import Allow
|
||||
|
||||
from vllm.entrypoints.chat_utils import make_tool_call_id
|
||||
from vllm.entrypoints.generate.base.protocol import (
|
||||
DeltaFunctionCall,
|
||||
DeltaMessage,
|
||||
DeltaToolCall,
|
||||
)
|
||||
from vllm.tool_parsers.utils import partial_json_loads
|
||||
from vllm.utils.mistral import is_mistral_tokenizer
|
||||
|
||||
if TYPE_CHECKING:
|
||||
@@ -23,63 +19,6 @@ else:
|
||||
TokenizerLike = object
|
||||
|
||||
|
||||
def _bracket_level_state(
|
||||
s: str, opening: str = "{", closing: str = "}"
|
||||
) -> tuple[int, bool, bool]:
|
||||
level = 0
|
||||
in_string = False
|
||||
escaped = False
|
||||
for char in s:
|
||||
if escaped:
|
||||
escaped = False
|
||||
continue
|
||||
if in_string and char == "\\":
|
||||
escaped = True
|
||||
continue
|
||||
if char == '"':
|
||||
in_string = not in_string
|
||||
continue
|
||||
if not in_string:
|
||||
if char == opening:
|
||||
level += 1
|
||||
elif char == closing:
|
||||
level -= 1
|
||||
return level, in_string, escaped
|
||||
|
||||
|
||||
def filter_delta_text(
|
||||
delta_text: str,
|
||||
previous_text: str,
|
||||
) -> tuple[str, bool]:
|
||||
"""Trim trailing tool-list delimiters from required-tool streaming text."""
|
||||
bracket_level, in_string, escaped = _bracket_level_state(previous_text)
|
||||
updated_delta = ""
|
||||
passed_zero = False
|
||||
for char in delta_text:
|
||||
if escaped:
|
||||
escaped = False
|
||||
elif in_string:
|
||||
if char == "\\":
|
||||
escaped = True
|
||||
elif char == '"':
|
||||
in_string = False
|
||||
elif char == '"':
|
||||
in_string = True
|
||||
elif char == "{":
|
||||
bracket_level += 1
|
||||
passed_zero = bracket_level == 0
|
||||
elif char == "}":
|
||||
bracket_level -= 1
|
||||
passed_zero = bracket_level == 0
|
||||
|
||||
if bracket_level != 0:
|
||||
updated_delta += char
|
||||
else:
|
||||
if not in_string and char == ",":
|
||||
break
|
||||
return updated_delta, passed_zero
|
||||
|
||||
|
||||
def extract_named_tool_call_streaming(
|
||||
*,
|
||||
delta_text: str,
|
||||
@@ -124,100 +63,236 @@ def extract_named_tool_call_streaming(
|
||||
)
|
||||
|
||||
|
||||
@dataclass
|
||||
class _RequiredToolCallSpan:
|
||||
"""Where one element of a ``tool_choice="required"`` array sits in the text.
|
||||
|
||||
The model output for ``tool_choice="required"`` is a JSON array of
|
||||
``{"name": ..., "parameters": ...}`` objects. ``name`` is only set once
|
||||
its string value is closed and decodes as JSON; ``name_end`` is the
|
||||
offset just past that closing quote. ``args_start``/``args_end`` are the
|
||||
character offsets of the ``parameters`` value (``args_end`` is ``None``
|
||||
while the value is still being generated).
|
||||
|
||||
All offsets are prefix-stable: scanning any prefix of the text yields the
|
||||
same spans truncated to that prefix. That is what lets one scan of the
|
||||
current text tell what an earlier, shorter text had already revealed.
|
||||
"""
|
||||
|
||||
name: str | None = None
|
||||
name_end: int | None = None
|
||||
args_start: int | None = None
|
||||
args_end: int | None = None
|
||||
|
||||
def ready_at(self, length: int) -> bool:
|
||||
"""Whether the first ``length`` characters of the text already made
|
||||
this call streamable: its name was complete and its parameters value
|
||||
had started."""
|
||||
return (
|
||||
self.name is not None
|
||||
and self.name_end is not None
|
||||
and self.name_end <= length
|
||||
and self.args_start is not None
|
||||
and self.args_start < length
|
||||
)
|
||||
|
||||
@property
|
||||
def ready(self) -> bool:
|
||||
return self.name is not None and self.args_start is not None
|
||||
|
||||
|
||||
# Parse states for one element object of the required-tool-call array.
|
||||
_EXPECT_KEY = 0
|
||||
_EXPECT_COLON = 1
|
||||
_EXPECT_VALUE = 2
|
||||
_IN_STRING_VALUE = 3
|
||||
_IN_CONTAINER_VALUE = 4
|
||||
_IN_SCALAR_VALUE = 5
|
||||
_AFTER_VALUE = 6
|
||||
|
||||
|
||||
def _scan_required_tool_calls(text: str) -> list[_RequiredToolCallSpan]:
|
||||
"""Locate every tool call of a (possibly partial) required-tool-call array.
|
||||
|
||||
A single string- and nesting-aware pass over ``text`` that records, for
|
||||
each top-level array element, its ``name`` (once complete) and the
|
||||
character span of its ``parameters`` value. Text after the closing ``]``
|
||||
of the array is ignored. Only the element level of the array is
|
||||
interpreted, so braces, brackets and quotes inside nested values and
|
||||
strings cannot confuse it.
|
||||
"""
|
||||
spans: list[_RequiredToolCallSpan] = []
|
||||
depth = 0 # nesting depth of ``{}`` and ``[]``; element objects sit at 2
|
||||
in_string = False
|
||||
escaped = False
|
||||
string_start = 0
|
||||
cur: _RequiredToolCallSpan | None = None
|
||||
expect = _EXPECT_KEY
|
||||
key: str | None = None
|
||||
value_start = 0
|
||||
|
||||
def end_value(pos: int) -> None:
|
||||
nonlocal expect
|
||||
assert cur is not None
|
||||
if key == "parameters":
|
||||
cur.args_end = pos
|
||||
elif key == "name" and expect == _IN_STRING_VALUE:
|
||||
try:
|
||||
cur.name = json.loads(text[value_start:pos])
|
||||
cur.name_end = pos
|
||||
except json.JSONDecodeError:
|
||||
# Not a valid JSON string: the call can never become ready,
|
||||
# which matches the non-streaming path rejecting the array.
|
||||
cur.name = None
|
||||
expect = _AFTER_VALUE
|
||||
|
||||
def start_value(pos: int, state: int) -> None:
|
||||
nonlocal expect, value_start
|
||||
assert cur is not None
|
||||
value_start = pos
|
||||
if key == "parameters":
|
||||
cur.args_start = pos
|
||||
expect = state
|
||||
|
||||
for i, ch in enumerate(text):
|
||||
if in_string:
|
||||
if escaped:
|
||||
escaped = False
|
||||
elif ch == "\\":
|
||||
escaped = True
|
||||
elif ch == '"':
|
||||
in_string = False
|
||||
if depth == 2 and cur is not None:
|
||||
if expect == _EXPECT_KEY:
|
||||
key = text[string_start + 1 : i]
|
||||
expect = _EXPECT_COLON
|
||||
elif expect == _IN_STRING_VALUE:
|
||||
end_value(i + 1)
|
||||
continue
|
||||
|
||||
if ch == '"':
|
||||
in_string = True
|
||||
string_start = i
|
||||
if depth == 2 and cur is not None and expect == _EXPECT_VALUE:
|
||||
start_value(i, _IN_STRING_VALUE)
|
||||
continue
|
||||
|
||||
if ch in "{[":
|
||||
if depth == 0:
|
||||
if ch != "[":
|
||||
return spans
|
||||
elif depth == 1:
|
||||
if ch != "{":
|
||||
return spans
|
||||
cur = _RequiredToolCallSpan()
|
||||
spans.append(cur)
|
||||
expect = _EXPECT_KEY
|
||||
key = None
|
||||
elif depth == 2 and cur is not None and expect == _EXPECT_VALUE:
|
||||
start_value(i, _IN_CONTAINER_VALUE)
|
||||
depth += 1
|
||||
continue
|
||||
|
||||
if ch in "}]":
|
||||
if depth == 2 and cur is not None and expect == _IN_SCALAR_VALUE:
|
||||
end_value(i)
|
||||
depth -= 1
|
||||
if depth == 2 and cur is not None and expect == _IN_CONTAINER_VALUE:
|
||||
end_value(i + 1)
|
||||
elif depth == 1:
|
||||
cur = None
|
||||
elif depth <= 0:
|
||||
return spans
|
||||
continue
|
||||
|
||||
if depth != 2 or cur is None:
|
||||
continue
|
||||
|
||||
if ch == ":":
|
||||
if expect == _EXPECT_COLON:
|
||||
expect = _EXPECT_VALUE
|
||||
elif ch == ",":
|
||||
if expect == _IN_SCALAR_VALUE:
|
||||
end_value(i)
|
||||
expect = _EXPECT_KEY
|
||||
key = None
|
||||
elif ch.isspace():
|
||||
if expect == _IN_SCALAR_VALUE:
|
||||
end_value(i)
|
||||
elif expect == _EXPECT_VALUE:
|
||||
start_value(i, _IN_SCALAR_VALUE)
|
||||
|
||||
return spans
|
||||
|
||||
|
||||
def extract_required_tool_call_streaming(
|
||||
*,
|
||||
previous_text: str,
|
||||
current_text: str | None,
|
||||
delta_text: str,
|
||||
function_name_returned: bool,
|
||||
tool_call_idx: int | None,
|
||||
tool_call_id_type: str,
|
||||
) -> tuple[DeltaMessage | None, bool]:
|
||||
if current_text is None or current_text == "":
|
||||
# if the current text is empty, we cannot parse it
|
||||
return None, function_name_returned
|
||||
try:
|
||||
flags = Allow.ALL
|
||||
obj, _ = partial_json_loads(current_text, flags)
|
||||
except (
|
||||
partial_json_parser.core.exceptions.MalformedJSON,
|
||||
json.JSONDecodeError,
|
||||
):
|
||||
obj = None
|
||||
"""Stream the tool calls of a ``tool_choice="required"`` JSON array.
|
||||
|
||||
# check if the current text is a valid array
|
||||
# containing a partial tool calling object
|
||||
# if not repeat
|
||||
if obj is None or not isinstance(obj, list) or not len(obj) > 0:
|
||||
function_name_returned = False
|
||||
delta_message = None
|
||||
else:
|
||||
_, finishes_previous_tool = filter_delta_text(delta_text, previous_text)
|
||||
# take the last tool call from the generated list
|
||||
current_tool_call = obj[-1]
|
||||
``previous_text`` must be a prefix of ``current_text`` (the accumulated
|
||||
text before and after this delta). Every array element that has something
|
||||
not yet sent is emitted, in array order, so a delta may carry several
|
||||
``DeltaToolCall`` entries: the first chunk of a call carries its id, name
|
||||
and the arguments generated so far; later chunks carry only new argument
|
||||
text. What was already sent is read off the same scan, since the spans
|
||||
are prefix-stable, so no state is needed beyond the two texts.
|
||||
|
||||
# once parameters have been generated the name is complete as well
|
||||
if not finishes_previous_tool and (
|
||||
"name" not in current_tool_call or "parameters" not in current_tool_call
|
||||
):
|
||||
function_name_returned = False
|
||||
delta_message = None
|
||||
else:
|
||||
if not function_name_returned:
|
||||
# get partly generated arguments from the latest tool call
|
||||
param_match = re.search(
|
||||
r'.*"parameters":\s*(.*)', current_text, re.DOTALL
|
||||
)
|
||||
if param_match:
|
||||
arguments = param_match.group(1)
|
||||
arguments_prefix = current_text[: param_match.start(1)]
|
||||
arguments, _ = filter_delta_text(arguments, arguments_prefix)
|
||||
else:
|
||||
arguments = ""
|
||||
Returns the delta (``None`` when nothing is new) and whether at least one
|
||||
call has been announced so far.
|
||||
"""
|
||||
if not current_text:
|
||||
return None, False
|
||||
|
||||
# if this iteration finishes a previous tool call but a
|
||||
# new incomplete tool is already generated, take the
|
||||
# previous from the list
|
||||
if finishes_previous_tool and "parameters" not in current_tool_call:
|
||||
current_tool_call = obj[-2]
|
||||
spans = _scan_required_tool_calls(current_text)
|
||||
sent_len = len(previous_text)
|
||||
|
||||
function_name_returned = True
|
||||
tool_call_id = make_tool_call_id(
|
||||
id_type=tool_call_id_type,
|
||||
func_name=current_tool_call["name"],
|
||||
idx=tool_call_idx,
|
||||
)
|
||||
delta_message = DeltaMessage(
|
||||
tool_calls=[
|
||||
DeltaToolCall(
|
||||
id=tool_call_id,
|
||||
function=DeltaFunctionCall(
|
||||
name=current_tool_call["name"], arguments=arguments
|
||||
),
|
||||
index=len(obj) - 1,
|
||||
type="function",
|
||||
)
|
||||
]
|
||||
)
|
||||
|
||||
else:
|
||||
delta_text, _ = filter_delta_text(delta_text, previous_text)
|
||||
|
||||
if delta_text != "":
|
||||
delta_message = DeltaMessage(
|
||||
tool_calls=[
|
||||
DeltaToolCall(
|
||||
function=DeltaFunctionCall(
|
||||
# OpenAI API returns None
|
||||
# instead of name every time
|
||||
name=None,
|
||||
arguments=delta_text,
|
||||
),
|
||||
index=len(obj) - 1,
|
||||
)
|
||||
]
|
||||
tool_calls: list[DeltaToolCall] = []
|
||||
started = 0
|
||||
for index, span in enumerate(spans):
|
||||
if not span.ready:
|
||||
# Elements are generated in order: nothing later can be ready.
|
||||
break
|
||||
assert span.args_start is not None
|
||||
end = span.args_end if span.args_end is not None else len(current_text)
|
||||
if span.ready_at(sent_len):
|
||||
# Already announced: send only the new argument text.
|
||||
sent_end = min(end, sent_len)
|
||||
new_arguments = current_text[sent_end:end]
|
||||
if new_arguments:
|
||||
tool_calls.append(
|
||||
DeltaToolCall(
|
||||
index=index,
|
||||
function=DeltaFunctionCall(
|
||||
# OpenAI API returns None instead of name every time
|
||||
name=None,
|
||||
arguments=new_arguments,
|
||||
),
|
||||
)
|
||||
else:
|
||||
delta_message = None
|
||||
)
|
||||
continue
|
||||
|
||||
return delta_message, function_name_returned
|
||||
idx = None if tool_call_idx is None else tool_call_idx + started
|
||||
started += 1
|
||||
tool_calls.append(
|
||||
DeltaToolCall(
|
||||
id=make_tool_call_id(
|
||||
id_type=tool_call_id_type, func_name=span.name, idx=idx
|
||||
),
|
||||
type="function",
|
||||
index=index,
|
||||
function=DeltaFunctionCall(
|
||||
name=span.name,
|
||||
arguments=current_text[span.args_start : end],
|
||||
),
|
||||
)
|
||||
)
|
||||
|
||||
function_name_returned = any(span.ready for span in spans)
|
||||
if not tool_calls:
|
||||
return None, function_name_returned
|
||||
return DeltaMessage(tool_calls=tool_calls), function_name_returned
|
||||
|
||||
Reference in New Issue
Block a user