[Refactor] Move unstreamed tool-arg flush from serving layer to parser (#44017)

Signed-off-by: sfeng33 <4florafeng@gmail.com>
This commit is contained in:
Flora Feng
2026-06-02 10:37:43 +08:00
committed by GitHub
parent 816cc73a9b
commit 9affc17a05
5 changed files with 138 additions and 264 deletions
+19
View File
@@ -79,6 +79,25 @@ class ToolParser:
else:
self.tools = []
def get_remaining_unstreamed_args(self) -> str:
"""Return tool call arguments parsed but not yet streamed."""
if not self.prev_tool_call_arr:
return ""
index = len(self.prev_tool_call_arr) - 1
args = self.prev_tool_call_arr[index].get("arguments", {})
if isinstance(args, str):
expected = args
else:
expected = json.dumps(args, ensure_ascii=False)
actual = (
self.streamed_args_for_tool[index]
if index < len(self.streamed_args_for_tool)
else ""
)
if expected.startswith(actual):
return expected[len(actual) :]
return ""
@cached_property
def vocab(self) -> dict[str, int]:
# NOTE: Only PreTrainedTokenizerFast is guaranteed to have .vocab