[Bugfix][Tool Parser] Migrate Granite to the streaming Parser Engine (#49648)
Signed-off-by: Nikhil Kulkarni <nikhilkulkarni1755@gmail.com> Co-authored-by: Claude Opus 4.8 <noreply@anthropic.com> Co-authored-by: Chauncey <chaunceyjiang@gmail.com>
This commit is contained in:
co-authored by
Claude Opus 4.8
Chauncey
parent
c961121519
commit
15ed1262e7
@@ -99,6 +99,10 @@ def _discover_pairings() -> list[_PairingInfo]:
|
||||
# terminal matching; combined-parser replay coverage lives in
|
||||
# test_inkling.py.
|
||||
continue
|
||||
if cfg.name == "granite":
|
||||
# Granite has a JSON-array tool body with no TOOL_END terminal;
|
||||
# its replay coverage lives in test_granite.py.
|
||||
continue
|
||||
|
||||
parser_cls = type(
|
||||
f"_Delegating{engine_cls.__name__}",
|
||||
|
||||
@@ -0,0 +1,234 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Tests for the engine-based Granite parser.
|
||||
|
||||
Granite emits tool calls as a JSON array following a marker
|
||||
(``<|tool_call|>`` for 3.0, ``<tool_call>`` for 3.1) with no closing
|
||||
terminal; the ``]`` ends the region and surrounding prose is content. The
|
||||
engine's ``tool_call_body_array`` mode splits the array into one call per
|
||||
element and ``_granite_arg_converter`` carves each element's ``arguments``.
|
||||
"""
|
||||
|
||||
import json
|
||||
|
||||
import pytest
|
||||
|
||||
from tests.parser.engine.conftest import make_mock_tokenizer
|
||||
from tests.parser.engine.trace_builder import _GRANITE_SCENARIOS, _build_granite
|
||||
from vllm.parser.granite import GraniteParser, _granite_arg_converter
|
||||
|
||||
TOOL_TOKEN = "<|tool_call|>"
|
||||
TOOL_STRING = "<tool_call>"
|
||||
|
||||
_GRANITE_VOCAB = {TOOL_TOKEN: 49154}
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mock_tokenizer():
|
||||
return make_mock_tokenizer(_GRANITE_VOCAB)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def parser(mock_tokenizer):
|
||||
return GraniteParser(mock_tokenizer)
|
||||
|
||||
|
||||
def _tokenize(text: str) -> list[tuple[int, str]]:
|
||||
"""Tokenize like the real stream: ``<|tool_call|>`` is one special token,
|
||||
plain text becomes one token per character."""
|
||||
tokens: list[tuple[int, str]] = []
|
||||
i = 0
|
||||
while i < len(text):
|
||||
if text.startswith(TOOL_TOKEN, i):
|
||||
tokens.append((_GRANITE_VOCAB[TOOL_TOKEN], TOOL_TOKEN))
|
||||
i += len(TOOL_TOKEN)
|
||||
else:
|
||||
tokens.append((ord(text[i]), text[i]))
|
||||
i += 1
|
||||
return tokens
|
||||
|
||||
|
||||
def _stream(parser, request, text: str, chunk_size: int):
|
||||
tokens = _tokenize(text)
|
||||
results = []
|
||||
previous_text = ""
|
||||
previous_token_ids: list[int] = []
|
||||
for start in range(0, len(tokens), chunk_size):
|
||||
batch = tokens[start : start + chunk_size]
|
||||
delta_text = "".join(t for _, t in batch)
|
||||
delta_token_ids = [tid for tid, _ in batch]
|
||||
current_text = previous_text + delta_text
|
||||
current_token_ids = previous_token_ids + delta_token_ids
|
||||
delta = parser.extract_tool_calls_streaming(
|
||||
previous_text=previous_text,
|
||||
current_text=current_text,
|
||||
delta_text=delta_text,
|
||||
previous_token_ids=tuple(previous_token_ids),
|
||||
current_token_ids=tuple(current_token_ids),
|
||||
delta_token_ids=tuple(delta_token_ids),
|
||||
request=request,
|
||||
)
|
||||
results.append(delta)
|
||||
previous_text = current_text
|
||||
previous_token_ids = current_token_ids
|
||||
finish = parser.finish_streaming()
|
||||
if finish is not None:
|
||||
results.append(finish)
|
||||
return results
|
||||
|
||||
|
||||
def _collect_content(results) -> str:
|
||||
return "".join(d.content for d in results if d and d.content)
|
||||
|
||||
|
||||
def _collect_names(results) -> list[str]:
|
||||
names: dict[int, str] = {}
|
||||
for d in results:
|
||||
if not (d and d.tool_calls):
|
||||
continue
|
||||
for tc in d.tool_calls:
|
||||
if tc.function and tc.function.name:
|
||||
names[tc.index] = names.get(tc.index, "") + tc.function.name
|
||||
return [names[i] for i in sorted(names)]
|
||||
|
||||
|
||||
def _collect_args(results) -> dict[int, str]:
|
||||
args: dict[int, str] = {}
|
||||
for d in results:
|
||||
if not (d and d.tool_calls):
|
||||
continue
|
||||
for tc in d.tool_calls:
|
||||
if tc.function and tc.function.arguments:
|
||||
args[tc.index] = args.get(tc.index, "") + tc.function.arguments
|
||||
return args
|
||||
|
||||
|
||||
class TestArgConverter:
|
||||
def test_complete_wrapper(self):
|
||||
raw = '{"name": "get_weather", "arguments": {"city": "SF"}}'
|
||||
assert _granite_arg_converter(raw, False) == '{"city": "SF"}'
|
||||
|
||||
def test_arguments_before_name(self):
|
||||
raw = '{"arguments": {"city": "SF"}, "name": "get_weather"}'
|
||||
assert _granite_arg_converter(raw, False) == '{"city": "SF"}'
|
||||
|
||||
def test_partial_before_arguments(self):
|
||||
assert _granite_arg_converter('{"name": "get_w', True) == ""
|
||||
|
||||
def test_partial_inside_arguments(self):
|
||||
raw = '{"name": "x", "arguments": {"a": 1'
|
||||
assert _granite_arg_converter(raw, True) == '{"a": 1'
|
||||
|
||||
def test_prefix_stability(self):
|
||||
# Each growing prefix must extend the previous converter output.
|
||||
full = '{"name": "x", "arguments": {"city": "Tokyo"}}'
|
||||
prev = ""
|
||||
for i in range(1, len(full) + 1):
|
||||
out = _granite_arg_converter(full[:i], True)
|
||||
assert out.startswith(prev) or prev.startswith(out) or out == ""
|
||||
if out:
|
||||
prev = out
|
||||
|
||||
def test_missing_arguments_defaults(self):
|
||||
assert _granite_arg_converter('{"name": "x"}', False) == "{}"
|
||||
assert _granite_arg_converter('{"name": "x"}', True) == ""
|
||||
|
||||
def test_non_object_arguments_rejected(self):
|
||||
with pytest.raises(ValueError, match="JSON object"):
|
||||
_granite_arg_converter('{"name": "x", "arguments": [1]}', False)
|
||||
|
||||
|
||||
class TestNonStreaming:
|
||||
def test_plain_text(self, parser, mock_request):
|
||||
reasoning, content, tools = parser.parse("Just a reply.", mock_request)
|
||||
assert reasoning is None
|
||||
assert content == "Just a reply."
|
||||
assert tools is None
|
||||
|
||||
def test_single_tool_token_marker(self, parser, mock_request):
|
||||
text = (
|
||||
f'{TOOL_TOKEN} [{{"name": "get_weather", "arguments": {{"city": "SF"}}}}]'
|
||||
)
|
||||
_, content, tools = parser.parse(text, mock_request)
|
||||
assert content is None
|
||||
assert [t.name for t in tools] == ["get_weather"]
|
||||
assert json.loads(tools[0].arguments) == {"city": "SF"}
|
||||
|
||||
def test_single_tool_string_marker(self, parser, mock_request):
|
||||
# Granite 3.1 uses the plain-text ``<tool_call>`` marker.
|
||||
text = f'{TOOL_STRING} [{{"name": "get_time", "arguments": {{}}}}]'
|
||||
_, content, tools = parser.parse(text, mock_request)
|
||||
assert [t.name for t in tools] == ["get_time"]
|
||||
assert json.loads(tools[0].arguments) == {}
|
||||
|
||||
def test_parallel_calls_one_array(self, parser, mock_request):
|
||||
text = (
|
||||
f'{TOOL_TOKEN} [{{"name": "a", "arguments": {{"x": 1}}}}, '
|
||||
f'{{"name": "b", "arguments": {{"y": [1, 2]}}}}]'
|
||||
)
|
||||
_, _, tools = parser.parse(text, mock_request)
|
||||
assert [t.name for t in tools] == ["a", "b"]
|
||||
assert json.loads(tools[0].arguments) == {"x": 1}
|
||||
assert json.loads(tools[1].arguments) == {"y": [1, 2]}
|
||||
|
||||
def test_surrounding_text_is_content(self, parser, mock_request):
|
||||
text = (
|
||||
"Let me check.\n"
|
||||
f'{TOOL_TOKEN} [{{"name": "get_weather", "arguments": {{"city": "SF"}}}}]'
|
||||
)
|
||||
_, content, tools = parser.parse(text, mock_request)
|
||||
assert content == "Let me check."
|
||||
assert [t.name for t in tools] == ["get_weather"]
|
||||
|
||||
def test_arguments_before_name(self, parser, mock_request):
|
||||
text = (
|
||||
f'{TOOL_TOKEN} [{{"arguments": {{"city": "SF"}}, "name": "get_weather"}}]'
|
||||
)
|
||||
_, _, tools = parser.parse(text, mock_request)
|
||||
assert [t.name for t in tools] == ["get_weather"]
|
||||
assert json.loads(tools[0].arguments) == {"city": "SF"}
|
||||
|
||||
def test_marker_without_array_is_not_a_tool_call(self, parser, mock_request):
|
||||
# A non-array body after the marker (malformed) yields no tool call.
|
||||
text = f'{TOOL_TOKEN} {{"name": "func", "arguments": {{}}}}'
|
||||
_, _, tools = parser.parse(text, mock_request)
|
||||
assert not tools
|
||||
|
||||
|
||||
class TestStreaming:
|
||||
@pytest.mark.parametrize("chunk_size", [1, 2, 3, 5, 11])
|
||||
def test_parallel_calls_chunk_invariance(self, parser, mock_request, chunk_size):
|
||||
text = (
|
||||
f'{TOOL_TOKEN} [{{"name": "get_weather", '
|
||||
f'"arguments": {{"city": "Tokyo"}}}}, '
|
||||
f'{{"name": "get_time", "arguments": {{"timezone": "Asia/Tokyo"}}}}]'
|
||||
)
|
||||
results = _stream(parser, mock_request, text, chunk_size)
|
||||
assert _collect_names(results) == ["get_weather", "get_time"]
|
||||
args = _collect_args(results)
|
||||
assert json.loads(args[0]) == {"city": "Tokyo"}
|
||||
assert json.loads(args[1]) == {"timezone": "Asia/Tokyo"}
|
||||
|
||||
@pytest.mark.parametrize("chunk_size", [1, 3, 7])
|
||||
def test_surrounding_text_streaming(self, parser, mock_request, chunk_size):
|
||||
text = (
|
||||
"Let me check.\n"
|
||||
f'{TOOL_TOKEN} [{{"name": "get_weather", "arguments": {{"city": "SF"}}}}]'
|
||||
)
|
||||
results = _stream(parser, mock_request, text, chunk_size)
|
||||
assert _collect_names(results) == ["get_weather"]
|
||||
assert "Let me check." in _collect_content(results)
|
||||
|
||||
def test_incomplete_array_at_eos(self, parser, mock_request):
|
||||
# Truncated mid-element must not raise and must not leak the marker.
|
||||
text = f'{TOOL_TOKEN} [{{"name": "func", "arguments": {{"a": 1'
|
||||
results = _stream(parser, mock_request, text, 3)
|
||||
assert TOOL_TOKEN not in _collect_content(results)
|
||||
|
||||
|
||||
class TestReplay:
|
||||
@pytest.mark.parametrize("scenario", _GRANITE_SCENARIOS, ids=lambda s: s.id)
|
||||
def test_replay_scenarios(self, scenario):
|
||||
# ``_build_granite`` replays the sample through GraniteParser at
|
||||
# chunk_size=1 and asserts reasoning/content/tool_calls (validate=True).
|
||||
_build_granite(scenario)
|
||||
@@ -78,6 +78,11 @@ def _discover_parsers() -> list[_ParserInfo]:
|
||||
# Inkling opts out of token-id terminal matching and has typed
|
||||
# structural blocks; its replay coverage lives in test_inkling.py.
|
||||
continue
|
||||
if cfg.name == "granite":
|
||||
# Granite has a JSON-array tool body with no TOOL_END terminal, so
|
||||
# it does not fit this token-terminal harness; its replay coverage
|
||||
# lives in test_granite.py.
|
||||
continue
|
||||
tool_end = cfg.token_id_terminals.get("TOOL_END")
|
||||
if not tool_end:
|
||||
raise RuntimeError(
|
||||
|
||||
@@ -34,6 +34,7 @@ from vllm.parser.engine.registered_adapters import (
|
||||
DeepSeekV41Parser,
|
||||
Gemma4Parser,
|
||||
Glm47MoeParser,
|
||||
GraniteParser,
|
||||
InklingParser,
|
||||
KimiK2Parser,
|
||||
MinimaxM2Parser,
|
||||
@@ -1042,6 +1043,89 @@ def _build_inkling(scenario: Scenario, validate: bool = True) -> Sample:
|
||||
return sample
|
||||
|
||||
|
||||
# ── Granite (JSON-array tool bodies, no reasoning) ───────────────────────
|
||||
|
||||
_GRANITE_VOCAB: dict[str, int] = {
|
||||
"<|tool_call|>": 49154,
|
||||
}
|
||||
|
||||
|
||||
def _granite_segments(scenario: Scenario) -> list[tuple[str, bool]]:
|
||||
segs: list[tuple[str, bool]] = []
|
||||
if scenario.content:
|
||||
# Granite has no reasoning; prose is plain content preceding the marker.
|
||||
segs.append((scenario.content, False))
|
||||
if scenario.tool_calls:
|
||||
segs.append(("<|tool_call|>", True))
|
||||
payload = json.dumps(
|
||||
[
|
||||
{"name": tc.name, "arguments": tc.arguments}
|
||||
for tc in scenario.tool_calls
|
||||
],
|
||||
ensure_ascii=False,
|
||||
separators=(", ", ": "),
|
||||
)
|
||||
segs.append((" " + payload, False))
|
||||
return segs
|
||||
|
||||
|
||||
def _granite_expected_content(scenario: Scenario) -> str | None:
|
||||
if scenario.tool_calls:
|
||||
if not scenario.content:
|
||||
return None
|
||||
return scenario.content.strip() or None
|
||||
return scenario.content
|
||||
|
||||
|
||||
def _build_granite(scenario: Scenario, validate: bool = True) -> Sample:
|
||||
sample = _make_sample(
|
||||
sample_id=f"granite-{scenario.id}",
|
||||
description=scenario.description,
|
||||
vocab=_GRANITE_VOCAB,
|
||||
segments=_granite_segments(scenario),
|
||||
expected_reasoning=None,
|
||||
expected_content=_granite_expected_content(scenario),
|
||||
expected_tool_calls=_expected_tc(scenario),
|
||||
tools=_expected_tools(scenario),
|
||||
)
|
||||
if validate:
|
||||
_validate_sample(sample, GraniteParser)
|
||||
return sample
|
||||
|
||||
|
||||
# Granite has no reasoning, so the shared reasoning-centric SCENARIOS do not
|
||||
# apply; these exercise the JSON-array tool body (single, parallel, surrounding
|
||||
# text) instead.
|
||||
_GRANITE_SCENARIOS: list[Scenario] = [
|
||||
Scenario(
|
||||
id="single-tool",
|
||||
description="Single tool call",
|
||||
tool_calls=[_READ_TOOL],
|
||||
),
|
||||
Scenario(
|
||||
id="parallel-tools",
|
||||
description="Parallel tool calls in one JSON array",
|
||||
tool_calls=[_BASH_TOOL, _WEATHER_TOOL],
|
||||
),
|
||||
Scenario(
|
||||
id="complex-json-args",
|
||||
description="Tool call with nested objects, arrays, numbers, booleans",
|
||||
tool_calls=[_COMPLEX_TOOL],
|
||||
),
|
||||
Scenario(
|
||||
id="content-only",
|
||||
description="Plain content response without tool calls",
|
||||
content="Hello! How can I help you today?",
|
||||
),
|
||||
Scenario(
|
||||
id="surrounding-text",
|
||||
description="Prose content preceding the tool call",
|
||||
content="Let me check the weather.",
|
||||
tool_calls=[_WEATHER_TOOL],
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
# ── Registry and public API ──────────────────────────────────────────
|
||||
|
||||
_BUILDERS: dict[str, Any] = {
|
||||
@@ -1049,6 +1133,7 @@ _BUILDERS: dict[str, Any] = {
|
||||
"deepseek_v4": _build_deepseek_v4,
|
||||
"deepseek_v41": functools.partial(_build_deepseek_v4, v41=True),
|
||||
"gemma4": _build_gemma4,
|
||||
"granite": _build_granite,
|
||||
"minimax_m2": _build_minimax_m2,
|
||||
"nemotron_v3": _build_nemotron_v3,
|
||||
"seed_oss": _build_seed_oss,
|
||||
|
||||
@@ -16,7 +16,7 @@ from tests.tool_parsers.utils import (
|
||||
split_string_into_token_deltas,
|
||||
)
|
||||
from vllm.tokenizers import get_tokenizer
|
||||
from vllm.tool_parsers.granite_tool_parser import GraniteToolParser
|
||||
from vllm.tool_parsers.granite_engine_tool_parser import GraniteEngineToolParser
|
||||
|
||||
|
||||
class TestGraniteToolParser(ToolParserTests):
|
||||
@@ -76,23 +76,6 @@ I'll get that information.""",
|
||||
single_tool_call_expected_content=None,
|
||||
parallel_tool_calls_count=2,
|
||||
parallel_tool_calls_names=["get_weather", "get_time"],
|
||||
# xfail markers
|
||||
xfail_streaming={
|
||||
"test_malformed_input": (
|
||||
"Streaming mode incorrectly creates tool call from malformed JSON"
|
||||
),
|
||||
"test_surrounding_text": (
|
||||
"Parser doesn't handle surrounding text correctly in streaming"
|
||||
),
|
||||
"test_streaming_reconstruction": (
|
||||
"Streaming mode doesn't strip <|tool_call|> marker from content"
|
||||
),
|
||||
},
|
||||
xfail_nonstreaming={
|
||||
"test_surrounding_text": (
|
||||
"Parser doesn't handle surrounding text correctly in non-streaming"
|
||||
),
|
||||
},
|
||||
)
|
||||
|
||||
# Granite-Specific Tests
|
||||
@@ -140,7 +123,7 @@ def test_streaming_parallel_calls_batched_deltas(granite_tokenizer, chunk_size):
|
||||
parallel calls must not drop the first call's name. granite streams
|
||||
arguments before name, so the name only completes as the next call appears.
|
||||
"""
|
||||
parser = GraniteToolParser(granite_tokenizer)
|
||||
parser = GraniteEngineToolParser(granite_tokenizer)
|
||||
model_output = (
|
||||
'<|tool_call|> [{"arguments": {"city": "Tokyo"}, "name": "get_weather"}, '
|
||||
'{"arguments": {"timezone": "Asia/Tokyo"}, "name": "get_time"}]'
|
||||
|
||||
@@ -82,6 +82,12 @@ class ParserEngineConfig:
|
||||
|
||||
tool_args_json: bool = True
|
||||
|
||||
# The tool region body is a single JSON array of ``{"name", "arguments"}``
|
||||
# objects (one marker → N calls, no closing terminal). The engine splits
|
||||
# the array into one tool call per element; the array's ``]`` ends the
|
||||
# region and any trailing text becomes content. Used by Granite.
|
||||
tool_call_body_array: bool = False
|
||||
|
||||
arg_structural_chars: frozenset[str] | None = None
|
||||
|
||||
# Special tokens exempt from auto-drop but not state-machine terminals.
|
||||
|
||||
@@ -13,6 +13,7 @@ from vllm.parser.deepseek_v41 import DeepSeekV41Parser
|
||||
from vllm.parser.engine.adapters import make_adapters
|
||||
from vllm.parser.gemma4 import Gemma4Parser
|
||||
from vllm.parser.glm47_moe import Glm47MoeParser
|
||||
from vllm.parser.granite import GraniteParser
|
||||
from vllm.parser.inkling import InklingParser
|
||||
from vllm.parser.kimi_k2 import KimiK2Parser
|
||||
from vllm.parser.minimax_m2 import MinimaxM2Parser
|
||||
@@ -46,6 +47,11 @@ from vllm.parser.seed_oss import SeedOssParser
|
||||
Gemma4ParserToolAdapter,
|
||||
) = make_adapters(Gemma4Parser)
|
||||
|
||||
(
|
||||
GraniteParserReasoningAdapter,
|
||||
GraniteParserToolAdapter,
|
||||
) = make_adapters(GraniteParser)
|
||||
|
||||
(
|
||||
NemotronV3ParserReasoningAdapter,
|
||||
NemotronV3ParserToolAdapter,
|
||||
|
||||
@@ -190,6 +190,12 @@ class StreamingParserEngine:
|
||||
self._args_in_string: bool = False
|
||||
self._args_escape_next: bool = False
|
||||
|
||||
def _reset_array_state(self) -> None:
|
||||
# State for ``tool_call_body_array``: whether the opening ``[`` has been
|
||||
# seen, and whether a ``{...}`` element is currently being streamed.
|
||||
self._array_started: bool = False
|
||||
self._array_in_element: bool = False
|
||||
|
||||
def reset(self, initial_state: ParserState | None = None) -> None:
|
||||
"""Reset mutable state for reuse across requests.
|
||||
|
||||
@@ -213,6 +219,7 @@ class StreamingParserEngine:
|
||||
self._message_header_token_count = 0
|
||||
self._in_skipped_tool_span = False
|
||||
self._reset_args_state()
|
||||
self._reset_array_state()
|
||||
|
||||
def feed(
|
||||
self,
|
||||
@@ -306,7 +313,12 @@ class StreamingParserEngine:
|
||||
ParserState.TOOL_NAME,
|
||||
ParserState.TOOL_BETWEEN,
|
||||
):
|
||||
if self.tool_index >= 0:
|
||||
# In array mode a closed element has already emitted its
|
||||
# TOOL_CALL_END; only a truncated (still-open) element needs one.
|
||||
end_open_call = (
|
||||
self._array_in_element if self.config.tool_call_body_array else True
|
||||
)
|
||||
if end_open_call and self.tool_index >= 0:
|
||||
events.append(
|
||||
SemanticEvent(
|
||||
EventType.TOOL_CALL_END,
|
||||
@@ -474,6 +486,8 @@ class StreamingParserEngine:
|
||||
self._message_header_token_count += token_count
|
||||
return []
|
||||
if self.state == ParserState.TOOL_ARGS:
|
||||
if self.config.tool_call_body_array:
|
||||
return self._feed_array_text(text)
|
||||
if self.config.tool_args_json:
|
||||
return self._feed_args_text(text)
|
||||
return [
|
||||
@@ -564,6 +578,8 @@ class StreamingParserEngine:
|
||||
self._args_in_string = False
|
||||
self._args_escape_next = False
|
||||
self._args_safe_end = 0
|
||||
if self.config.tool_call_body_array:
|
||||
self._reset_array_state()
|
||||
|
||||
return events
|
||||
|
||||
@@ -634,3 +650,88 @@ class StreamingParserEngine:
|
||||
tool_index=self.tool_index,
|
||||
)
|
||||
]
|
||||
|
||||
# ── JSON-array tool body (tool_call_body_array) ─────────────────────
|
||||
|
||||
def _feed_array_text(self, text: str) -> list[SemanticEvent]:
|
||||
"""Split a JSON-array tool body into one tool call per element.
|
||||
|
||||
Granite emits ``[{"name":..,"arguments":{..}}, ...]`` after the tool
|
||||
marker. Element interiors are streamed through the string/brace-aware
|
||||
:meth:`_feed_args_char`, so each element behaves exactly like a
|
||||
single-object tool body; the array's ``]`` ends the region and any
|
||||
trailing text becomes content.
|
||||
"""
|
||||
events: list[SemanticEvent] = []
|
||||
for i, ch in enumerate(text):
|
||||
if self.state != ParserState.TOOL_ARGS:
|
||||
# The array closed mid-chunk; the rest of the text is content.
|
||||
events.extend(self._emit_for_state(text[i:]))
|
||||
break
|
||||
events.extend(self._feed_array_char(ch))
|
||||
return events
|
||||
|
||||
def _feed_array_char(self, ch: str) -> list[SemanticEvent]:
|
||||
if self._array_in_element:
|
||||
events = self._feed_args_char(ch)
|
||||
# Element object closed: top-level '}' with all braces balanced.
|
||||
if (
|
||||
ch == "}"
|
||||
and not self._args_in_string
|
||||
and not self._args_escape_next
|
||||
and self._args_brace_depth == 0
|
||||
):
|
||||
events.extend(self._end_array_element())
|
||||
return events
|
||||
|
||||
# Between elements only structural array characters are expected.
|
||||
if ch in " \t\r\n":
|
||||
return []
|
||||
if not self._array_started:
|
||||
if ch == "[":
|
||||
self._array_started = True
|
||||
return []
|
||||
return self._bail_array_to_content(ch)
|
||||
if ch == ",":
|
||||
return []
|
||||
if ch == "{":
|
||||
return self._start_array_element(ch)
|
||||
if ch == "]":
|
||||
self.state = ParserState.CONTENT
|
||||
self._reset_array_state()
|
||||
return []
|
||||
return self._bail_array_to_content(ch)
|
||||
|
||||
def _start_array_element(self, ch: str) -> list[SemanticEvent]:
|
||||
self.tool_index += 1
|
||||
self._reset_args_state()
|
||||
self._array_in_element = True
|
||||
events = [SemanticEvent(EventType.TOOL_CALL_START, tool_index=self.tool_index)]
|
||||
events.extend(self._feed_args_char(ch))
|
||||
return events
|
||||
|
||||
def _end_array_element(self) -> list[SemanticEvent]:
|
||||
self._array_in_element = False
|
||||
events: list[SemanticEvent] = []
|
||||
# Flush the held closing '}' so the slot body is the full element.
|
||||
if self._args_buffer:
|
||||
events.append(
|
||||
SemanticEvent(
|
||||
EventType.ARG_VALUE_CHUNK,
|
||||
value=self._args_buffer,
|
||||
tool_index=self.tool_index,
|
||||
)
|
||||
)
|
||||
self._args_buffer = ""
|
||||
self._args_safe_end = 0
|
||||
events.append(
|
||||
SemanticEvent(EventType.TOOL_CALL_END, tool_index=self.tool_index)
|
||||
)
|
||||
return events
|
||||
|
||||
def _bail_array_to_content(self, ch: str) -> list[SemanticEvent]:
|
||||
# Marker not followed by a JSON array (malformed) — revert to content
|
||||
# so the remaining tokens are emitted as plain text, no tool call.
|
||||
self.state = ParserState.CONTENT
|
||||
self._reset_array_state()
|
||||
return self._emit_for_state(ch)
|
||||
|
||||
@@ -0,0 +1,88 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Granite parser for JSON-array tool calls.
|
||||
|
||||
Granite 3.0/3.1 emit tool calls as a JSON array following a marker::
|
||||
|
||||
<|tool_call|> [{"name": "get_weather", "arguments": {"city": "SF"}}] # 3.0
|
||||
<tool_call> [{"name": "get_weather", "arguments": {"city": "SF"}}] # 3.1
|
||||
|
||||
``<|tool_call|>`` is a single special token; ``<tool_call>`` is plain text.
|
||||
The array may hold several calls, has no closing marker (the ``]`` ends it),
|
||||
and any surrounding prose is content. The engine's ``tool_call_body_array``
|
||||
mode splits the array into one call per element and extracts each name via the
|
||||
name-from-args path; ``_granite_arg_converter`` carves the ``arguments`` object
|
||||
out of each ``{"name":..,"arguments":{..}}`` element. Granite has no reasoning.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import functools
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from vllm.parser.engine.parser_engine import ParserEngine
|
||||
from vllm.parser.engine.parser_engine_config import (
|
||||
ParserEngineConfig,
|
||||
ParserState,
|
||||
Transition,
|
||||
)
|
||||
from vllm.parser.utils import extract_json_member_object
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.tokenizers import TokenizerLike
|
||||
from vllm.tool_parsers.abstract_tool_parser import Tool
|
||||
|
||||
TOOL_TOKEN = "<|tool_call|>" # granite 3.0 (single special token)
|
||||
TOOL_STRING = "<tool_call>" # granite 3.1 (plain text)
|
||||
|
||||
|
||||
def _granite_arg_converter(raw_args: str, partial: bool) -> str:
|
||||
"""Carve the ``arguments`` object out of a ``{"name":..,"arguments":{..}}``
|
||||
array element (see :func:`extract_json_member_object`)."""
|
||||
span = extract_json_member_object(raw_args, "arguments")
|
||||
if span is None:
|
||||
return "" if partial else "{}"
|
||||
return span
|
||||
|
||||
|
||||
@functools.cache
|
||||
def granite_config() -> ParserEngineConfig:
|
||||
return ParserEngineConfig(
|
||||
name="granite",
|
||||
initial_state=ParserState.CONTENT,
|
||||
terminals={
|
||||
"TOOL_TOKEN": TOOL_TOKEN,
|
||||
"TOOL_STRING": TOOL_STRING,
|
||||
},
|
||||
token_id_terminals={
|
||||
"TOOL_TOKEN": TOOL_TOKEN,
|
||||
},
|
||||
transitions={
|
||||
# The marker only switches into the tool region; the array splitter
|
||||
# emits TOOL_CALL_START per element (empty events here).
|
||||
(ParserState.CONTENT, "TOOL_TOKEN"): Transition(
|
||||
ParserState.TOOL_ARGS,
|
||||
(),
|
||||
),
|
||||
(ParserState.CONTENT, "TOOL_STRING"): Transition(
|
||||
ParserState.TOOL_ARGS,
|
||||
(),
|
||||
),
|
||||
},
|
||||
arg_converter=_granite_arg_converter,
|
||||
tool_call_body_array=True,
|
||||
stream_arg_deltas=True,
|
||||
)
|
||||
|
||||
|
||||
class GraniteParser(ParserEngine):
|
||||
CONFIG_NAME = "granite"
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
tokenizer: TokenizerLike,
|
||||
tools: list[Tool] | None = None,
|
||||
**kwargs,
|
||||
) -> None:
|
||||
kwargs.setdefault("parser_engine_config", granite_config())
|
||||
super().__init__(tokenizer, tools, **kwargs)
|
||||
+4
-90
@@ -37,6 +37,7 @@ from vllm.parser.engine.parser_engine_config import (
|
||||
ParserState,
|
||||
Transition,
|
||||
)
|
||||
from vllm.parser.utils import extract_json_member_object
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.tokenizers import TokenizerLike
|
||||
@@ -62,84 +63,6 @@ INKLING_SPECIAL_TOKENS = (
|
||||
END_MESSAGE,
|
||||
)
|
||||
|
||||
_WS = " \t\r\n"
|
||||
|
||||
|
||||
def _scan_json_value(raw: str, start: int) -> int | None:
|
||||
"""Return the end index (exclusive) of the JSON object starting at
|
||||
``raw[start]``, or ``None`` when the object is still unterminated."""
|
||||
depth = 0
|
||||
in_string = False
|
||||
escape = False
|
||||
for i in range(start, len(raw)):
|
||||
ch = raw[i]
|
||||
if escape:
|
||||
escape = False
|
||||
continue
|
||||
if in_string:
|
||||
if ch == "\\":
|
||||
escape = True
|
||||
elif ch == '"':
|
||||
in_string = False
|
||||
continue
|
||||
if ch == '"':
|
||||
in_string = True
|
||||
elif ch == "{":
|
||||
depth += 1
|
||||
elif ch == "}":
|
||||
depth -= 1
|
||||
if depth == 0:
|
||||
return i + 1
|
||||
return None
|
||||
|
||||
|
||||
def _args_value_span(raw: str) -> str | None:
|
||||
"""Extract the raw text span of the top-level ``"args"`` value from a
|
||||
(possibly incomplete) ``{"name":...,"args":{...}}`` wrapper.
|
||||
|
||||
Returns the verbatim substring (prefix-stable across growing input,
|
||||
which the engine's argument-delta diffing relies on), possibly an
|
||||
unterminated object prefix; ``None`` when the value has not started.
|
||||
Raises ``ValueError`` when the value is not a JSON object.
|
||||
"""
|
||||
depth = 0
|
||||
in_string = False
|
||||
escape = False
|
||||
string_start = -1
|
||||
last_string: str | None = None
|
||||
for i, ch in enumerate(raw):
|
||||
if escape:
|
||||
escape = False
|
||||
continue
|
||||
if in_string:
|
||||
if ch == "\\":
|
||||
escape = True
|
||||
elif ch == '"':
|
||||
in_string = False
|
||||
if depth == 1:
|
||||
last_string = raw[string_start + 1 : i]
|
||||
continue
|
||||
if ch == '"':
|
||||
in_string = True
|
||||
string_start = i
|
||||
elif ch == ":" and depth == 1 and last_string == "args":
|
||||
value_start = i + 1
|
||||
while value_start < len(raw) and raw[value_start] in _WS:
|
||||
value_start += 1
|
||||
if value_start >= len(raw):
|
||||
return None
|
||||
if raw[value_start] != "{":
|
||||
raise ValueError("Inkling tool call args must be a JSON object")
|
||||
value_end = _scan_json_value(raw, value_start)
|
||||
if value_end is None:
|
||||
return raw[value_start:]
|
||||
return raw[value_start:value_end]
|
||||
elif ch in "{[":
|
||||
depth += 1
|
||||
elif ch in "}]":
|
||||
depth -= 1
|
||||
return None
|
||||
|
||||
|
||||
def _inkling_arg_converter(raw_args: str, partial: bool) -> str:
|
||||
"""Carve the ``args`` object out of the tool-call JSON wrapper.
|
||||
@@ -150,19 +73,10 @@ def _inkling_arg_converter(raw_args: str, partial: bool) -> str:
|
||||
converter, ``_compute_arg_delta`` streams the wrapper verbatim into
|
||||
the OpenAI ``arguments`` field (``converter is None -> raw delta``,
|
||||
unconditionally; ``stream_arg_deltas=False`` does not stop it).
|
||||
|
||||
Why a hand-rolled scanner instead of (partial) ``json.loads`` +
|
||||
``json.dumps``: the engine diffs successive converter outputs and
|
||||
requires each to extend the previous one (``startswith``); a
|
||||
violation silently drops argument deltas. Re-serialization changes
|
||||
whitespace and closes unterminated structures differently across
|
||||
ticks, so the only prefix-stable output is a verbatim substring of
|
||||
the input. The scanner is also string/escape-aware so an ``"args"``
|
||||
literal inside the name or a string value cannot mislead it, and it
|
||||
still recovers a partial span when EOS truncates the wrapper (where
|
||||
``json.loads`` would fail).
|
||||
:func:`extract_json_member_object` returns a prefix-stable verbatim
|
||||
substring, which the engine's argument-delta diffing requires.
|
||||
"""
|
||||
span = _args_value_span(raw_args)
|
||||
span = extract_json_member_object(raw_args, "args")
|
||||
if span is None:
|
||||
# No args value yet (streaming) or none at all (treat as empty).
|
||||
return "" if partial else "{}"
|
||||
|
||||
@@ -63,3 +63,88 @@ def count_history_tool_calls(
|
||||
return 0
|
||||
|
||||
return count_response_history_tool_calls(request_input)
|
||||
|
||||
|
||||
_JSON_WS = " \t\r\n"
|
||||
|
||||
|
||||
def scan_json_object_end(raw: str, start: int) -> int | None:
|
||||
"""Return the end index (exclusive) of the JSON object starting at
|
||||
``raw[start]`` (which must be ``{``), or ``None`` when it is still
|
||||
unterminated. String- and escape-aware so braces inside strings do not
|
||||
mislead the depth count."""
|
||||
depth = 0
|
||||
in_string = False
|
||||
escape = False
|
||||
for i in range(start, len(raw)):
|
||||
ch = raw[i]
|
||||
if escape:
|
||||
escape = False
|
||||
continue
|
||||
if in_string:
|
||||
if ch == "\\":
|
||||
escape = True
|
||||
elif ch == '"':
|
||||
in_string = False
|
||||
continue
|
||||
if ch == '"':
|
||||
in_string = True
|
||||
elif ch == "{":
|
||||
depth += 1
|
||||
elif ch == "}":
|
||||
depth -= 1
|
||||
if depth == 0:
|
||||
return i + 1
|
||||
return None
|
||||
|
||||
|
||||
def extract_json_member_object(raw: str, key: str) -> str | None:
|
||||
"""Extract the raw text span of the top-level object-valued member ``key``
|
||||
from a (possibly incomplete) ``{...}`` JSON wrapper.
|
||||
|
||||
Returns the verbatim substring (prefix-stable as the input grows, which
|
||||
the engine's argument-delta diffing relies on), possibly an unterminated
|
||||
object prefix; ``None`` when the value has not started. Raises
|
||||
``ValueError`` when the member is present but its value is not a JSON
|
||||
object. A verbatim substring (not re-serialized JSON) is required because
|
||||
the engine diffs successive converter outputs and expects each to extend
|
||||
the previous one; the scanner is string/escape-aware so a matching literal
|
||||
inside another value cannot mislead it, and it recovers a partial span when
|
||||
the wrapper is truncated (where ``json.loads`` would fail)."""
|
||||
depth = 0
|
||||
in_string = False
|
||||
escape = False
|
||||
string_start = -1
|
||||
last_string: str | None = None
|
||||
for i, ch in enumerate(raw):
|
||||
if escape:
|
||||
escape = False
|
||||
continue
|
||||
if in_string:
|
||||
if ch == "\\":
|
||||
escape = True
|
||||
elif ch == '"':
|
||||
in_string = False
|
||||
if depth == 1:
|
||||
last_string = raw[string_start + 1 : i]
|
||||
continue
|
||||
if ch == '"':
|
||||
in_string = True
|
||||
string_start = i
|
||||
elif ch == ":" and depth == 1 and last_string == key:
|
||||
value_start = i + 1
|
||||
while value_start < len(raw) and raw[value_start] in _JSON_WS:
|
||||
value_start += 1
|
||||
if value_start >= len(raw):
|
||||
return None
|
||||
if raw[value_start] != "{":
|
||||
raise ValueError(f"member {key!r} must be a JSON object")
|
||||
value_end = scan_json_object_end(raw, value_start)
|
||||
if value_end is None:
|
||||
return raw[value_start:]
|
||||
return raw[value_start:value_end]
|
||||
elif ch in "{[":
|
||||
depth += 1
|
||||
elif ch in "}]":
|
||||
depth -= 1
|
||||
return None
|
||||
|
||||
@@ -75,8 +75,8 @@ _TOOL_PARSERS_TO_REGISTER = {
|
||||
"Granite20bFCToolParser",
|
||||
),
|
||||
"granite": (
|
||||
"granite_tool_parser",
|
||||
"GraniteToolParser",
|
||||
"granite_engine_tool_parser",
|
||||
"GraniteEngineToolParser",
|
||||
),
|
||||
"granite4": (
|
||||
"granite4_tool_parser",
|
||||
|
||||
@@ -0,0 +1,8 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from vllm.parser.engine.registered_adapters import GraniteParserToolAdapter
|
||||
|
||||
|
||||
class GraniteEngineToolParser(GraniteParserToolAdapter): # type: ignore[valid-type, misc]
|
||||
structural_tag_model = None
|
||||
@@ -1,257 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import json
|
||||
from collections.abc import Sequence
|
||||
|
||||
import partial_json_parser
|
||||
from partial_json_parser.core.options import Allow
|
||||
|
||||
from vllm.entrypoints.chat_utils import make_tool_call_id
|
||||
from vllm.entrypoints.generate.base.protocol import (
|
||||
DeltaFunctionCall,
|
||||
DeltaMessage,
|
||||
DeltaToolCall,
|
||||
ExtractedToolCallInformation,
|
||||
FunctionCall,
|
||||
ToolCall,
|
||||
)
|
||||
from vllm.entrypoints.openai.chat_completion.protocol import (
|
||||
ChatCompletionRequest,
|
||||
)
|
||||
from vllm.logger import init_logger
|
||||
from vllm.tokenizers import TokenizerLike
|
||||
from vllm.tool_parsers.abstract_tool_parser import (
|
||||
Tool,
|
||||
ToolParser,
|
||||
)
|
||||
from vllm.tool_parsers.utils import (
|
||||
consume_space,
|
||||
find_common_prefix,
|
||||
is_complete_json,
|
||||
partial_json_loads,
|
||||
)
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
class GraniteToolParser(ToolParser):
|
||||
"""Tool call parser for the granite 3.0 models. Intended
|
||||
for use with the examples/tool_chat_template_granite.jinja
|
||||
template.
|
||||
|
||||
Used when --enable-auto-tool-choice --tool-call-parser granite
|
||||
are all set
|
||||
"""
|
||||
|
||||
def __init__(self, tokenizer: TokenizerLike, tools: list[Tool] | None = None):
|
||||
super().__init__(tokenizer, tools)
|
||||
# for granite 3.0, the token `<|tool_call|>`
|
||||
self.bot_token = "<|tool_call|>"
|
||||
# for granite 3.1, the string `<tool_call>`
|
||||
self.bot_string = "<tool_call>"
|
||||
|
||||
def extract_tool_calls(
|
||||
self, model_output: str, request: ChatCompletionRequest
|
||||
) -> ExtractedToolCallInformation:
|
||||
stripped = (
|
||||
model_output.strip()
|
||||
.removeprefix(self.bot_token)
|
||||
.removeprefix(self.bot_string)
|
||||
.lstrip()
|
||||
)
|
||||
if not stripped or stripped[0] != "[":
|
||||
return ExtractedToolCallInformation(
|
||||
tools_called=False, tool_calls=[], content=model_output
|
||||
)
|
||||
try:
|
||||
raw_function_calls = json.loads(stripped)
|
||||
if not isinstance(raw_function_calls, list):
|
||||
raise Exception(
|
||||
f"Expected dict or list, got {type(raw_function_calls)}"
|
||||
)
|
||||
|
||||
logger.debug("Extracted %d tool calls", len(raw_function_calls))
|
||||
tool_calls = [
|
||||
ToolCall(
|
||||
type="function",
|
||||
function=FunctionCall(
|
||||
name=function_call["name"],
|
||||
# function call args are JSON but as a string
|
||||
arguments=json.dumps(
|
||||
function_call["arguments"], ensure_ascii=False
|
||||
),
|
||||
),
|
||||
)
|
||||
for function_call in raw_function_calls
|
||||
]
|
||||
|
||||
return ExtractedToolCallInformation(
|
||||
tools_called=True,
|
||||
tool_calls=tool_calls,
|
||||
content=None,
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
logger.error("Error in extracting tool call from response %s", e)
|
||||
return ExtractedToolCallInformation(
|
||||
tools_called=False, tool_calls=[], content=model_output
|
||||
)
|
||||
|
||||
def extract_tool_calls_streaming(
|
||||
self,
|
||||
previous_text: str,
|
||||
current_text: str,
|
||||
delta_text: str,
|
||||
previous_token_ids: Sequence[int],
|
||||
current_token_ids: Sequence[int],
|
||||
delta_token_ids: Sequence[int],
|
||||
request: ChatCompletionRequest,
|
||||
) -> DeltaMessage | None:
|
||||
start_idx = consume_space(0, current_text)
|
||||
if current_text[start_idx:].startswith(self.bot_token):
|
||||
start_idx = consume_space(start_idx + len(self.bot_token), current_text)
|
||||
if current_text[start_idx:].startswith(self.bot_string):
|
||||
start_idx = consume_space(start_idx + len(self.bot_string), current_text)
|
||||
if (
|
||||
not current_text
|
||||
or start_idx >= len(current_text)
|
||||
or current_text[start_idx] != "["
|
||||
):
|
||||
return DeltaMessage(content=delta_text)
|
||||
|
||||
# bit mask flags for partial JSON parsing. If the name hasn't been
|
||||
# sent yet, don't allow sending
|
||||
# an incomplete string since OpenAI only ever (as far as I have
|
||||
# seen) allows sending the entire tool/ function name at once.
|
||||
flags = Allow.ALL if self.current_tool_name_sent else Allow.ALL & ~Allow.STR
|
||||
try:
|
||||
tool_call_arr = None
|
||||
is_complete = None
|
||||
try:
|
||||
tool_calls, end_idx = partial_json_loads(
|
||||
current_text[start_idx:], flags
|
||||
)
|
||||
if type(tool_calls) is list:
|
||||
tool_call_arr = tool_calls
|
||||
else:
|
||||
return DeltaMessage(content=delta_text)
|
||||
|
||||
is_complete = [True] * len(tool_calls)
|
||||
if not is_complete_json(current_text[start_idx : start_idx + end_idx]):
|
||||
is_complete[-1] = False
|
||||
except partial_json_parser.core.exceptions.MalformedJSON:
|
||||
logger.debug("not enough tokens to parse into JSON yet")
|
||||
return None
|
||||
|
||||
# case -- if no tokens have been streamed for the tool, e.g.
|
||||
# only the array brackets, stream nothing
|
||||
if not tool_call_arr:
|
||||
return None
|
||||
|
||||
# select as the current tool call the one we're on the state at
|
||||
current_tool_call: dict = tool_call_arr[self.current_tool_id]
|
||||
|
||||
delta = None
|
||||
# Only advance once the current tool name is streamed; granite
|
||||
# emits arguments before name, so advancing early would drop it.
|
||||
if len(tool_call_arr) > self.current_tool_id + 1 and (
|
||||
self.current_tool_id < 0 or self.current_tool_name_sent
|
||||
):
|
||||
# if we're moving on to a new call, first make sure we
|
||||
# haven't missed anything in the previous one that was
|
||||
# auto-generated due to JSON completions, but wasn't
|
||||
# streamed to the client yet.
|
||||
if self.current_tool_id >= 0:
|
||||
cur_arguments = current_tool_call.get("arguments")
|
||||
if cur_arguments:
|
||||
cur_args_json = json.dumps(cur_arguments, ensure_ascii=False)
|
||||
sent = len(self.streamed_args_for_tool[self.current_tool_id])
|
||||
argument_diff = cur_args_json[sent:]
|
||||
|
||||
logger.debug("got arguments diff: %s", argument_diff)
|
||||
delta = DeltaMessage(
|
||||
tool_calls=[
|
||||
DeltaToolCall(
|
||||
index=self.current_tool_id,
|
||||
function=DeltaFunctionCall(
|
||||
arguments=argument_diff
|
||||
).model_dump(exclude_none=True),
|
||||
)
|
||||
]
|
||||
)
|
||||
self.streamed_args_for_tool[self.current_tool_id] += (
|
||||
argument_diff
|
||||
)
|
||||
|
||||
# re-set stuff pertaining to progress in the current tool
|
||||
self.current_tool_id += 1
|
||||
self.current_tool_name_sent = False
|
||||
self.streamed_args_for_tool.append("")
|
||||
logger.debug("starting on new tool %d", self.current_tool_id)
|
||||
return delta
|
||||
|
||||
# if the current tool name hasn't been sent, send if available
|
||||
# - otherwise send nothing
|
||||
elif not self.current_tool_name_sent:
|
||||
function_name = current_tool_call.get("name")
|
||||
if function_name:
|
||||
delta = DeltaMessage(
|
||||
tool_calls=[
|
||||
DeltaToolCall(
|
||||
index=self.current_tool_id,
|
||||
type="function",
|
||||
id=make_tool_call_id(),
|
||||
function=DeltaFunctionCall(
|
||||
name=function_name
|
||||
).model_dump(exclude_none=True),
|
||||
)
|
||||
]
|
||||
)
|
||||
self.current_tool_name_sent = True
|
||||
|
||||
# now we know we're on the same tool call and we're streaming
|
||||
# arguments
|
||||
else:
|
||||
cur_arguments = current_tool_call.get("arguments")
|
||||
|
||||
if cur_arguments:
|
||||
sent = len(self.streamed_args_for_tool[self.current_tool_id])
|
||||
cur_args_json = json.dumps(cur_arguments, ensure_ascii=False)
|
||||
prev_arguments = self.prev_tool_call_arr[self.current_tool_id].get(
|
||||
"arguments"
|
||||
)
|
||||
|
||||
argument_diff = None
|
||||
if is_complete[self.current_tool_id]:
|
||||
argument_diff = cur_args_json[sent:]
|
||||
elif prev_arguments:
|
||||
prev_args_json = json.dumps(prev_arguments, ensure_ascii=False)
|
||||
if cur_args_json != prev_args_json:
|
||||
prefix = find_common_prefix(prev_args_json, cur_args_json)
|
||||
argument_diff = prefix[sent:]
|
||||
|
||||
if argument_diff is not None:
|
||||
delta = DeltaMessage(
|
||||
tool_calls=[
|
||||
DeltaToolCall(
|
||||
index=self.current_tool_id,
|
||||
function=DeltaFunctionCall(
|
||||
arguments=argument_diff
|
||||
).model_dump(exclude_none=True),
|
||||
)
|
||||
]
|
||||
)
|
||||
self.streamed_args_for_tool[self.current_tool_id] += (
|
||||
argument_diff
|
||||
)
|
||||
|
||||
self.prev_tool_call_arr = tool_call_arr
|
||||
return delta
|
||||
|
||||
except Exception as e:
|
||||
logger.error("Error trying to handle streaming tool call: %s", e)
|
||||
logger.debug(
|
||||
"Skipping chunk as a result of tool streaming extraction error"
|
||||
)
|
||||
return None
|
||||
Reference in New Issue
Block a user