[Bugfix][Tool Parser] Migrate Granite to the streaming Parser Engine (#49648)

Signed-off-by: Nikhil Kulkarni <nikhilkulkarni1755@gmail.com>
Co-authored-by: Claude Opus 4.8 <noreply@anthropic.com>
Co-authored-by: Chauncey <chaunceyjiang@gmail.com>
This commit is contained in:
Nikhil Kulkarni
2026-09-23 10:37:40 +00:00
committed by GitHub
co-authored by Claude Opus 4.8 Chauncey
parent c961121519
commit 15ed1262e7
14 changed files with 631 additions and 369 deletions
@@ -99,6 +99,10 @@ def _discover_pairings() -> list[_PairingInfo]:
# terminal matching; combined-parser replay coverage lives in
# test_inkling.py.
continue
if cfg.name == "granite":
# Granite has a JSON-array tool body with no TOOL_END terminal;
# its replay coverage lives in test_granite.py.
continue
parser_cls = type(
f"_Delegating{engine_cls.__name__}",
+234
View File
@@ -0,0 +1,234 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests for the engine-based Granite parser.
Granite emits tool calls as a JSON array following a marker
(``<|tool_call|>`` for 3.0, ``<tool_call>`` for 3.1) with no closing
terminal; the ``]`` ends the region and surrounding prose is content. The
engine's ``tool_call_body_array`` mode splits the array into one call per
element and ``_granite_arg_converter`` carves each element's ``arguments``.
"""
import json
import pytest
from tests.parser.engine.conftest import make_mock_tokenizer
from tests.parser.engine.trace_builder import _GRANITE_SCENARIOS, _build_granite
from vllm.parser.granite import GraniteParser, _granite_arg_converter
TOOL_TOKEN = "<|tool_call|>"
TOOL_STRING = "<tool_call>"
_GRANITE_VOCAB = {TOOL_TOKEN: 49154}
@pytest.fixture
def mock_tokenizer():
return make_mock_tokenizer(_GRANITE_VOCAB)
@pytest.fixture
def parser(mock_tokenizer):
return GraniteParser(mock_tokenizer)
def _tokenize(text: str) -> list[tuple[int, str]]:
"""Tokenize like the real stream: ``<|tool_call|>`` is one special token,
plain text becomes one token per character."""
tokens: list[tuple[int, str]] = []
i = 0
while i < len(text):
if text.startswith(TOOL_TOKEN, i):
tokens.append((_GRANITE_VOCAB[TOOL_TOKEN], TOOL_TOKEN))
i += len(TOOL_TOKEN)
else:
tokens.append((ord(text[i]), text[i]))
i += 1
return tokens
def _stream(parser, request, text: str, chunk_size: int):
tokens = _tokenize(text)
results = []
previous_text = ""
previous_token_ids: list[int] = []
for start in range(0, len(tokens), chunk_size):
batch = tokens[start : start + chunk_size]
delta_text = "".join(t for _, t in batch)
delta_token_ids = [tid for tid, _ in batch]
current_text = previous_text + delta_text
current_token_ids = previous_token_ids + delta_token_ids
delta = parser.extract_tool_calls_streaming(
previous_text=previous_text,
current_text=current_text,
delta_text=delta_text,
previous_token_ids=tuple(previous_token_ids),
current_token_ids=tuple(current_token_ids),
delta_token_ids=tuple(delta_token_ids),
request=request,
)
results.append(delta)
previous_text = current_text
previous_token_ids = current_token_ids
finish = parser.finish_streaming()
if finish is not None:
results.append(finish)
return results
def _collect_content(results) -> str:
return "".join(d.content for d in results if d and d.content)
def _collect_names(results) -> list[str]:
names: dict[int, str] = {}
for d in results:
if not (d and d.tool_calls):
continue
for tc in d.tool_calls:
if tc.function and tc.function.name:
names[tc.index] = names.get(tc.index, "") + tc.function.name
return [names[i] for i in sorted(names)]
def _collect_args(results) -> dict[int, str]:
args: dict[int, str] = {}
for d in results:
if not (d and d.tool_calls):
continue
for tc in d.tool_calls:
if tc.function and tc.function.arguments:
args[tc.index] = args.get(tc.index, "") + tc.function.arguments
return args
class TestArgConverter:
def test_complete_wrapper(self):
raw = '{"name": "get_weather", "arguments": {"city": "SF"}}'
assert _granite_arg_converter(raw, False) == '{"city": "SF"}'
def test_arguments_before_name(self):
raw = '{"arguments": {"city": "SF"}, "name": "get_weather"}'
assert _granite_arg_converter(raw, False) == '{"city": "SF"}'
def test_partial_before_arguments(self):
assert _granite_arg_converter('{"name": "get_w', True) == ""
def test_partial_inside_arguments(self):
raw = '{"name": "x", "arguments": {"a": 1'
assert _granite_arg_converter(raw, True) == '{"a": 1'
def test_prefix_stability(self):
# Each growing prefix must extend the previous converter output.
full = '{"name": "x", "arguments": {"city": "Tokyo"}}'
prev = ""
for i in range(1, len(full) + 1):
out = _granite_arg_converter(full[:i], True)
assert out.startswith(prev) or prev.startswith(out) or out == ""
if out:
prev = out
def test_missing_arguments_defaults(self):
assert _granite_arg_converter('{"name": "x"}', False) == "{}"
assert _granite_arg_converter('{"name": "x"}', True) == ""
def test_non_object_arguments_rejected(self):
with pytest.raises(ValueError, match="JSON object"):
_granite_arg_converter('{"name": "x", "arguments": [1]}', False)
class TestNonStreaming:
def test_plain_text(self, parser, mock_request):
reasoning, content, tools = parser.parse("Just a reply.", mock_request)
assert reasoning is None
assert content == "Just a reply."
assert tools is None
def test_single_tool_token_marker(self, parser, mock_request):
text = (
f'{TOOL_TOKEN} [{{"name": "get_weather", "arguments": {{"city": "SF"}}}}]'
)
_, content, tools = parser.parse(text, mock_request)
assert content is None
assert [t.name for t in tools] == ["get_weather"]
assert json.loads(tools[0].arguments) == {"city": "SF"}
def test_single_tool_string_marker(self, parser, mock_request):
# Granite 3.1 uses the plain-text ``<tool_call>`` marker.
text = f'{TOOL_STRING} [{{"name": "get_time", "arguments": {{}}}}]'
_, content, tools = parser.parse(text, mock_request)
assert [t.name for t in tools] == ["get_time"]
assert json.loads(tools[0].arguments) == {}
def test_parallel_calls_one_array(self, parser, mock_request):
text = (
f'{TOOL_TOKEN} [{{"name": "a", "arguments": {{"x": 1}}}}, '
f'{{"name": "b", "arguments": {{"y": [1, 2]}}}}]'
)
_, _, tools = parser.parse(text, mock_request)
assert [t.name for t in tools] == ["a", "b"]
assert json.loads(tools[0].arguments) == {"x": 1}
assert json.loads(tools[1].arguments) == {"y": [1, 2]}
def test_surrounding_text_is_content(self, parser, mock_request):
text = (
"Let me check.\n"
f'{TOOL_TOKEN} [{{"name": "get_weather", "arguments": {{"city": "SF"}}}}]'
)
_, content, tools = parser.parse(text, mock_request)
assert content == "Let me check."
assert [t.name for t in tools] == ["get_weather"]
def test_arguments_before_name(self, parser, mock_request):
text = (
f'{TOOL_TOKEN} [{{"arguments": {{"city": "SF"}}, "name": "get_weather"}}]'
)
_, _, tools = parser.parse(text, mock_request)
assert [t.name for t in tools] == ["get_weather"]
assert json.loads(tools[0].arguments) == {"city": "SF"}
def test_marker_without_array_is_not_a_tool_call(self, parser, mock_request):
# A non-array body after the marker (malformed) yields no tool call.
text = f'{TOOL_TOKEN} {{"name": "func", "arguments": {{}}}}'
_, _, tools = parser.parse(text, mock_request)
assert not tools
class TestStreaming:
@pytest.mark.parametrize("chunk_size", [1, 2, 3, 5, 11])
def test_parallel_calls_chunk_invariance(self, parser, mock_request, chunk_size):
text = (
f'{TOOL_TOKEN} [{{"name": "get_weather", '
f'"arguments": {{"city": "Tokyo"}}}}, '
f'{{"name": "get_time", "arguments": {{"timezone": "Asia/Tokyo"}}}}]'
)
results = _stream(parser, mock_request, text, chunk_size)
assert _collect_names(results) == ["get_weather", "get_time"]
args = _collect_args(results)
assert json.loads(args[0]) == {"city": "Tokyo"}
assert json.loads(args[1]) == {"timezone": "Asia/Tokyo"}
@pytest.mark.parametrize("chunk_size", [1, 3, 7])
def test_surrounding_text_streaming(self, parser, mock_request, chunk_size):
text = (
"Let me check.\n"
f'{TOOL_TOKEN} [{{"name": "get_weather", "arguments": {{"city": "SF"}}}}]'
)
results = _stream(parser, mock_request, text, chunk_size)
assert _collect_names(results) == ["get_weather"]
assert "Let me check." in _collect_content(results)
def test_incomplete_array_at_eos(self, parser, mock_request):
# Truncated mid-element must not raise and must not leak the marker.
text = f'{TOOL_TOKEN} [{{"name": "func", "arguments": {{"a": 1'
results = _stream(parser, mock_request, text, 3)
assert TOOL_TOKEN not in _collect_content(results)
class TestReplay:
@pytest.mark.parametrize("scenario", _GRANITE_SCENARIOS, ids=lambda s: s.id)
def test_replay_scenarios(self, scenario):
# ``_build_granite`` replays the sample through GraniteParser at
# chunk_size=1 and asserts reasoning/content/tool_calls (validate=True).
_build_granite(scenario)
+5
View File
@@ -78,6 +78,11 @@ def _discover_parsers() -> list[_ParserInfo]:
# Inkling opts out of token-id terminal matching and has typed
# structural blocks; its replay coverage lives in test_inkling.py.
continue
if cfg.name == "granite":
# Granite has a JSON-array tool body with no TOOL_END terminal, so
# it does not fit this token-terminal harness; its replay coverage
# lives in test_granite.py.
continue
tool_end = cfg.token_id_terminals.get("TOOL_END")
if not tool_end:
raise RuntimeError(
+85
View File
@@ -34,6 +34,7 @@ from vllm.parser.engine.registered_adapters import (
DeepSeekV41Parser,
Gemma4Parser,
Glm47MoeParser,
GraniteParser,
InklingParser,
KimiK2Parser,
MinimaxM2Parser,
@@ -1042,6 +1043,89 @@ def _build_inkling(scenario: Scenario, validate: bool = True) -> Sample:
return sample
# ── Granite (JSON-array tool bodies, no reasoning) ───────────────────────
_GRANITE_VOCAB: dict[str, int] = {
"<|tool_call|>": 49154,
}
def _granite_segments(scenario: Scenario) -> list[tuple[str, bool]]:
segs: list[tuple[str, bool]] = []
if scenario.content:
# Granite has no reasoning; prose is plain content preceding the marker.
segs.append((scenario.content, False))
if scenario.tool_calls:
segs.append(("<|tool_call|>", True))
payload = json.dumps(
[
{"name": tc.name, "arguments": tc.arguments}
for tc in scenario.tool_calls
],
ensure_ascii=False,
separators=(", ", ": "),
)
segs.append((" " + payload, False))
return segs
def _granite_expected_content(scenario: Scenario) -> str | None:
if scenario.tool_calls:
if not scenario.content:
return None
return scenario.content.strip() or None
return scenario.content
def _build_granite(scenario: Scenario, validate: bool = True) -> Sample:
sample = _make_sample(
sample_id=f"granite-{scenario.id}",
description=scenario.description,
vocab=_GRANITE_VOCAB,
segments=_granite_segments(scenario),
expected_reasoning=None,
expected_content=_granite_expected_content(scenario),
expected_tool_calls=_expected_tc(scenario),
tools=_expected_tools(scenario),
)
if validate:
_validate_sample(sample, GraniteParser)
return sample
# Granite has no reasoning, so the shared reasoning-centric SCENARIOS do not
# apply; these exercise the JSON-array tool body (single, parallel, surrounding
# text) instead.
_GRANITE_SCENARIOS: list[Scenario] = [
Scenario(
id="single-tool",
description="Single tool call",
tool_calls=[_READ_TOOL],
),
Scenario(
id="parallel-tools",
description="Parallel tool calls in one JSON array",
tool_calls=[_BASH_TOOL, _WEATHER_TOOL],
),
Scenario(
id="complex-json-args",
description="Tool call with nested objects, arrays, numbers, booleans",
tool_calls=[_COMPLEX_TOOL],
),
Scenario(
id="content-only",
description="Plain content response without tool calls",
content="Hello! How can I help you today?",
),
Scenario(
id="surrounding-text",
description="Prose content preceding the tool call",
content="Let me check the weather.",
tool_calls=[_WEATHER_TOOL],
),
]
# ── Registry and public API ──────────────────────────────────────────
_BUILDERS: dict[str, Any] = {
@@ -1049,6 +1133,7 @@ _BUILDERS: dict[str, Any] = {
"deepseek_v4": _build_deepseek_v4,
"deepseek_v41": functools.partial(_build_deepseek_v4, v41=True),
"gemma4": _build_gemma4,
"granite": _build_granite,
"minimax_m2": _build_minimax_m2,
"nemotron_v3": _build_nemotron_v3,
"seed_oss": _build_seed_oss,
+2 -19
View File
@@ -16,7 +16,7 @@ from tests.tool_parsers.utils import (
split_string_into_token_deltas,
)
from vllm.tokenizers import get_tokenizer
from vllm.tool_parsers.granite_tool_parser import GraniteToolParser
from vllm.tool_parsers.granite_engine_tool_parser import GraniteEngineToolParser
class TestGraniteToolParser(ToolParserTests):
@@ -76,23 +76,6 @@ I'll get that information.""",
single_tool_call_expected_content=None,
parallel_tool_calls_count=2,
parallel_tool_calls_names=["get_weather", "get_time"],
# xfail markers
xfail_streaming={
"test_malformed_input": (
"Streaming mode incorrectly creates tool call from malformed JSON"
),
"test_surrounding_text": (
"Parser doesn't handle surrounding text correctly in streaming"
),
"test_streaming_reconstruction": (
"Streaming mode doesn't strip <|tool_call|> marker from content"
),
},
xfail_nonstreaming={
"test_surrounding_text": (
"Parser doesn't handle surrounding text correctly in non-streaming"
),
},
)
# Granite-Specific Tests
@@ -140,7 +123,7 @@ def test_streaming_parallel_calls_batched_deltas(granite_tokenizer, chunk_size):
parallel calls must not drop the first call's name. granite streams
arguments before name, so the name only completes as the next call appears.
"""
parser = GraniteToolParser(granite_tokenizer)
parser = GraniteEngineToolParser(granite_tokenizer)
model_output = (
'<|tool_call|> [{"arguments": {"city": "Tokyo"}, "name": "get_weather"}, '
'{"arguments": {"timezone": "Asia/Tokyo"}, "name": "get_time"}]'
@@ -82,6 +82,12 @@ class ParserEngineConfig:
tool_args_json: bool = True
# The tool region body is a single JSON array of ``{"name", "arguments"}``
# objects (one marker → N calls, no closing terminal). The engine splits
# the array into one tool call per element; the array's ``]`` ends the
# region and any trailing text becomes content. Used by Granite.
tool_call_body_array: bool = False
arg_structural_chars: frozenset[str] | None = None
# Special tokens exempt from auto-drop but not state-machine terminals.
@@ -13,6 +13,7 @@ from vllm.parser.deepseek_v41 import DeepSeekV41Parser
from vllm.parser.engine.adapters import make_adapters
from vllm.parser.gemma4 import Gemma4Parser
from vllm.parser.glm47_moe import Glm47MoeParser
from vllm.parser.granite import GraniteParser
from vllm.parser.inkling import InklingParser
from vllm.parser.kimi_k2 import KimiK2Parser
from vllm.parser.minimax_m2 import MinimaxM2Parser
@@ -46,6 +47,11 @@ from vllm.parser.seed_oss import SeedOssParser
Gemma4ParserToolAdapter,
) = make_adapters(Gemma4Parser)
(
GraniteParserReasoningAdapter,
GraniteParserToolAdapter,
) = make_adapters(GraniteParser)
(
NemotronV3ParserReasoningAdapter,
NemotronV3ParserToolAdapter,
+102 -1
View File
@@ -190,6 +190,12 @@ class StreamingParserEngine:
self._args_in_string: bool = False
self._args_escape_next: bool = False
def _reset_array_state(self) -> None:
# State for ``tool_call_body_array``: whether the opening ``[`` has been
# seen, and whether a ``{...}`` element is currently being streamed.
self._array_started: bool = False
self._array_in_element: bool = False
def reset(self, initial_state: ParserState | None = None) -> None:
"""Reset mutable state for reuse across requests.
@@ -213,6 +219,7 @@ class StreamingParserEngine:
self._message_header_token_count = 0
self._in_skipped_tool_span = False
self._reset_args_state()
self._reset_array_state()
def feed(
self,
@@ -306,7 +313,12 @@ class StreamingParserEngine:
ParserState.TOOL_NAME,
ParserState.TOOL_BETWEEN,
):
if self.tool_index >= 0:
# In array mode a closed element has already emitted its
# TOOL_CALL_END; only a truncated (still-open) element needs one.
end_open_call = (
self._array_in_element if self.config.tool_call_body_array else True
)
if end_open_call and self.tool_index >= 0:
events.append(
SemanticEvent(
EventType.TOOL_CALL_END,
@@ -474,6 +486,8 @@ class StreamingParserEngine:
self._message_header_token_count += token_count
return []
if self.state == ParserState.TOOL_ARGS:
if self.config.tool_call_body_array:
return self._feed_array_text(text)
if self.config.tool_args_json:
return self._feed_args_text(text)
return [
@@ -564,6 +578,8 @@ class StreamingParserEngine:
self._args_in_string = False
self._args_escape_next = False
self._args_safe_end = 0
if self.config.tool_call_body_array:
self._reset_array_state()
return events
@@ -634,3 +650,88 @@ class StreamingParserEngine:
tool_index=self.tool_index,
)
]
# ── JSON-array tool body (tool_call_body_array) ─────────────────────
def _feed_array_text(self, text: str) -> list[SemanticEvent]:
"""Split a JSON-array tool body into one tool call per element.
Granite emits ``[{"name":..,"arguments":{..}}, ...]`` after the tool
marker. Element interiors are streamed through the string/brace-aware
:meth:`_feed_args_char`, so each element behaves exactly like a
single-object tool body; the array's ``]`` ends the region and any
trailing text becomes content.
"""
events: list[SemanticEvent] = []
for i, ch in enumerate(text):
if self.state != ParserState.TOOL_ARGS:
# The array closed mid-chunk; the rest of the text is content.
events.extend(self._emit_for_state(text[i:]))
break
events.extend(self._feed_array_char(ch))
return events
def _feed_array_char(self, ch: str) -> list[SemanticEvent]:
if self._array_in_element:
events = self._feed_args_char(ch)
# Element object closed: top-level '}' with all braces balanced.
if (
ch == "}"
and not self._args_in_string
and not self._args_escape_next
and self._args_brace_depth == 0
):
events.extend(self._end_array_element())
return events
# Between elements only structural array characters are expected.
if ch in " \t\r\n":
return []
if not self._array_started:
if ch == "[":
self._array_started = True
return []
return self._bail_array_to_content(ch)
if ch == ",":
return []
if ch == "{":
return self._start_array_element(ch)
if ch == "]":
self.state = ParserState.CONTENT
self._reset_array_state()
return []
return self._bail_array_to_content(ch)
def _start_array_element(self, ch: str) -> list[SemanticEvent]:
self.tool_index += 1
self._reset_args_state()
self._array_in_element = True
events = [SemanticEvent(EventType.TOOL_CALL_START, tool_index=self.tool_index)]
events.extend(self._feed_args_char(ch))
return events
def _end_array_element(self) -> list[SemanticEvent]:
self._array_in_element = False
events: list[SemanticEvent] = []
# Flush the held closing '}' so the slot body is the full element.
if self._args_buffer:
events.append(
SemanticEvent(
EventType.ARG_VALUE_CHUNK,
value=self._args_buffer,
tool_index=self.tool_index,
)
)
self._args_buffer = ""
self._args_safe_end = 0
events.append(
SemanticEvent(EventType.TOOL_CALL_END, tool_index=self.tool_index)
)
return events
def _bail_array_to_content(self, ch: str) -> list[SemanticEvent]:
# Marker not followed by a JSON array (malformed) — revert to content
# so the remaining tokens are emitted as plain text, no tool call.
self.state = ParserState.CONTENT
self._reset_array_state()
return self._emit_for_state(ch)
+88
View File
@@ -0,0 +1,88 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Granite parser for JSON-array tool calls.
Granite 3.0/3.1 emit tool calls as a JSON array following a marker::
<|tool_call|> [{"name": "get_weather", "arguments": {"city": "SF"}}] # 3.0
<tool_call> [{"name": "get_weather", "arguments": {"city": "SF"}}] # 3.1
``<|tool_call|>`` is a single special token; ``<tool_call>`` is plain text.
The array may hold several calls, has no closing marker (the ``]`` ends it),
and any surrounding prose is content. The engine's ``tool_call_body_array``
mode splits the array into one call per element and extracts each name via the
name-from-args path; ``_granite_arg_converter`` carves the ``arguments`` object
out of each ``{"name":..,"arguments":{..}}`` element. Granite has no reasoning.
"""
from __future__ import annotations
import functools
from typing import TYPE_CHECKING
from vllm.parser.engine.parser_engine import ParserEngine
from vllm.parser.engine.parser_engine_config import (
ParserEngineConfig,
ParserState,
Transition,
)
from vllm.parser.utils import extract_json_member_object
if TYPE_CHECKING:
from vllm.tokenizers import TokenizerLike
from vllm.tool_parsers.abstract_tool_parser import Tool
TOOL_TOKEN = "<|tool_call|>" # granite 3.0 (single special token)
TOOL_STRING = "<tool_call>" # granite 3.1 (plain text)
def _granite_arg_converter(raw_args: str, partial: bool) -> str:
"""Carve the ``arguments`` object out of a ``{"name":..,"arguments":{..}}``
array element (see :func:`extract_json_member_object`)."""
span = extract_json_member_object(raw_args, "arguments")
if span is None:
return "" if partial else "{}"
return span
@functools.cache
def granite_config() -> ParserEngineConfig:
return ParserEngineConfig(
name="granite",
initial_state=ParserState.CONTENT,
terminals={
"TOOL_TOKEN": TOOL_TOKEN,
"TOOL_STRING": TOOL_STRING,
},
token_id_terminals={
"TOOL_TOKEN": TOOL_TOKEN,
},
transitions={
# The marker only switches into the tool region; the array splitter
# emits TOOL_CALL_START per element (empty events here).
(ParserState.CONTENT, "TOOL_TOKEN"): Transition(
ParserState.TOOL_ARGS,
(),
),
(ParserState.CONTENT, "TOOL_STRING"): Transition(
ParserState.TOOL_ARGS,
(),
),
},
arg_converter=_granite_arg_converter,
tool_call_body_array=True,
stream_arg_deltas=True,
)
class GraniteParser(ParserEngine):
CONFIG_NAME = "granite"
def __init__(
self,
tokenizer: TokenizerLike,
tools: list[Tool] | None = None,
**kwargs,
) -> None:
kwargs.setdefault("parser_engine_config", granite_config())
super().__init__(tokenizer, tools, **kwargs)
+4 -90
View File
@@ -37,6 +37,7 @@ from vllm.parser.engine.parser_engine_config import (
ParserState,
Transition,
)
from vllm.parser.utils import extract_json_member_object
if TYPE_CHECKING:
from vllm.tokenizers import TokenizerLike
@@ -62,84 +63,6 @@ INKLING_SPECIAL_TOKENS = (
END_MESSAGE,
)
_WS = " \t\r\n"
def _scan_json_value(raw: str, start: int) -> int | None:
"""Return the end index (exclusive) of the JSON object starting at
``raw[start]``, or ``None`` when the object is still unterminated."""
depth = 0
in_string = False
escape = False
for i in range(start, len(raw)):
ch = raw[i]
if escape:
escape = False
continue
if in_string:
if ch == "\\":
escape = True
elif ch == '"':
in_string = False
continue
if ch == '"':
in_string = True
elif ch == "{":
depth += 1
elif ch == "}":
depth -= 1
if depth == 0:
return i + 1
return None
def _args_value_span(raw: str) -> str | None:
"""Extract the raw text span of the top-level ``"args"`` value from a
(possibly incomplete) ``{"name":...,"args":{...}}`` wrapper.
Returns the verbatim substring (prefix-stable across growing input,
which the engine's argument-delta diffing relies on), possibly an
unterminated object prefix; ``None`` when the value has not started.
Raises ``ValueError`` when the value is not a JSON object.
"""
depth = 0
in_string = False
escape = False
string_start = -1
last_string: str | None = None
for i, ch in enumerate(raw):
if escape:
escape = False
continue
if in_string:
if ch == "\\":
escape = True
elif ch == '"':
in_string = False
if depth == 1:
last_string = raw[string_start + 1 : i]
continue
if ch == '"':
in_string = True
string_start = i
elif ch == ":" and depth == 1 and last_string == "args":
value_start = i + 1
while value_start < len(raw) and raw[value_start] in _WS:
value_start += 1
if value_start >= len(raw):
return None
if raw[value_start] != "{":
raise ValueError("Inkling tool call args must be a JSON object")
value_end = _scan_json_value(raw, value_start)
if value_end is None:
return raw[value_start:]
return raw[value_start:value_end]
elif ch in "{[":
depth += 1
elif ch in "}]":
depth -= 1
return None
def _inkling_arg_converter(raw_args: str, partial: bool) -> str:
"""Carve the ``args`` object out of the tool-call JSON wrapper.
@@ -150,19 +73,10 @@ def _inkling_arg_converter(raw_args: str, partial: bool) -> str:
converter, ``_compute_arg_delta`` streams the wrapper verbatim into
the OpenAI ``arguments`` field (``converter is None -> raw delta``,
unconditionally; ``stream_arg_deltas=False`` does not stop it).
Why a hand-rolled scanner instead of (partial) ``json.loads`` +
``json.dumps``: the engine diffs successive converter outputs and
requires each to extend the previous one (``startswith``); a
violation silently drops argument deltas. Re-serialization changes
whitespace and closes unterminated structures differently across
ticks, so the only prefix-stable output is a verbatim substring of
the input. The scanner is also string/escape-aware so an ``"args"``
literal inside the name or a string value cannot mislead it, and it
still recovers a partial span when EOS truncates the wrapper (where
``json.loads`` would fail).
:func:`extract_json_member_object` returns a prefix-stable verbatim
substring, which the engine's argument-delta diffing requires.
"""
span = _args_value_span(raw_args)
span = extract_json_member_object(raw_args, "args")
if span is None:
# No args value yet (streaming) or none at all (treat as empty).
return "" if partial else "{}"
+85
View File
@@ -63,3 +63,88 @@ def count_history_tool_calls(
return 0
return count_response_history_tool_calls(request_input)
_JSON_WS = " \t\r\n"
def scan_json_object_end(raw: str, start: int) -> int | None:
"""Return the end index (exclusive) of the JSON object starting at
``raw[start]`` (which must be ``{``), or ``None`` when it is still
unterminated. String- and escape-aware so braces inside strings do not
mislead the depth count."""
depth = 0
in_string = False
escape = False
for i in range(start, len(raw)):
ch = raw[i]
if escape:
escape = False
continue
if in_string:
if ch == "\\":
escape = True
elif ch == '"':
in_string = False
continue
if ch == '"':
in_string = True
elif ch == "{":
depth += 1
elif ch == "}":
depth -= 1
if depth == 0:
return i + 1
return None
def extract_json_member_object(raw: str, key: str) -> str | None:
"""Extract the raw text span of the top-level object-valued member ``key``
from a (possibly incomplete) ``{...}`` JSON wrapper.
Returns the verbatim substring (prefix-stable as the input grows, which
the engine's argument-delta diffing relies on), possibly an unterminated
object prefix; ``None`` when the value has not started. Raises
``ValueError`` when the member is present but its value is not a JSON
object. A verbatim substring (not re-serialized JSON) is required because
the engine diffs successive converter outputs and expects each to extend
the previous one; the scanner is string/escape-aware so a matching literal
inside another value cannot mislead it, and it recovers a partial span when
the wrapper is truncated (where ``json.loads`` would fail)."""
depth = 0
in_string = False
escape = False
string_start = -1
last_string: str | None = None
for i, ch in enumerate(raw):
if escape:
escape = False
continue
if in_string:
if ch == "\\":
escape = True
elif ch == '"':
in_string = False
if depth == 1:
last_string = raw[string_start + 1 : i]
continue
if ch == '"':
in_string = True
string_start = i
elif ch == ":" and depth == 1 and last_string == key:
value_start = i + 1
while value_start < len(raw) and raw[value_start] in _JSON_WS:
value_start += 1
if value_start >= len(raw):
return None
if raw[value_start] != "{":
raise ValueError(f"member {key!r} must be a JSON object")
value_end = scan_json_object_end(raw, value_start)
if value_end is None:
return raw[value_start:]
return raw[value_start:value_end]
elif ch in "{[":
depth += 1
elif ch in "}]":
depth -= 1
return None
+2 -2
View File
@@ -75,8 +75,8 @@ _TOOL_PARSERS_TO_REGISTER = {
"Granite20bFCToolParser",
),
"granite": (
"granite_tool_parser",
"GraniteToolParser",
"granite_engine_tool_parser",
"GraniteEngineToolParser",
),
"granite4": (
"granite4_tool_parser",
@@ -0,0 +1,8 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from vllm.parser.engine.registered_adapters import GraniteParserToolAdapter
class GraniteEngineToolParser(GraniteParserToolAdapter): # type: ignore[valid-type, misc]
structural_tag_model = None
-257
View File
@@ -1,257 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import json
from collections.abc import Sequence
import partial_json_parser
from partial_json_parser.core.options import Allow
from vllm.entrypoints.chat_utils import make_tool_call_id
from vllm.entrypoints.generate.base.protocol import (
DeltaFunctionCall,
DeltaMessage,
DeltaToolCall,
ExtractedToolCallInformation,
FunctionCall,
ToolCall,
)
from vllm.entrypoints.openai.chat_completion.protocol import (
ChatCompletionRequest,
)
from vllm.logger import init_logger
from vllm.tokenizers import TokenizerLike
from vllm.tool_parsers.abstract_tool_parser import (
Tool,
ToolParser,
)
from vllm.tool_parsers.utils import (
consume_space,
find_common_prefix,
is_complete_json,
partial_json_loads,
)
logger = init_logger(__name__)
class GraniteToolParser(ToolParser):
"""Tool call parser for the granite 3.0 models. Intended
for use with the examples/tool_chat_template_granite.jinja
template.
Used when --enable-auto-tool-choice --tool-call-parser granite
are all set
"""
def __init__(self, tokenizer: TokenizerLike, tools: list[Tool] | None = None):
super().__init__(tokenizer, tools)
# for granite 3.0, the token `<|tool_call|>`
self.bot_token = "<|tool_call|>"
# for granite 3.1, the string `<tool_call>`
self.bot_string = "<tool_call>"
def extract_tool_calls(
self, model_output: str, request: ChatCompletionRequest
) -> ExtractedToolCallInformation:
stripped = (
model_output.strip()
.removeprefix(self.bot_token)
.removeprefix(self.bot_string)
.lstrip()
)
if not stripped or stripped[0] != "[":
return ExtractedToolCallInformation(
tools_called=False, tool_calls=[], content=model_output
)
try:
raw_function_calls = json.loads(stripped)
if not isinstance(raw_function_calls, list):
raise Exception(
f"Expected dict or list, got {type(raw_function_calls)}"
)
logger.debug("Extracted %d tool calls", len(raw_function_calls))
tool_calls = [
ToolCall(
type="function",
function=FunctionCall(
name=function_call["name"],
# function call args are JSON but as a string
arguments=json.dumps(
function_call["arguments"], ensure_ascii=False
),
),
)
for function_call in raw_function_calls
]
return ExtractedToolCallInformation(
tools_called=True,
tool_calls=tool_calls,
content=None,
)
except Exception as e:
logger.error("Error in extracting tool call from response %s", e)
return ExtractedToolCallInformation(
tools_called=False, tool_calls=[], content=model_output
)
def extract_tool_calls_streaming(
self,
previous_text: str,
current_text: str,
delta_text: str,
previous_token_ids: Sequence[int],
current_token_ids: Sequence[int],
delta_token_ids: Sequence[int],
request: ChatCompletionRequest,
) -> DeltaMessage | None:
start_idx = consume_space(0, current_text)
if current_text[start_idx:].startswith(self.bot_token):
start_idx = consume_space(start_idx + len(self.bot_token), current_text)
if current_text[start_idx:].startswith(self.bot_string):
start_idx = consume_space(start_idx + len(self.bot_string), current_text)
if (
not current_text
or start_idx >= len(current_text)
or current_text[start_idx] != "["
):
return DeltaMessage(content=delta_text)
# bit mask flags for partial JSON parsing. If the name hasn't been
# sent yet, don't allow sending
# an incomplete string since OpenAI only ever (as far as I have
# seen) allows sending the entire tool/ function name at once.
flags = Allow.ALL if self.current_tool_name_sent else Allow.ALL & ~Allow.STR
try:
tool_call_arr = None
is_complete = None
try:
tool_calls, end_idx = partial_json_loads(
current_text[start_idx:], flags
)
if type(tool_calls) is list:
tool_call_arr = tool_calls
else:
return DeltaMessage(content=delta_text)
is_complete = [True] * len(tool_calls)
if not is_complete_json(current_text[start_idx : start_idx + end_idx]):
is_complete[-1] = False
except partial_json_parser.core.exceptions.MalformedJSON:
logger.debug("not enough tokens to parse into JSON yet")
return None
# case -- if no tokens have been streamed for the tool, e.g.
# only the array brackets, stream nothing
if not tool_call_arr:
return None
# select as the current tool call the one we're on the state at
current_tool_call: dict = tool_call_arr[self.current_tool_id]
delta = None
# Only advance once the current tool name is streamed; granite
# emits arguments before name, so advancing early would drop it.
if len(tool_call_arr) > self.current_tool_id + 1 and (
self.current_tool_id < 0 or self.current_tool_name_sent
):
# if we're moving on to a new call, first make sure we
# haven't missed anything in the previous one that was
# auto-generated due to JSON completions, but wasn't
# streamed to the client yet.
if self.current_tool_id >= 0:
cur_arguments = current_tool_call.get("arguments")
if cur_arguments:
cur_args_json = json.dumps(cur_arguments, ensure_ascii=False)
sent = len(self.streamed_args_for_tool[self.current_tool_id])
argument_diff = cur_args_json[sent:]
logger.debug("got arguments diff: %s", argument_diff)
delta = DeltaMessage(
tool_calls=[
DeltaToolCall(
index=self.current_tool_id,
function=DeltaFunctionCall(
arguments=argument_diff
).model_dump(exclude_none=True),
)
]
)
self.streamed_args_for_tool[self.current_tool_id] += (
argument_diff
)
# re-set stuff pertaining to progress in the current tool
self.current_tool_id += 1
self.current_tool_name_sent = False
self.streamed_args_for_tool.append("")
logger.debug("starting on new tool %d", self.current_tool_id)
return delta
# if the current tool name hasn't been sent, send if available
# - otherwise send nothing
elif not self.current_tool_name_sent:
function_name = current_tool_call.get("name")
if function_name:
delta = DeltaMessage(
tool_calls=[
DeltaToolCall(
index=self.current_tool_id,
type="function",
id=make_tool_call_id(),
function=DeltaFunctionCall(
name=function_name
).model_dump(exclude_none=True),
)
]
)
self.current_tool_name_sent = True
# now we know we're on the same tool call and we're streaming
# arguments
else:
cur_arguments = current_tool_call.get("arguments")
if cur_arguments:
sent = len(self.streamed_args_for_tool[self.current_tool_id])
cur_args_json = json.dumps(cur_arguments, ensure_ascii=False)
prev_arguments = self.prev_tool_call_arr[self.current_tool_id].get(
"arguments"
)
argument_diff = None
if is_complete[self.current_tool_id]:
argument_diff = cur_args_json[sent:]
elif prev_arguments:
prev_args_json = json.dumps(prev_arguments, ensure_ascii=False)
if cur_args_json != prev_args_json:
prefix = find_common_prefix(prev_args_json, cur_args_json)
argument_diff = prefix[sent:]
if argument_diff is not None:
delta = DeltaMessage(
tool_calls=[
DeltaToolCall(
index=self.current_tool_id,
function=DeltaFunctionCall(
arguments=argument_diff
).model_dump(exclude_none=True),
)
]
)
self.streamed_args_for_tool[self.current_tool_id] += (
argument_diff
)
self.prev_tool_call_arr = tool_call_arr
return delta
except Exception as e:
logger.error("Error trying to handle streaming tool call: %s", e)
logger.debug(
"Skipping chunk as a result of tool streaming extraction error"
)
return None