[Bugfix][Tool Parser] Migrate Granite to the streaming Parser Engine (#49648)

Signed-off-by: Nikhil Kulkarni <nikhilkulkarni1755@gmail.com>
Co-authored-by: Claude Opus 4.8 <noreply@anthropic.com>
Co-authored-by: Chauncey <chaunceyjiang@gmail.com>
This commit is contained in:
Nikhil Kulkarni
2026-09-23 10:37:40 +00:00
committed by GitHub
co-authored by Claude Opus 4.8 Chauncey
parent c961121519
commit 15ed1262e7
14 changed files with 631 additions and 369 deletions
@@ -99,6 +99,10 @@ def _discover_pairings() -> list[_PairingInfo]:
# terminal matching; combined-parser replay coverage lives in
# test_inkling.py.
continue
if cfg.name == "granite":
# Granite has a JSON-array tool body with no TOOL_END terminal;
# its replay coverage lives in test_granite.py.
continue
parser_cls = type(
f"_Delegating{engine_cls.__name__}",
+234
View File
@@ -0,0 +1,234 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests for the engine-based Granite parser.
Granite emits tool calls as a JSON array following a marker
(``<|tool_call|>`` for 3.0, ``<tool_call>`` for 3.1) with no closing
terminal; the ``]`` ends the region and surrounding prose is content. The
engine's ``tool_call_body_array`` mode splits the array into one call per
element and ``_granite_arg_converter`` carves each element's ``arguments``.
"""
import json
import pytest
from tests.parser.engine.conftest import make_mock_tokenizer
from tests.parser.engine.trace_builder import _GRANITE_SCENARIOS, _build_granite
from vllm.parser.granite import GraniteParser, _granite_arg_converter
TOOL_TOKEN = "<|tool_call|>"
TOOL_STRING = "<tool_call>"
_GRANITE_VOCAB = {TOOL_TOKEN: 49154}
@pytest.fixture
def mock_tokenizer():
return make_mock_tokenizer(_GRANITE_VOCAB)
@pytest.fixture
def parser(mock_tokenizer):
return GraniteParser(mock_tokenizer)
def _tokenize(text: str) -> list[tuple[int, str]]:
"""Tokenize like the real stream: ``<|tool_call|>`` is one special token,
plain text becomes one token per character."""
tokens: list[tuple[int, str]] = []
i = 0
while i < len(text):
if text.startswith(TOOL_TOKEN, i):
tokens.append((_GRANITE_VOCAB[TOOL_TOKEN], TOOL_TOKEN))
i += len(TOOL_TOKEN)
else:
tokens.append((ord(text[i]), text[i]))
i += 1
return tokens
def _stream(parser, request, text: str, chunk_size: int):
tokens = _tokenize(text)
results = []
previous_text = ""
previous_token_ids: list[int] = []
for start in range(0, len(tokens), chunk_size):
batch = tokens[start : start + chunk_size]
delta_text = "".join(t for _, t in batch)
delta_token_ids = [tid for tid, _ in batch]
current_text = previous_text + delta_text
current_token_ids = previous_token_ids + delta_token_ids
delta = parser.extract_tool_calls_streaming(
previous_text=previous_text,
current_text=current_text,
delta_text=delta_text,
previous_token_ids=tuple(previous_token_ids),
current_token_ids=tuple(current_token_ids),
delta_token_ids=tuple(delta_token_ids),
request=request,
)
results.append(delta)
previous_text = current_text
previous_token_ids = current_token_ids
finish = parser.finish_streaming()
if finish is not None:
results.append(finish)
return results
def _collect_content(results) -> str:
return "".join(d.content for d in results if d and d.content)
def _collect_names(results) -> list[str]:
names: dict[int, str] = {}
for d in results:
if not (d and d.tool_calls):
continue
for tc in d.tool_calls:
if tc.function and tc.function.name:
names[tc.index] = names.get(tc.index, "") + tc.function.name
return [names[i] for i in sorted(names)]
def _collect_args(results) -> dict[int, str]:
args: dict[int, str] = {}
for d in results:
if not (d and d.tool_calls):
continue
for tc in d.tool_calls:
if tc.function and tc.function.arguments:
args[tc.index] = args.get(tc.index, "") + tc.function.arguments
return args
class TestArgConverter:
def test_complete_wrapper(self):
raw = '{"name": "get_weather", "arguments": {"city": "SF"}}'
assert _granite_arg_converter(raw, False) == '{"city": "SF"}'
def test_arguments_before_name(self):
raw = '{"arguments": {"city": "SF"}, "name": "get_weather"}'
assert _granite_arg_converter(raw, False) == '{"city": "SF"}'
def test_partial_before_arguments(self):
assert _granite_arg_converter('{"name": "get_w', True) == ""
def test_partial_inside_arguments(self):
raw = '{"name": "x", "arguments": {"a": 1'
assert _granite_arg_converter(raw, True) == '{"a": 1'
def test_prefix_stability(self):
# Each growing prefix must extend the previous converter output.
full = '{"name": "x", "arguments": {"city": "Tokyo"}}'
prev = ""
for i in range(1, len(full) + 1):
out = _granite_arg_converter(full[:i], True)
assert out.startswith(prev) or prev.startswith(out) or out == ""
if out:
prev = out
def test_missing_arguments_defaults(self):
assert _granite_arg_converter('{"name": "x"}', False) == "{}"
assert _granite_arg_converter('{"name": "x"}', True) == ""
def test_non_object_arguments_rejected(self):
with pytest.raises(ValueError, match="JSON object"):
_granite_arg_converter('{"name": "x", "arguments": [1]}', False)
class TestNonStreaming:
def test_plain_text(self, parser, mock_request):
reasoning, content, tools = parser.parse("Just a reply.", mock_request)
assert reasoning is None
assert content == "Just a reply."
assert tools is None
def test_single_tool_token_marker(self, parser, mock_request):
text = (
f'{TOOL_TOKEN} [{{"name": "get_weather", "arguments": {{"city": "SF"}}}}]'
)
_, content, tools = parser.parse(text, mock_request)
assert content is None
assert [t.name for t in tools] == ["get_weather"]
assert json.loads(tools[0].arguments) == {"city": "SF"}
def test_single_tool_string_marker(self, parser, mock_request):
# Granite 3.1 uses the plain-text ``<tool_call>`` marker.
text = f'{TOOL_STRING} [{{"name": "get_time", "arguments": {{}}}}]'
_, content, tools = parser.parse(text, mock_request)
assert [t.name for t in tools] == ["get_time"]
assert json.loads(tools[0].arguments) == {}
def test_parallel_calls_one_array(self, parser, mock_request):
text = (
f'{TOOL_TOKEN} [{{"name": "a", "arguments": {{"x": 1}}}}, '
f'{{"name": "b", "arguments": {{"y": [1, 2]}}}}]'
)
_, _, tools = parser.parse(text, mock_request)
assert [t.name for t in tools] == ["a", "b"]
assert json.loads(tools[0].arguments) == {"x": 1}
assert json.loads(tools[1].arguments) == {"y": [1, 2]}
def test_surrounding_text_is_content(self, parser, mock_request):
text = (
"Let me check.\n"
f'{TOOL_TOKEN} [{{"name": "get_weather", "arguments": {{"city": "SF"}}}}]'
)
_, content, tools = parser.parse(text, mock_request)
assert content == "Let me check."
assert [t.name for t in tools] == ["get_weather"]
def test_arguments_before_name(self, parser, mock_request):
text = (
f'{TOOL_TOKEN} [{{"arguments": {{"city": "SF"}}, "name": "get_weather"}}]'
)
_, _, tools = parser.parse(text, mock_request)
assert [t.name for t in tools] == ["get_weather"]
assert json.loads(tools[0].arguments) == {"city": "SF"}
def test_marker_without_array_is_not_a_tool_call(self, parser, mock_request):
# A non-array body after the marker (malformed) yields no tool call.
text = f'{TOOL_TOKEN} {{"name": "func", "arguments": {{}}}}'
_, _, tools = parser.parse(text, mock_request)
assert not tools
class TestStreaming:
@pytest.mark.parametrize("chunk_size", [1, 2, 3, 5, 11])
def test_parallel_calls_chunk_invariance(self, parser, mock_request, chunk_size):
text = (
f'{TOOL_TOKEN} [{{"name": "get_weather", '
f'"arguments": {{"city": "Tokyo"}}}}, '
f'{{"name": "get_time", "arguments": {{"timezone": "Asia/Tokyo"}}}}]'
)
results = _stream(parser, mock_request, text, chunk_size)
assert _collect_names(results) == ["get_weather", "get_time"]
args = _collect_args(results)
assert json.loads(args[0]) == {"city": "Tokyo"}
assert json.loads(args[1]) == {"timezone": "Asia/Tokyo"}
@pytest.mark.parametrize("chunk_size", [1, 3, 7])
def test_surrounding_text_streaming(self, parser, mock_request, chunk_size):
text = (
"Let me check.\n"
f'{TOOL_TOKEN} [{{"name": "get_weather", "arguments": {{"city": "SF"}}}}]'
)
results = _stream(parser, mock_request, text, chunk_size)
assert _collect_names(results) == ["get_weather"]
assert "Let me check." in _collect_content(results)
def test_incomplete_array_at_eos(self, parser, mock_request):
# Truncated mid-element must not raise and must not leak the marker.
text = f'{TOOL_TOKEN} [{{"name": "func", "arguments": {{"a": 1'
results = _stream(parser, mock_request, text, 3)
assert TOOL_TOKEN not in _collect_content(results)
class TestReplay:
@pytest.mark.parametrize("scenario", _GRANITE_SCENARIOS, ids=lambda s: s.id)
def test_replay_scenarios(self, scenario):
# ``_build_granite`` replays the sample through GraniteParser at
# chunk_size=1 and asserts reasoning/content/tool_calls (validate=True).
_build_granite(scenario)
+5
View File
@@ -78,6 +78,11 @@ def _discover_parsers() -> list[_ParserInfo]:
# Inkling opts out of token-id terminal matching and has typed
# structural blocks; its replay coverage lives in test_inkling.py.
continue
if cfg.name == "granite":
# Granite has a JSON-array tool body with no TOOL_END terminal, so
# it does not fit this token-terminal harness; its replay coverage
# lives in test_granite.py.
continue
tool_end = cfg.token_id_terminals.get("TOOL_END")
if not tool_end:
raise RuntimeError(
+85
View File
@@ -34,6 +34,7 @@ from vllm.parser.engine.registered_adapters import (
DeepSeekV41Parser,
Gemma4Parser,
Glm47MoeParser,
GraniteParser,
InklingParser,
KimiK2Parser,
MinimaxM2Parser,
@@ -1042,6 +1043,89 @@ def _build_inkling(scenario: Scenario, validate: bool = True) -> Sample:
return sample
# ── Granite (JSON-array tool bodies, no reasoning) ───────────────────────
_GRANITE_VOCAB: dict[str, int] = {
"<|tool_call|>": 49154,
}
def _granite_segments(scenario: Scenario) -> list[tuple[str, bool]]:
segs: list[tuple[str, bool]] = []
if scenario.content:
# Granite has no reasoning; prose is plain content preceding the marker.
segs.append((scenario.content, False))
if scenario.tool_calls:
segs.append(("<|tool_call|>", True))
payload = json.dumps(
[
{"name": tc.name, "arguments": tc.arguments}
for tc in scenario.tool_calls
],
ensure_ascii=False,
separators=(", ", ": "),
)
segs.append((" " + payload, False))
return segs
def _granite_expected_content(scenario: Scenario) -> str | None:
if scenario.tool_calls:
if not scenario.content:
return None
return scenario.content.strip() or None
return scenario.content
def _build_granite(scenario: Scenario, validate: bool = True) -> Sample:
sample = _make_sample(
sample_id=f"granite-{scenario.id}",
description=scenario.description,
vocab=_GRANITE_VOCAB,
segments=_granite_segments(scenario),
expected_reasoning=None,
expected_content=_granite_expected_content(scenario),
expected_tool_calls=_expected_tc(scenario),
tools=_expected_tools(scenario),
)
if validate:
_validate_sample(sample, GraniteParser)
return sample
# Granite has no reasoning, so the shared reasoning-centric SCENARIOS do not
# apply; these exercise the JSON-array tool body (single, parallel, surrounding
# text) instead.
_GRANITE_SCENARIOS: list[Scenario] = [
Scenario(
id="single-tool",
description="Single tool call",
tool_calls=[_READ_TOOL],
),
Scenario(
id="parallel-tools",
description="Parallel tool calls in one JSON array",
tool_calls=[_BASH_TOOL, _WEATHER_TOOL],
),
Scenario(
id="complex-json-args",
description="Tool call with nested objects, arrays, numbers, booleans",
tool_calls=[_COMPLEX_TOOL],
),
Scenario(
id="content-only",
description="Plain content response without tool calls",
content="Hello! How can I help you today?",
),
Scenario(
id="surrounding-text",
description="Prose content preceding the tool call",
content="Let me check the weather.",
tool_calls=[_WEATHER_TOOL],
),
]
# ── Registry and public API ──────────────────────────────────────────
_BUILDERS: dict[str, Any] = {
@@ -1049,6 +1133,7 @@ _BUILDERS: dict[str, Any] = {
"deepseek_v4": _build_deepseek_v4,
"deepseek_v41": functools.partial(_build_deepseek_v4, v41=True),
"gemma4": _build_gemma4,
"granite": _build_granite,
"minimax_m2": _build_minimax_m2,
"nemotron_v3": _build_nemotron_v3,
"seed_oss": _build_seed_oss,