diff --git a/tests/parser/engine/test_delegating_replay.py b/tests/parser/engine/test_delegating_replay.py index 2bc4338785..5cd48dd870 100644 --- a/tests/parser/engine/test_delegating_replay.py +++ b/tests/parser/engine/test_delegating_replay.py @@ -99,6 +99,10 @@ def _discover_pairings() -> list[_PairingInfo]: # terminal matching; combined-parser replay coverage lives in # test_inkling.py. continue + if cfg.name == "granite": + # Granite has a JSON-array tool body with no TOOL_END terminal; + # its replay coverage lives in test_granite.py. + continue parser_cls = type( f"_Delegating{engine_cls.__name__}", diff --git a/tests/parser/engine/test_granite.py b/tests/parser/engine/test_granite.py new file mode 100644 index 0000000000..ddc7e2bfe0 --- /dev/null +++ b/tests/parser/engine/test_granite.py @@ -0,0 +1,234 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Tests for the engine-based Granite parser. + +Granite emits tool calls as a JSON array following a marker +(``<|tool_call|>`` for 3.0, ```` for 3.1) with no closing +terminal; the ``]`` ends the region and surrounding prose is content. The +engine's ``tool_call_body_array`` mode splits the array into one call per +element and ``_granite_arg_converter`` carves each element's ``arguments``. +""" + +import json + +import pytest + +from tests.parser.engine.conftest import make_mock_tokenizer +from tests.parser.engine.trace_builder import _GRANITE_SCENARIOS, _build_granite +from vllm.parser.granite import GraniteParser, _granite_arg_converter + +TOOL_TOKEN = "<|tool_call|>" +TOOL_STRING = "" + +_GRANITE_VOCAB = {TOOL_TOKEN: 49154} + + +@pytest.fixture +def mock_tokenizer(): + return make_mock_tokenizer(_GRANITE_VOCAB) + + +@pytest.fixture +def parser(mock_tokenizer): + return GraniteParser(mock_tokenizer) + + +def _tokenize(text: str) -> list[tuple[int, str]]: + """Tokenize like the real stream: ``<|tool_call|>`` is one special token, + plain text becomes one token per character.""" + tokens: list[tuple[int, str]] = [] + i = 0 + while i < len(text): + if text.startswith(TOOL_TOKEN, i): + tokens.append((_GRANITE_VOCAB[TOOL_TOKEN], TOOL_TOKEN)) + i += len(TOOL_TOKEN) + else: + tokens.append((ord(text[i]), text[i])) + i += 1 + return tokens + + +def _stream(parser, request, text: str, chunk_size: int): + tokens = _tokenize(text) + results = [] + previous_text = "" + previous_token_ids: list[int] = [] + for start in range(0, len(tokens), chunk_size): + batch = tokens[start : start + chunk_size] + delta_text = "".join(t for _, t in batch) + delta_token_ids = [tid for tid, _ in batch] + current_text = previous_text + delta_text + current_token_ids = previous_token_ids + delta_token_ids + delta = parser.extract_tool_calls_streaming( + previous_text=previous_text, + current_text=current_text, + delta_text=delta_text, + previous_token_ids=tuple(previous_token_ids), + current_token_ids=tuple(current_token_ids), + delta_token_ids=tuple(delta_token_ids), + request=request, + ) + results.append(delta) + previous_text = current_text + previous_token_ids = current_token_ids + finish = parser.finish_streaming() + if finish is not None: + results.append(finish) + return results + + +def _collect_content(results) -> str: + return "".join(d.content for d in results if d and d.content) + + +def _collect_names(results) -> list[str]: + names: dict[int, str] = {} + for d in results: + if not (d and d.tool_calls): + continue + for tc in d.tool_calls: + if tc.function and tc.function.name: + names[tc.index] = names.get(tc.index, "") + tc.function.name + return [names[i] for i in sorted(names)] + + +def _collect_args(results) -> dict[int, str]: + args: dict[int, str] = {} + for d in results: + if not (d and d.tool_calls): + continue + for tc in d.tool_calls: + if tc.function and tc.function.arguments: + args[tc.index] = args.get(tc.index, "") + tc.function.arguments + return args + + +class TestArgConverter: + def test_complete_wrapper(self): + raw = '{"name": "get_weather", "arguments": {"city": "SF"}}' + assert _granite_arg_converter(raw, False) == '{"city": "SF"}' + + def test_arguments_before_name(self): + raw = '{"arguments": {"city": "SF"}, "name": "get_weather"}' + assert _granite_arg_converter(raw, False) == '{"city": "SF"}' + + def test_partial_before_arguments(self): + assert _granite_arg_converter('{"name": "get_w', True) == "" + + def test_partial_inside_arguments(self): + raw = '{"name": "x", "arguments": {"a": 1' + assert _granite_arg_converter(raw, True) == '{"a": 1' + + def test_prefix_stability(self): + # Each growing prefix must extend the previous converter output. + full = '{"name": "x", "arguments": {"city": "Tokyo"}}' + prev = "" + for i in range(1, len(full) + 1): + out = _granite_arg_converter(full[:i], True) + assert out.startswith(prev) or prev.startswith(out) or out == "" + if out: + prev = out + + def test_missing_arguments_defaults(self): + assert _granite_arg_converter('{"name": "x"}', False) == "{}" + assert _granite_arg_converter('{"name": "x"}', True) == "" + + def test_non_object_arguments_rejected(self): + with pytest.raises(ValueError, match="JSON object"): + _granite_arg_converter('{"name": "x", "arguments": [1]}', False) + + +class TestNonStreaming: + def test_plain_text(self, parser, mock_request): + reasoning, content, tools = parser.parse("Just a reply.", mock_request) + assert reasoning is None + assert content == "Just a reply." + assert tools is None + + def test_single_tool_token_marker(self, parser, mock_request): + text = ( + f'{TOOL_TOKEN} [{{"name": "get_weather", "arguments": {{"city": "SF"}}}}]' + ) + _, content, tools = parser.parse(text, mock_request) + assert content is None + assert [t.name for t in tools] == ["get_weather"] + assert json.loads(tools[0].arguments) == {"city": "SF"} + + def test_single_tool_string_marker(self, parser, mock_request): + # Granite 3.1 uses the plain-text ```` marker. + text = f'{TOOL_STRING} [{{"name": "get_time", "arguments": {{}}}}]' + _, content, tools = parser.parse(text, mock_request) + assert [t.name for t in tools] == ["get_time"] + assert json.loads(tools[0].arguments) == {} + + def test_parallel_calls_one_array(self, parser, mock_request): + text = ( + f'{TOOL_TOKEN} [{{"name": "a", "arguments": {{"x": 1}}}}, ' + f'{{"name": "b", "arguments": {{"y": [1, 2]}}}}]' + ) + _, _, tools = parser.parse(text, mock_request) + assert [t.name for t in tools] == ["a", "b"] + assert json.loads(tools[0].arguments) == {"x": 1} + assert json.loads(tools[1].arguments) == {"y": [1, 2]} + + def test_surrounding_text_is_content(self, parser, mock_request): + text = ( + "Let me check.\n" + f'{TOOL_TOKEN} [{{"name": "get_weather", "arguments": {{"city": "SF"}}}}]' + ) + _, content, tools = parser.parse(text, mock_request) + assert content == "Let me check." + assert [t.name for t in tools] == ["get_weather"] + + def test_arguments_before_name(self, parser, mock_request): + text = ( + f'{TOOL_TOKEN} [{{"arguments": {{"city": "SF"}}, "name": "get_weather"}}]' + ) + _, _, tools = parser.parse(text, mock_request) + assert [t.name for t in tools] == ["get_weather"] + assert json.loads(tools[0].arguments) == {"city": "SF"} + + def test_marker_without_array_is_not_a_tool_call(self, parser, mock_request): + # A non-array body after the marker (malformed) yields no tool call. + text = f'{TOOL_TOKEN} {{"name": "func", "arguments": {{}}}}' + _, _, tools = parser.parse(text, mock_request) + assert not tools + + +class TestStreaming: + @pytest.mark.parametrize("chunk_size", [1, 2, 3, 5, 11]) + def test_parallel_calls_chunk_invariance(self, parser, mock_request, chunk_size): + text = ( + f'{TOOL_TOKEN} [{{"name": "get_weather", ' + f'"arguments": {{"city": "Tokyo"}}}}, ' + f'{{"name": "get_time", "arguments": {{"timezone": "Asia/Tokyo"}}}}]' + ) + results = _stream(parser, mock_request, text, chunk_size) + assert _collect_names(results) == ["get_weather", "get_time"] + args = _collect_args(results) + assert json.loads(args[0]) == {"city": "Tokyo"} + assert json.loads(args[1]) == {"timezone": "Asia/Tokyo"} + + @pytest.mark.parametrize("chunk_size", [1, 3, 7]) + def test_surrounding_text_streaming(self, parser, mock_request, chunk_size): + text = ( + "Let me check.\n" + f'{TOOL_TOKEN} [{{"name": "get_weather", "arguments": {{"city": "SF"}}}}]' + ) + results = _stream(parser, mock_request, text, chunk_size) + assert _collect_names(results) == ["get_weather"] + assert "Let me check." in _collect_content(results) + + def test_incomplete_array_at_eos(self, parser, mock_request): + # Truncated mid-element must not raise and must not leak the marker. + text = f'{TOOL_TOKEN} [{{"name": "func", "arguments": {{"a": 1' + results = _stream(parser, mock_request, text, 3) + assert TOOL_TOKEN not in _collect_content(results) + + +class TestReplay: + @pytest.mark.parametrize("scenario", _GRANITE_SCENARIOS, ids=lambda s: s.id) + def test_replay_scenarios(self, scenario): + # ``_build_granite`` replays the sample through GraniteParser at + # chunk_size=1 and asserts reasoning/content/tool_calls (validate=True). + _build_granite(scenario) diff --git a/tests/parser/engine/test_replay.py b/tests/parser/engine/test_replay.py index fee74f8d2c..fbb911d7ae 100644 --- a/tests/parser/engine/test_replay.py +++ b/tests/parser/engine/test_replay.py @@ -78,6 +78,11 @@ def _discover_parsers() -> list[_ParserInfo]: # Inkling opts out of token-id terminal matching and has typed # structural blocks; its replay coverage lives in test_inkling.py. continue + if cfg.name == "granite": + # Granite has a JSON-array tool body with no TOOL_END terminal, so + # it does not fit this token-terminal harness; its replay coverage + # lives in test_granite.py. + continue tool_end = cfg.token_id_terminals.get("TOOL_END") if not tool_end: raise RuntimeError( diff --git a/tests/parser/engine/trace_builder.py b/tests/parser/engine/trace_builder.py index 36e347fc57..2851edd4d0 100644 --- a/tests/parser/engine/trace_builder.py +++ b/tests/parser/engine/trace_builder.py @@ -34,6 +34,7 @@ from vllm.parser.engine.registered_adapters import ( DeepSeekV41Parser, Gemma4Parser, Glm47MoeParser, + GraniteParser, InklingParser, KimiK2Parser, MinimaxM2Parser, @@ -1042,6 +1043,89 @@ def _build_inkling(scenario: Scenario, validate: bool = True) -> Sample: return sample +# ── Granite (JSON-array tool bodies, no reasoning) ─────────────────────── + +_GRANITE_VOCAB: dict[str, int] = { + "<|tool_call|>": 49154, +} + + +def _granite_segments(scenario: Scenario) -> list[tuple[str, bool]]: + segs: list[tuple[str, bool]] = [] + if scenario.content: + # Granite has no reasoning; prose is plain content preceding the marker. + segs.append((scenario.content, False)) + if scenario.tool_calls: + segs.append(("<|tool_call|>", True)) + payload = json.dumps( + [ + {"name": tc.name, "arguments": tc.arguments} + for tc in scenario.tool_calls + ], + ensure_ascii=False, + separators=(", ", ": "), + ) + segs.append((" " + payload, False)) + return segs + + +def _granite_expected_content(scenario: Scenario) -> str | None: + if scenario.tool_calls: + if not scenario.content: + return None + return scenario.content.strip() or None + return scenario.content + + +def _build_granite(scenario: Scenario, validate: bool = True) -> Sample: + sample = _make_sample( + sample_id=f"granite-{scenario.id}", + description=scenario.description, + vocab=_GRANITE_VOCAB, + segments=_granite_segments(scenario), + expected_reasoning=None, + expected_content=_granite_expected_content(scenario), + expected_tool_calls=_expected_tc(scenario), + tools=_expected_tools(scenario), + ) + if validate: + _validate_sample(sample, GraniteParser) + return sample + + +# Granite has no reasoning, so the shared reasoning-centric SCENARIOS do not +# apply; these exercise the JSON-array tool body (single, parallel, surrounding +# text) instead. +_GRANITE_SCENARIOS: list[Scenario] = [ + Scenario( + id="single-tool", + description="Single tool call", + tool_calls=[_READ_TOOL], + ), + Scenario( + id="parallel-tools", + description="Parallel tool calls in one JSON array", + tool_calls=[_BASH_TOOL, _WEATHER_TOOL], + ), + Scenario( + id="complex-json-args", + description="Tool call with nested objects, arrays, numbers, booleans", + tool_calls=[_COMPLEX_TOOL], + ), + Scenario( + id="content-only", + description="Plain content response without tool calls", + content="Hello! How can I help you today?", + ), + Scenario( + id="surrounding-text", + description="Prose content preceding the tool call", + content="Let me check the weather.", + tool_calls=[_WEATHER_TOOL], + ), +] + + # ── Registry and public API ────────────────────────────────────────── _BUILDERS: dict[str, Any] = { @@ -1049,6 +1133,7 @@ _BUILDERS: dict[str, Any] = { "deepseek_v4": _build_deepseek_v4, "deepseek_v41": functools.partial(_build_deepseek_v4, v41=True), "gemma4": _build_gemma4, + "granite": _build_granite, "minimax_m2": _build_minimax_m2, "nemotron_v3": _build_nemotron_v3, "seed_oss": _build_seed_oss, diff --git a/tests/tool_parsers/test_granite_tool_parser.py b/tests/tool_parsers/test_granite_tool_parser.py index af3386112f..5c51f389be 100644 --- a/tests/tool_parsers/test_granite_tool_parser.py +++ b/tests/tool_parsers/test_granite_tool_parser.py @@ -16,7 +16,7 @@ from tests.tool_parsers.utils import ( split_string_into_token_deltas, ) from vllm.tokenizers import get_tokenizer -from vllm.tool_parsers.granite_tool_parser import GraniteToolParser +from vllm.tool_parsers.granite_engine_tool_parser import GraniteEngineToolParser class TestGraniteToolParser(ToolParserTests): @@ -76,23 +76,6 @@ I'll get that information.""", single_tool_call_expected_content=None, parallel_tool_calls_count=2, parallel_tool_calls_names=["get_weather", "get_time"], - # xfail markers - xfail_streaming={ - "test_malformed_input": ( - "Streaming mode incorrectly creates tool call from malformed JSON" - ), - "test_surrounding_text": ( - "Parser doesn't handle surrounding text correctly in streaming" - ), - "test_streaming_reconstruction": ( - "Streaming mode doesn't strip <|tool_call|> marker from content" - ), - }, - xfail_nonstreaming={ - "test_surrounding_text": ( - "Parser doesn't handle surrounding text correctly in non-streaming" - ), - }, ) # Granite-Specific Tests @@ -140,7 +123,7 @@ def test_streaming_parallel_calls_batched_deltas(granite_tokenizer, chunk_size): parallel calls must not drop the first call's name. granite streams arguments before name, so the name only completes as the next call appears. """ - parser = GraniteToolParser(granite_tokenizer) + parser = GraniteEngineToolParser(granite_tokenizer) model_output = ( '<|tool_call|> [{"arguments": {"city": "Tokyo"}, "name": "get_weather"}, ' '{"arguments": {"timezone": "Asia/Tokyo"}, "name": "get_time"}]' diff --git a/vllm/parser/engine/parser_engine_config.py b/vllm/parser/engine/parser_engine_config.py index b8d7dc6e46..215fa0d9bd 100644 --- a/vllm/parser/engine/parser_engine_config.py +++ b/vllm/parser/engine/parser_engine_config.py @@ -82,6 +82,12 @@ class ParserEngineConfig: tool_args_json: bool = True + # The tool region body is a single JSON array of ``{"name", "arguments"}`` + # objects (one marker → N calls, no closing terminal). The engine splits + # the array into one tool call per element; the array's ``]`` ends the + # region and any trailing text becomes content. Used by Granite. + tool_call_body_array: bool = False + arg_structural_chars: frozenset[str] | None = None # Special tokens exempt from auto-drop but not state-machine terminals. diff --git a/vllm/parser/engine/registered_adapters.py b/vllm/parser/engine/registered_adapters.py index 8e10a1b5ef..f7f749948d 100644 --- a/vllm/parser/engine/registered_adapters.py +++ b/vllm/parser/engine/registered_adapters.py @@ -13,6 +13,7 @@ from vllm.parser.deepseek_v41 import DeepSeekV41Parser from vllm.parser.engine.adapters import make_adapters from vllm.parser.gemma4 import Gemma4Parser from vllm.parser.glm47_moe import Glm47MoeParser +from vllm.parser.granite import GraniteParser from vllm.parser.inkling import InklingParser from vllm.parser.kimi_k2 import KimiK2Parser from vllm.parser.minimax_m2 import MinimaxM2Parser @@ -46,6 +47,11 @@ from vllm.parser.seed_oss import SeedOssParser Gemma4ParserToolAdapter, ) = make_adapters(Gemma4Parser) +( + GraniteParserReasoningAdapter, + GraniteParserToolAdapter, +) = make_adapters(GraniteParser) + ( NemotronV3ParserReasoningAdapter, NemotronV3ParserToolAdapter, diff --git a/vllm/parser/engine/streaming_parser_engine.py b/vllm/parser/engine/streaming_parser_engine.py index 21d053ef28..3fd1ecdcb6 100644 --- a/vllm/parser/engine/streaming_parser_engine.py +++ b/vllm/parser/engine/streaming_parser_engine.py @@ -190,6 +190,12 @@ class StreamingParserEngine: self._args_in_string: bool = False self._args_escape_next: bool = False + def _reset_array_state(self) -> None: + # State for ``tool_call_body_array``: whether the opening ``[`` has been + # seen, and whether a ``{...}`` element is currently being streamed. + self._array_started: bool = False + self._array_in_element: bool = False + def reset(self, initial_state: ParserState | None = None) -> None: """Reset mutable state for reuse across requests. @@ -213,6 +219,7 @@ class StreamingParserEngine: self._message_header_token_count = 0 self._in_skipped_tool_span = False self._reset_args_state() + self._reset_array_state() def feed( self, @@ -306,7 +313,12 @@ class StreamingParserEngine: ParserState.TOOL_NAME, ParserState.TOOL_BETWEEN, ): - if self.tool_index >= 0: + # In array mode a closed element has already emitted its + # TOOL_CALL_END; only a truncated (still-open) element needs one. + end_open_call = ( + self._array_in_element if self.config.tool_call_body_array else True + ) + if end_open_call and self.tool_index >= 0: events.append( SemanticEvent( EventType.TOOL_CALL_END, @@ -474,6 +486,8 @@ class StreamingParserEngine: self._message_header_token_count += token_count return [] if self.state == ParserState.TOOL_ARGS: + if self.config.tool_call_body_array: + return self._feed_array_text(text) if self.config.tool_args_json: return self._feed_args_text(text) return [ @@ -564,6 +578,8 @@ class StreamingParserEngine: self._args_in_string = False self._args_escape_next = False self._args_safe_end = 0 + if self.config.tool_call_body_array: + self._reset_array_state() return events @@ -634,3 +650,88 @@ class StreamingParserEngine: tool_index=self.tool_index, ) ] + + # ── JSON-array tool body (tool_call_body_array) ───────────────────── + + def _feed_array_text(self, text: str) -> list[SemanticEvent]: + """Split a JSON-array tool body into one tool call per element. + + Granite emits ``[{"name":..,"arguments":{..}}, ...]`` after the tool + marker. Element interiors are streamed through the string/brace-aware + :meth:`_feed_args_char`, so each element behaves exactly like a + single-object tool body; the array's ``]`` ends the region and any + trailing text becomes content. + """ + events: list[SemanticEvent] = [] + for i, ch in enumerate(text): + if self.state != ParserState.TOOL_ARGS: + # The array closed mid-chunk; the rest of the text is content. + events.extend(self._emit_for_state(text[i:])) + break + events.extend(self._feed_array_char(ch)) + return events + + def _feed_array_char(self, ch: str) -> list[SemanticEvent]: + if self._array_in_element: + events = self._feed_args_char(ch) + # Element object closed: top-level '}' with all braces balanced. + if ( + ch == "}" + and not self._args_in_string + and not self._args_escape_next + and self._args_brace_depth == 0 + ): + events.extend(self._end_array_element()) + return events + + # Between elements only structural array characters are expected. + if ch in " \t\r\n": + return [] + if not self._array_started: + if ch == "[": + self._array_started = True + return [] + return self._bail_array_to_content(ch) + if ch == ",": + return [] + if ch == "{": + return self._start_array_element(ch) + if ch == "]": + self.state = ParserState.CONTENT + self._reset_array_state() + return [] + return self._bail_array_to_content(ch) + + def _start_array_element(self, ch: str) -> list[SemanticEvent]: + self.tool_index += 1 + self._reset_args_state() + self._array_in_element = True + events = [SemanticEvent(EventType.TOOL_CALL_START, tool_index=self.tool_index)] + events.extend(self._feed_args_char(ch)) + return events + + def _end_array_element(self) -> list[SemanticEvent]: + self._array_in_element = False + events: list[SemanticEvent] = [] + # Flush the held closing '}' so the slot body is the full element. + if self._args_buffer: + events.append( + SemanticEvent( + EventType.ARG_VALUE_CHUNK, + value=self._args_buffer, + tool_index=self.tool_index, + ) + ) + self._args_buffer = "" + self._args_safe_end = 0 + events.append( + SemanticEvent(EventType.TOOL_CALL_END, tool_index=self.tool_index) + ) + return events + + def _bail_array_to_content(self, ch: str) -> list[SemanticEvent]: + # Marker not followed by a JSON array (malformed) — revert to content + # so the remaining tokens are emitted as plain text, no tool call. + self.state = ParserState.CONTENT + self._reset_array_state() + return self._emit_for_state(ch) diff --git a/vllm/parser/granite.py b/vllm/parser/granite.py new file mode 100644 index 0000000000..afa2021225 --- /dev/null +++ b/vllm/parser/granite.py @@ -0,0 +1,88 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Granite parser for JSON-array tool calls. + +Granite 3.0/3.1 emit tool calls as a JSON array following a marker:: + + <|tool_call|> [{"name": "get_weather", "arguments": {"city": "SF"}}] # 3.0 + [{"name": "get_weather", "arguments": {"city": "SF"}}] # 3.1 + +``<|tool_call|>`` is a single special token; ```` is plain text. +The array may hold several calls, has no closing marker (the ``]`` ends it), +and any surrounding prose is content. The engine's ``tool_call_body_array`` +mode splits the array into one call per element and extracts each name via the +name-from-args path; ``_granite_arg_converter`` carves the ``arguments`` object +out of each ``{"name":..,"arguments":{..}}`` element. Granite has no reasoning. +""" + +from __future__ import annotations + +import functools +from typing import TYPE_CHECKING + +from vllm.parser.engine.parser_engine import ParserEngine +from vllm.parser.engine.parser_engine_config import ( + ParserEngineConfig, + ParserState, + Transition, +) +from vllm.parser.utils import extract_json_member_object + +if TYPE_CHECKING: + from vllm.tokenizers import TokenizerLike + from vllm.tool_parsers.abstract_tool_parser import Tool + +TOOL_TOKEN = "<|tool_call|>" # granite 3.0 (single special token) +TOOL_STRING = "" # granite 3.1 (plain text) + + +def _granite_arg_converter(raw_args: str, partial: bool) -> str: + """Carve the ``arguments`` object out of a ``{"name":..,"arguments":{..}}`` + array element (see :func:`extract_json_member_object`).""" + span = extract_json_member_object(raw_args, "arguments") + if span is None: + return "" if partial else "{}" + return span + + +@functools.cache +def granite_config() -> ParserEngineConfig: + return ParserEngineConfig( + name="granite", + initial_state=ParserState.CONTENT, + terminals={ + "TOOL_TOKEN": TOOL_TOKEN, + "TOOL_STRING": TOOL_STRING, + }, + token_id_terminals={ + "TOOL_TOKEN": TOOL_TOKEN, + }, + transitions={ + # The marker only switches into the tool region; the array splitter + # emits TOOL_CALL_START per element (empty events here). + (ParserState.CONTENT, "TOOL_TOKEN"): Transition( + ParserState.TOOL_ARGS, + (), + ), + (ParserState.CONTENT, "TOOL_STRING"): Transition( + ParserState.TOOL_ARGS, + (), + ), + }, + arg_converter=_granite_arg_converter, + tool_call_body_array=True, + stream_arg_deltas=True, + ) + + +class GraniteParser(ParserEngine): + CONFIG_NAME = "granite" + + def __init__( + self, + tokenizer: TokenizerLike, + tools: list[Tool] | None = None, + **kwargs, + ) -> None: + kwargs.setdefault("parser_engine_config", granite_config()) + super().__init__(tokenizer, tools, **kwargs) diff --git a/vllm/parser/inkling.py b/vllm/parser/inkling.py index 2081fb2784..cc786c66dd 100644 --- a/vllm/parser/inkling.py +++ b/vllm/parser/inkling.py @@ -37,6 +37,7 @@ from vllm.parser.engine.parser_engine_config import ( ParserState, Transition, ) +from vllm.parser.utils import extract_json_member_object if TYPE_CHECKING: from vllm.tokenizers import TokenizerLike @@ -62,84 +63,6 @@ INKLING_SPECIAL_TOKENS = ( END_MESSAGE, ) -_WS = " \t\r\n" - - -def _scan_json_value(raw: str, start: int) -> int | None: - """Return the end index (exclusive) of the JSON object starting at - ``raw[start]``, or ``None`` when the object is still unterminated.""" - depth = 0 - in_string = False - escape = False - for i in range(start, len(raw)): - ch = raw[i] - if escape: - escape = False - continue - if in_string: - if ch == "\\": - escape = True - elif ch == '"': - in_string = False - continue - if ch == '"': - in_string = True - elif ch == "{": - depth += 1 - elif ch == "}": - depth -= 1 - if depth == 0: - return i + 1 - return None - - -def _args_value_span(raw: str) -> str | None: - """Extract the raw text span of the top-level ``"args"`` value from a - (possibly incomplete) ``{"name":...,"args":{...}}`` wrapper. - - Returns the verbatim substring (prefix-stable across growing input, - which the engine's argument-delta diffing relies on), possibly an - unterminated object prefix; ``None`` when the value has not started. - Raises ``ValueError`` when the value is not a JSON object. - """ - depth = 0 - in_string = False - escape = False - string_start = -1 - last_string: str | None = None - for i, ch in enumerate(raw): - if escape: - escape = False - continue - if in_string: - if ch == "\\": - escape = True - elif ch == '"': - in_string = False - if depth == 1: - last_string = raw[string_start + 1 : i] - continue - if ch == '"': - in_string = True - string_start = i - elif ch == ":" and depth == 1 and last_string == "args": - value_start = i + 1 - while value_start < len(raw) and raw[value_start] in _WS: - value_start += 1 - if value_start >= len(raw): - return None - if raw[value_start] != "{": - raise ValueError("Inkling tool call args must be a JSON object") - value_end = _scan_json_value(raw, value_start) - if value_end is None: - return raw[value_start:] - return raw[value_start:value_end] - elif ch in "{[": - depth += 1 - elif ch in "}]": - depth -= 1 - return None - def _inkling_arg_converter(raw_args: str, partial: bool) -> str: """Carve the ``args`` object out of the tool-call JSON wrapper. @@ -150,19 +73,10 @@ def _inkling_arg_converter(raw_args: str, partial: bool) -> str: converter, ``_compute_arg_delta`` streams the wrapper verbatim into the OpenAI ``arguments`` field (``converter is None -> raw delta``, unconditionally; ``stream_arg_deltas=False`` does not stop it). - - Why a hand-rolled scanner instead of (partial) ``json.loads`` + - ``json.dumps``: the engine diffs successive converter outputs and - requires each to extend the previous one (``startswith``); a - violation silently drops argument deltas. Re-serialization changes - whitespace and closes unterminated structures differently across - ticks, so the only prefix-stable output is a verbatim substring of - the input. The scanner is also string/escape-aware so an ``"args"`` - literal inside the name or a string value cannot mislead it, and it - still recovers a partial span when EOS truncates the wrapper (where - ``json.loads`` would fail). + :func:`extract_json_member_object` returns a prefix-stable verbatim + substring, which the engine's argument-delta diffing requires. """ - span = _args_value_span(raw_args) + span = extract_json_member_object(raw_args, "args") if span is None: # No args value yet (streaming) or none at all (treat as empty). return "" if partial else "{}" diff --git a/vllm/parser/utils.py b/vllm/parser/utils.py index 51382cd290..9194b7ef18 100644 --- a/vllm/parser/utils.py +++ b/vllm/parser/utils.py @@ -63,3 +63,88 @@ def count_history_tool_calls( return 0 return count_response_history_tool_calls(request_input) + + +_JSON_WS = " \t\r\n" + + +def scan_json_object_end(raw: str, start: int) -> int | None: + """Return the end index (exclusive) of the JSON object starting at + ``raw[start]`` (which must be ``{``), or ``None`` when it is still + unterminated. String- and escape-aware so braces inside strings do not + mislead the depth count.""" + depth = 0 + in_string = False + escape = False + for i in range(start, len(raw)): + ch = raw[i] + if escape: + escape = False + continue + if in_string: + if ch == "\\": + escape = True + elif ch == '"': + in_string = False + continue + if ch == '"': + in_string = True + elif ch == "{": + depth += 1 + elif ch == "}": + depth -= 1 + if depth == 0: + return i + 1 + return None + + +def extract_json_member_object(raw: str, key: str) -> str | None: + """Extract the raw text span of the top-level object-valued member ``key`` + from a (possibly incomplete) ``{...}`` JSON wrapper. + + Returns the verbatim substring (prefix-stable as the input grows, which + the engine's argument-delta diffing relies on), possibly an unterminated + object prefix; ``None`` when the value has not started. Raises + ``ValueError`` when the member is present but its value is not a JSON + object. A verbatim substring (not re-serialized JSON) is required because + the engine diffs successive converter outputs and expects each to extend + the previous one; the scanner is string/escape-aware so a matching literal + inside another value cannot mislead it, and it recovers a partial span when + the wrapper is truncated (where ``json.loads`` would fail).""" + depth = 0 + in_string = False + escape = False + string_start = -1 + last_string: str | None = None + for i, ch in enumerate(raw): + if escape: + escape = False + continue + if in_string: + if ch == "\\": + escape = True + elif ch == '"': + in_string = False + if depth == 1: + last_string = raw[string_start + 1 : i] + continue + if ch == '"': + in_string = True + string_start = i + elif ch == ":" and depth == 1 and last_string == key: + value_start = i + 1 + while value_start < len(raw) and raw[value_start] in _JSON_WS: + value_start += 1 + if value_start >= len(raw): + return None + if raw[value_start] != "{": + raise ValueError(f"member {key!r} must be a JSON object") + value_end = scan_json_object_end(raw, value_start) + if value_end is None: + return raw[value_start:] + return raw[value_start:value_end] + elif ch in "{[": + depth += 1 + elif ch in "}]": + depth -= 1 + return None diff --git a/vllm/tool_parsers/__init__.py b/vllm/tool_parsers/__init__.py index c43a34772f..2515009d0e 100644 --- a/vllm/tool_parsers/__init__.py +++ b/vllm/tool_parsers/__init__.py @@ -75,8 +75,8 @@ _TOOL_PARSERS_TO_REGISTER = { "Granite20bFCToolParser", ), "granite": ( - "granite_tool_parser", - "GraniteToolParser", + "granite_engine_tool_parser", + "GraniteEngineToolParser", ), "granite4": ( "granite4_tool_parser", diff --git a/vllm/tool_parsers/granite_engine_tool_parser.py b/vllm/tool_parsers/granite_engine_tool_parser.py new file mode 100644 index 0000000000..1fb701d5a4 --- /dev/null +++ b/vllm/tool_parsers/granite_engine_tool_parser.py @@ -0,0 +1,8 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +from vllm.parser.engine.registered_adapters import GraniteParserToolAdapter + + +class GraniteEngineToolParser(GraniteParserToolAdapter): # type: ignore[valid-type, misc] + structural_tag_model = None diff --git a/vllm/tool_parsers/granite_tool_parser.py b/vllm/tool_parsers/granite_tool_parser.py deleted file mode 100644 index 1e51c3227e..0000000000 --- a/vllm/tool_parsers/granite_tool_parser.py +++ /dev/null @@ -1,257 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project - -import json -from collections.abc import Sequence - -import partial_json_parser -from partial_json_parser.core.options import Allow - -from vllm.entrypoints.chat_utils import make_tool_call_id -from vllm.entrypoints.generate.base.protocol import ( - DeltaFunctionCall, - DeltaMessage, - DeltaToolCall, - ExtractedToolCallInformation, - FunctionCall, - ToolCall, -) -from vllm.entrypoints.openai.chat_completion.protocol import ( - ChatCompletionRequest, -) -from vllm.logger import init_logger -from vllm.tokenizers import TokenizerLike -from vllm.tool_parsers.abstract_tool_parser import ( - Tool, - ToolParser, -) -from vllm.tool_parsers.utils import ( - consume_space, - find_common_prefix, - is_complete_json, - partial_json_loads, -) - -logger = init_logger(__name__) - - -class GraniteToolParser(ToolParser): - """Tool call parser for the granite 3.0 models. Intended - for use with the examples/tool_chat_template_granite.jinja - template. - - Used when --enable-auto-tool-choice --tool-call-parser granite - are all set - """ - - def __init__(self, tokenizer: TokenizerLike, tools: list[Tool] | None = None): - super().__init__(tokenizer, tools) - # for granite 3.0, the token `<|tool_call|>` - self.bot_token = "<|tool_call|>" - # for granite 3.1, the string `` - self.bot_string = "" - - def extract_tool_calls( - self, model_output: str, request: ChatCompletionRequest - ) -> ExtractedToolCallInformation: - stripped = ( - model_output.strip() - .removeprefix(self.bot_token) - .removeprefix(self.bot_string) - .lstrip() - ) - if not stripped or stripped[0] != "[": - return ExtractedToolCallInformation( - tools_called=False, tool_calls=[], content=model_output - ) - try: - raw_function_calls = json.loads(stripped) - if not isinstance(raw_function_calls, list): - raise Exception( - f"Expected dict or list, got {type(raw_function_calls)}" - ) - - logger.debug("Extracted %d tool calls", len(raw_function_calls)) - tool_calls = [ - ToolCall( - type="function", - function=FunctionCall( - name=function_call["name"], - # function call args are JSON but as a string - arguments=json.dumps( - function_call["arguments"], ensure_ascii=False - ), - ), - ) - for function_call in raw_function_calls - ] - - return ExtractedToolCallInformation( - tools_called=True, - tool_calls=tool_calls, - content=None, - ) - - except Exception as e: - logger.error("Error in extracting tool call from response %s", e) - return ExtractedToolCallInformation( - tools_called=False, tool_calls=[], content=model_output - ) - - def extract_tool_calls_streaming( - self, - previous_text: str, - current_text: str, - delta_text: str, - previous_token_ids: Sequence[int], - current_token_ids: Sequence[int], - delta_token_ids: Sequence[int], - request: ChatCompletionRequest, - ) -> DeltaMessage | None: - start_idx = consume_space(0, current_text) - if current_text[start_idx:].startswith(self.bot_token): - start_idx = consume_space(start_idx + len(self.bot_token), current_text) - if current_text[start_idx:].startswith(self.bot_string): - start_idx = consume_space(start_idx + len(self.bot_string), current_text) - if ( - not current_text - or start_idx >= len(current_text) - or current_text[start_idx] != "[" - ): - return DeltaMessage(content=delta_text) - - # bit mask flags for partial JSON parsing. If the name hasn't been - # sent yet, don't allow sending - # an incomplete string since OpenAI only ever (as far as I have - # seen) allows sending the entire tool/ function name at once. - flags = Allow.ALL if self.current_tool_name_sent else Allow.ALL & ~Allow.STR - try: - tool_call_arr = None - is_complete = None - try: - tool_calls, end_idx = partial_json_loads( - current_text[start_idx:], flags - ) - if type(tool_calls) is list: - tool_call_arr = tool_calls - else: - return DeltaMessage(content=delta_text) - - is_complete = [True] * len(tool_calls) - if not is_complete_json(current_text[start_idx : start_idx + end_idx]): - is_complete[-1] = False - except partial_json_parser.core.exceptions.MalformedJSON: - logger.debug("not enough tokens to parse into JSON yet") - return None - - # case -- if no tokens have been streamed for the tool, e.g. - # only the array brackets, stream nothing - if not tool_call_arr: - return None - - # select as the current tool call the one we're on the state at - current_tool_call: dict = tool_call_arr[self.current_tool_id] - - delta = None - # Only advance once the current tool name is streamed; granite - # emits arguments before name, so advancing early would drop it. - if len(tool_call_arr) > self.current_tool_id + 1 and ( - self.current_tool_id < 0 or self.current_tool_name_sent - ): - # if we're moving on to a new call, first make sure we - # haven't missed anything in the previous one that was - # auto-generated due to JSON completions, but wasn't - # streamed to the client yet. - if self.current_tool_id >= 0: - cur_arguments = current_tool_call.get("arguments") - if cur_arguments: - cur_args_json = json.dumps(cur_arguments, ensure_ascii=False) - sent = len(self.streamed_args_for_tool[self.current_tool_id]) - argument_diff = cur_args_json[sent:] - - logger.debug("got arguments diff: %s", argument_diff) - delta = DeltaMessage( - tool_calls=[ - DeltaToolCall( - index=self.current_tool_id, - function=DeltaFunctionCall( - arguments=argument_diff - ).model_dump(exclude_none=True), - ) - ] - ) - self.streamed_args_for_tool[self.current_tool_id] += ( - argument_diff - ) - - # re-set stuff pertaining to progress in the current tool - self.current_tool_id += 1 - self.current_tool_name_sent = False - self.streamed_args_for_tool.append("") - logger.debug("starting on new tool %d", self.current_tool_id) - return delta - - # if the current tool name hasn't been sent, send if available - # - otherwise send nothing - elif not self.current_tool_name_sent: - function_name = current_tool_call.get("name") - if function_name: - delta = DeltaMessage( - tool_calls=[ - DeltaToolCall( - index=self.current_tool_id, - type="function", - id=make_tool_call_id(), - function=DeltaFunctionCall( - name=function_name - ).model_dump(exclude_none=True), - ) - ] - ) - self.current_tool_name_sent = True - - # now we know we're on the same tool call and we're streaming - # arguments - else: - cur_arguments = current_tool_call.get("arguments") - - if cur_arguments: - sent = len(self.streamed_args_for_tool[self.current_tool_id]) - cur_args_json = json.dumps(cur_arguments, ensure_ascii=False) - prev_arguments = self.prev_tool_call_arr[self.current_tool_id].get( - "arguments" - ) - - argument_diff = None - if is_complete[self.current_tool_id]: - argument_diff = cur_args_json[sent:] - elif prev_arguments: - prev_args_json = json.dumps(prev_arguments, ensure_ascii=False) - if cur_args_json != prev_args_json: - prefix = find_common_prefix(prev_args_json, cur_args_json) - argument_diff = prefix[sent:] - - if argument_diff is not None: - delta = DeltaMessage( - tool_calls=[ - DeltaToolCall( - index=self.current_tool_id, - function=DeltaFunctionCall( - arguments=argument_diff - ).model_dump(exclude_none=True), - ) - ] - ) - self.streamed_args_for_tool[self.current_tool_id] += ( - argument_diff - ) - - self.prev_tool_call_arr = tool_call_arr - return delta - - except Exception as e: - logger.error("Error trying to handle streaming tool call: %s", e) - logger.debug( - "Skipping chunk as a result of tool streaming extraction error" - ) - return None