[Bugfix][Parser] Prevent Inkling block-end leakage with tools (#51391)

Signed-off-by: cherry77-cloud <1615405@qq.com>
Co-authored-by: Ben Browning <56071+bbrowning@users.noreply.github.com>
Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
cherry77-cloud
2026-08-08 12:34:32 -04:00
committed by GitHub
co-authored by Ben Browning Claude Opus 4.8
parent 58d3918e3e
commit 75231eff2f
3 changed files with 352 additions and 30 deletions
+212
View File
@@ -19,6 +19,10 @@ from tests.parser.engine.streaming_helpers import (
collect_function_name,
collect_tool_arguments,
)
from vllm.entrypoints.openai.chat_completion.protocol import (
ChatCompletionToolsParam,
FunctionDefinition,
)
from vllm.parser.engine.parser_engine_config import ParserState
from vllm.parser.inkling import InklingParser, _inkling_arg_converter
from vllm.parser.parser_manager import ParserManager
@@ -140,6 +144,51 @@ def _collect_reasoning(results) -> str:
return "".join(d.reasoning for d, _ in results if d and d.reasoning)
def _function_tool(name: str = "get_weather") -> ChatCompletionToolsParam:
"""A real function-tool definition, as a request would carry it."""
return ChatCompletionToolsParam(
function=FunctionDefinition(
name=name,
parameters={"type": "object", "properties": {}},
),
)
def _delegating(mock_tokenizer, tools=None):
"""Build the served reasoning+tool DelegatingParser for Inkling."""
parser_cls = ParserManager.get_parser(
tool_parser_name="inkling",
reasoning_parser_name="inkling",
enable_auto_tools=True,
)
return parser_cls(mock_tokenizer, tools or [])
def _stream_delegating(parser, request, text, chunk_size, prompt_token_ids):
"""Stream ``text`` through ``DelegatingParser.parse_delta``, ``chunk_size``
tokens per delta; return ``(content, reasoning, ordered tool names)``."""
tokens = _tokenize(text)
content, reasoning, tools = "", "", {}
for start in range(0, len(tokens), chunk_size):
batch = tokens[start : start + chunk_size]
delta = parser.parse_delta(
"".join(t for _, t in batch),
[tid for tid, _ in batch],
request,
prompt_token_ids=prompt_token_ids if start == 0 else None,
finished=(start + chunk_size >= len(tokens)),
)
if delta and delta.content:
content += delta.content
if delta and delta.reasoning:
reasoning += delta.reasoning
if delta and delta.tool_calls:
for tc in delta.tool_calls:
if tc.function and tc.function.name:
tools[tc.index] = tc.function.name
return content, reasoning, [tools[k] for k in sorted(tools)]
class TestArgConverter:
def test_complete_wrapper(self):
raw = '{"name":"get_weather","args":{"city":"SF"}}'
@@ -533,3 +582,166 @@ class TestRegisteredAdapters:
assert result.tools_called
assert result.tool_calls[0].function.name == "f"
assert json.loads(result.tool_calls[0].function.arguments) == {"a": 1}
class TestDelegatingTwoPass:
"""Served reasoning+tool ``DelegatingParser`` path (issue #51387).
Unlike ``TestStreaming`` above (single engine, ``skip_tool_parsing=False``),
these drive the real two-pass parser, where the reasoning pass runs with
``skip_tool_parsing=True`` — the path where the trailing-marker leak lived.
The bug is specific to the *tools-enabled* plain-text/reasoning path, so the
content-only cases (which emit no tool call and would still pass in a
no-tools mode) offer a real function tool on both the request and the
parser, keeping ``tool_choice="auto"``.
"""
GEN_PROMPT = [_TML_VOCAB[MSG_MODEL]]
def test_plain_text_non_streaming(self, mock_tokenizer, mock_request):
tools = [_function_tool()]
mock_request.tools = tools
_, content, calls = _delegating(mock_tokenizer, tools).parse(
f"{TEXT_START}The answer is 42.{END_MESSAGE}",
mock_request,
enable_auto_tools=True,
)
assert content == "The answer is 42."
assert END_MESSAGE not in content
assert not calls
@pytest.mark.parametrize("chunk_size", [1, 3, 7, 64])
def test_plain_text_streaming(self, mock_tokenizer, mock_request, chunk_size):
tools = [_function_tool()]
mock_request.tools = tools
content, _, calls = _stream_delegating(
_delegating(mock_tokenizer, tools),
mock_request,
f"{TEXT_START}The answer is 42.{END_MESSAGE}",
chunk_size,
self.GEN_PROMPT,
)
assert content == "The answer is 42."
assert END_MESSAGE not in content
assert not calls
def test_reasoning_then_text(self, mock_tokenizer, mock_request):
tools = [_function_tool()]
mock_request.tools = tools
reasoning, content, _ = _delegating(mock_tokenizer, tools).parse(
f"{THINK_START}plan{END_MESSAGE}"
f"{MSG_MODEL}{TEXT_START}Let me look.{END_MESSAGE}",
mock_request,
enable_auto_tools=True,
)
assert reasoning == "plan"
assert content == "Let me look."
assert END_MESSAGE not in content
def test_multi_tool_round_trip(self, mock_tokenizer, mock_request):
_, _, tools = _delegating(mock_tokenizer).parse(
_tool_block("f", '{"a":1}') + MSG_MODEL + _tool_block("g", '{"b":2}'),
mock_request,
enable_auto_tools=True,
)
assert [t.name for t in tools] == ["f", "g"]
def test_text_then_tool(self, mock_tokenizer, mock_request):
_, content, tools = _delegating(mock_tokenizer).parse(
f"{TEXT_START}intro{END_MESSAGE}{MSG_MODEL}" + _tool_block("f", "{}"),
mock_request,
enable_auto_tools=True,
)
assert content == "intro"
assert [t.name for t in tools] == ["f"]
def test_tool_then_text(self, mock_tokenizer, mock_request):
_, content, tools = _delegating(mock_tokenizer).parse(
_tool_block("f", "{}") + MSG_MODEL + f"{TEXT_START}done{END_MESSAGE}",
mock_request,
enable_auto_tools=True,
)
assert content == "done"
assert [t.name for t in tools] == ["f"]
def test_two_tools_then_text(self, mock_tokenizer, mock_request):
_, content, tools = _delegating(mock_tokenizer).parse(
_tool_block("f", "{}")
+ MSG_MODEL
+ _tool_block("g", "{}")
+ MSG_MODEL
+ f"{TEXT_START}after{END_MESSAGE}",
mock_request,
enable_auto_tools=True,
)
assert content == "after"
assert [t.name for t in tools] == ["f", "g"]
def test_reasoning_then_tool_non_streaming(self, mock_tokenizer, mock_request):
reasoning, _, tools = _delegating(mock_tokenizer).parse(
f"{THINK_START}think{END_MESSAGE}{MSG_MODEL}" + _tool_block("f", '{"x":1}'),
mock_request,
enable_auto_tools=True,
)
assert reasoning == "think"
assert [t.name for t in tools] == ["f"]
def test_end_sampling_text_closer_consumed(self, mock_tokenizer, mock_request):
tools = [_function_tool()]
mock_request.tools = tools
_, content, calls = _delegating(mock_tokenizer, tools).parse(
f"{TEXT_START}hi{END_SAMPLING}",
mock_request,
enable_auto_tools=True,
)
assert content == "hi"
assert END_SAMPLING not in content
assert not calls
def test_end_sampling_tool_closer_round_trips(self, mock_tokenizer, mock_request):
_, _, tools = _delegating(mock_tokenizer).parse(
f'{TOOL_JSON}{{"name":"f","args":{{}}}}{END_SAMPLING}',
mock_request,
enable_auto_tools=True,
)
assert [t.name for t in tools] == ["f"]
def test_incomplete_tool_at_eos(self, mock_tokenizer, mock_request):
_, _, tools = _delegating(mock_tokenizer).parse(
f'{TOOL_JSON}{{"name":"d","args":{{"k":"v"',
mock_request,
enable_auto_tools=True,
)
assert [t.name for t in tools] == ["d"]
def test_reset_reuse_after_incomplete_span(self, mock_tokenizer, mock_request):
parser = _delegating(mock_tokenizer)
parser.parse(
f'{TOOL_JSON}{{"name":"d","args":{{"k":"v"',
mock_request,
enable_auto_tools=True,
)
_, content, _ = parser.parse(
f"{TEXT_START}fresh{END_MESSAGE}",
mock_request,
enable_auto_tools=True,
)
assert content == "fresh"
assert END_MESSAGE not in content
@pytest.mark.parametrize("chunk_size", [1, 3, 64])
def test_reasoning_then_tool_streaming(
self, mock_tokenizer, mock_request, chunk_size
):
content, _, tools = _stream_delegating(
_delegating(mock_tokenizer),
mock_request,
f"{THINK_START}plan{END_MESSAGE}{MSG_MODEL}"
+ _tool_block("get_weather", '{"city":"SF"}'),
chunk_size,
self.GEN_PROMPT,
)
assert tools == ["get_weather"]
assert TOOL_JSON not in content
assert END_MESSAGE not in content