[Bugfix] Drop incomplete tool-call markup in non-streaming to match streaming (#47562)
Signed-off-by: Suvrakamal Das <jyb2084.usa@gmail.com> Co-authored-by: Claude <noreply@anthropic.com> Co-authored-by: Ben Browning <56071+bbrowning@users.noreply.github.com>
This commit is contained in:
co-authored by
Claude
Ben Browning
parent
2f01039666
commit
e7cf4730d6
@@ -869,3 +869,113 @@ def test_content_tool_start_emits_reasoning_end_in_reasoning_pass():
|
||||
EventType.TEXT_CHUNK,
|
||||
]
|
||||
assert events[1].value == TOOL_JSON
|
||||
|
||||
|
||||
def test_skip_reasoning_parsing_inert_for_shared_markers():
|
||||
"""``skip_reasoning_parsing`` may only bypass reasoning-exclusive
|
||||
markers. Inkling has none — ``<|end_message|>`` is labelled THINK_END
|
||||
yet also closes text, header, and tool blocks — so the flag must be
|
||||
inert: a tool block still closes through its transition and following
|
||||
text returns to CONTENT instead of leaking into the argument stream."""
|
||||
engine = StreamingParserEngine(inkling_config(), tokenizer=None)
|
||||
engine.skip_reasoning_parsing = True
|
||||
engine.reset(initial_state=ParserState.CONTENT)
|
||||
events = engine.parse_complete(
|
||||
f'{TOOL_JSON}{{"name":"f","args":{{}}}}{END_MESSAGE}after'
|
||||
)
|
||||
types = [e.type for e in events]
|
||||
end_idx = types.index(EventType.TOOL_CALL_END)
|
||||
text_after = [e.value for e in events[end_idx:] if e.type == EventType.TEXT_CHUNK]
|
||||
assert text_after == ["after"]
|
||||
|
||||
|
||||
class TestToolParserWithoutReasoningParser:
|
||||
"""Inkling served with only the tool parser, no ``--reasoning-parser``.
|
||||
|
||||
That configuration turns on ``skip_reasoning_parsing``, and Inkling
|
||||
labels ``<|end_message|>`` as THINK_END; a bypass keyed on the label
|
||||
would neutralize the closer of every block kind, leaking markers into
|
||||
content and leaving tool calls unterminated. Structure must keep
|
||||
parsing exactly as with the reasoning parser attached."""
|
||||
|
||||
GEN_PROMPT = [_TML_VOCAB[MSG_MODEL]]
|
||||
|
||||
def _tool_only(self, mock_tokenizer, tools=None):
|
||||
parser_cls = ParserManager.get_parser(
|
||||
tool_parser_name="inkling",
|
||||
enable_auto_tools=True,
|
||||
)
|
||||
return parser_cls(mock_tokenizer, tools or [])
|
||||
|
||||
def test_plain_text_non_streaming(self, mock_tokenizer, mock_request):
|
||||
tools = [_function_tool()]
|
||||
mock_request.tools = tools
|
||||
_, content, calls = self._tool_only(mock_tokenizer, tools).parse(
|
||||
f"{TEXT_START}The answer is 42.{END_MESSAGE}",
|
||||
mock_request,
|
||||
enable_auto_tools=True,
|
||||
)
|
||||
assert content == "The answer is 42."
|
||||
assert not calls
|
||||
|
||||
@pytest.mark.parametrize("chunk_size", [1, 3, 64])
|
||||
def test_plain_text_streaming(self, mock_tokenizer, mock_request, chunk_size):
|
||||
tools = [_function_tool()]
|
||||
mock_request.tools = tools
|
||||
content, _, names, _ = _stream_delegating(
|
||||
self._tool_only(mock_tokenizer, tools),
|
||||
mock_request,
|
||||
f"{TEXT_START}The answer is 42.{END_MESSAGE}",
|
||||
chunk_size,
|
||||
self.GEN_PROMPT,
|
||||
)
|
||||
assert content == "The answer is 42."
|
||||
assert not names
|
||||
|
||||
def test_tool_block_non_streaming(self, mock_tokenizer, mock_request):
|
||||
_, content, calls = self._tool_only(mock_tokenizer).parse(
|
||||
_tool_block("get_weather", '{"city":"SF"}'),
|
||||
mock_request,
|
||||
enable_auto_tools=True,
|
||||
)
|
||||
assert [c.name for c in calls] == ["get_weather"]
|
||||
assert not content
|
||||
|
||||
@pytest.mark.parametrize("chunk_size", [1, 3, 64])
|
||||
def test_tool_block_streaming(self, mock_tokenizer, mock_request, chunk_size):
|
||||
tools = [_function_tool()]
|
||||
mock_request.tools = tools
|
||||
content, _, names, args = _stream_delegating(
|
||||
self._tool_only(mock_tokenizer, tools),
|
||||
mock_request,
|
||||
_tool_block("get_weather", '{"city":"SF"}'),
|
||||
chunk_size,
|
||||
self.GEN_PROMPT,
|
||||
)
|
||||
assert names == ["get_weather"]
|
||||
assert json.loads(args[0]) == {"city": "SF"}
|
||||
assert content == ""
|
||||
|
||||
def test_thinking_then_text_keeps_structure(self, mock_tokenizer, mock_request):
|
||||
tools = [_function_tool()]
|
||||
mock_request.tools = tools
|
||||
_, content, calls = self._tool_only(mock_tokenizer, tools).parse(
|
||||
f"{THINK_START}plan{END_MESSAGE}"
|
||||
f"{MSG_MODEL}{TEXT_START}Let me look.{END_MESSAGE}",
|
||||
mock_request,
|
||||
enable_auto_tools=True,
|
||||
)
|
||||
assert content == "Let me look."
|
||||
assert END_MESSAGE not in content
|
||||
assert THINK_START not in content
|
||||
assert not calls
|
||||
|
||||
def test_thinking_then_tool_still_promotes(self, mock_tokenizer, mock_request):
|
||||
_, content, calls = self._tool_only(mock_tokenizer).parse(
|
||||
f"{THINK_START}think{END_MESSAGE}{MSG_MODEL}"
|
||||
+ _tool_block("get_weather", '{"x":1}'),
|
||||
mock_request,
|
||||
enable_auto_tools=True,
|
||||
)
|
||||
assert [c.name for c in calls] == ["get_weather"]
|
||||
assert not content
|
||||
|
||||
@@ -1909,3 +1909,200 @@ class TestDropSpecialTokens:
|
||||
e.value for e in events if e.type == EventType.REASONING_CHUNK
|
||||
)
|
||||
assert "<bos>" not in reasoning_text
|
||||
|
||||
|
||||
# ── TestTruncatedToolOpenerStreamParity ──────────────────────────────
|
||||
|
||||
|
||||
class TestTruncatedToolOpenerStreamParity:
|
||||
"""Regression tests for #47137: when generation terminates inside a
|
||||
``<tool_call>`` opener that has not been promoted to a tool call
|
||||
(via ``max_tokens`` or a ``stop`` string), the non-streaming path
|
||||
must drop the incomplete markup, matching the streaming path."""
|
||||
|
||||
_QWEN3_VOCAB = {
|
||||
"<tool_call>": 100,
|
||||
"</tool_call>": 101,
|
||||
}
|
||||
|
||||
def _make_parser(self):
|
||||
from vllm.parser.parser_manager import ParserManager
|
||||
|
||||
parser_cls = ParserManager.get_parser(
|
||||
tool_parser_name="qwen3_coder",
|
||||
enable_auto_tools=True,
|
||||
)
|
||||
tokenizer = make_mock_tokenizer(self._QWEN3_VOCAB)
|
||||
return parser_cls(tokenizer, [])
|
||||
|
||||
def _stream_content(self, request, chunks: list[str]) -> str:
|
||||
parser = self._make_parser()
|
||||
content = ""
|
||||
for i, chunk in enumerate(chunks):
|
||||
delta_token_ids = [
|
||||
tid for text, tid in self._QWEN3_VOCAB.items() if text in chunk
|
||||
]
|
||||
delta = parser.parse_delta(
|
||||
chunk,
|
||||
delta_token_ids,
|
||||
request,
|
||||
prompt_token_ids=[1],
|
||||
finished=(i == len(chunks) - 1),
|
||||
)
|
||||
if delta and delta.content:
|
||||
content += delta.content
|
||||
return content
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"chunks",
|
||||
[
|
||||
["<tool_call>"],
|
||||
["<tool_call>", "\n"],
|
||||
["<tool_call>", "\n", "<"],
|
||||
["<tool_call>", "\n", "<function"],
|
||||
],
|
||||
)
|
||||
def test_truncated_opener_dropped_in_both_paths(self, mock_request, chunks):
|
||||
"""A cutoff inside the opener yields no content and no tool calls
|
||||
in both the non-streaming and streaming paths."""
|
||||
text = "".join(chunks)
|
||||
|
||||
_, content, tool_calls = self._make_parser().parse(
|
||||
text, mock_request, enable_auto_tools=True
|
||||
)
|
||||
streamed = self._stream_content(mock_request, chunks)
|
||||
|
||||
assert not tool_calls
|
||||
assert (content or "") == streamed == ""
|
||||
|
||||
def test_content_before_truncated_opener_preserved(self, mock_request):
|
||||
"""Only the incomplete markup is dropped; content generated before
|
||||
the opener is returned identically by both paths."""
|
||||
chunks = ["Checking the weather. ", "<tool_call>", "\n", "<function"]
|
||||
text = "".join(chunks)
|
||||
|
||||
_, content, tool_calls = self._make_parser().parse(
|
||||
text, mock_request, enable_auto_tools=True
|
||||
)
|
||||
streamed = self._stream_content(mock_request, chunks)
|
||||
|
||||
assert not tool_calls
|
||||
assert content == streamed == "Checking the weather. "
|
||||
|
||||
def test_content_around_unpromoted_tool_block_stays_ordered(self, mock_request):
|
||||
"""Text surrounding a complete-but-unparsable tool block keeps its
|
||||
original order in the non-streaming path, matching streaming."""
|
||||
chunks = ["A ", "<tool_call>", "garbage", "</tool_call>", " B"]
|
||||
text = "".join(chunks)
|
||||
|
||||
_, content, tool_calls = self._make_parser().parse(
|
||||
text, mock_request, enable_auto_tools=True
|
||||
)
|
||||
streamed = self._stream_content(mock_request, chunks)
|
||||
|
||||
assert not tool_calls
|
||||
assert content == streamed == "A B"
|
||||
|
||||
def test_complete_tool_call_still_promoted(self, mock_request):
|
||||
"""Sanity check: a complete tool call still parses in the
|
||||
non-streaming path after the truncation fix."""
|
||||
text = (
|
||||
"<tool_call>\n"
|
||||
"<function=get_weather>\n"
|
||||
"<parameter=city>Tokyo</parameter>\n"
|
||||
"</function>\n"
|
||||
"</tool_call>"
|
||||
)
|
||||
_, content, tool_calls = self._make_parser().parse(
|
||||
text, mock_request, enable_auto_tools=True
|
||||
)
|
||||
assert tool_calls is not None and len(tool_calls) == 1
|
||||
assert tool_calls[0].name == "get_weather"
|
||||
assert not content
|
||||
|
||||
|
||||
class TestThinkMarkupWithoutReasoningParser:
|
||||
"""With no reasoning parser configured, reasoning markup is plain
|
||||
content: engine-backed tool parsers must return it verbatim in both
|
||||
the non-streaming and streaming paths, not consume the markers or
|
||||
reclassify the block as reasoning and drop it."""
|
||||
|
||||
_VOCAB = {
|
||||
"<think>": 90,
|
||||
"</think>": 91,
|
||||
"<tool_call>": 100,
|
||||
"</tool_call>": 101,
|
||||
}
|
||||
|
||||
# Two distinct failure modes are covered: qwen3_coder consumed a
|
||||
# stray </think> marker in CONTENT state (unbalanced markup),
|
||||
# deepseek_v4 re-entered REASONING on <think> and dropped the block
|
||||
# from non-streaming content entirely.
|
||||
_PARSERS = ["qwen3_coder", "deepseek_v4"]
|
||||
|
||||
def _make_parser(self, tool_parser_name):
|
||||
from vllm.parser.parser_manager import ParserManager
|
||||
|
||||
parser_cls = ParserManager.get_parser(
|
||||
tool_parser_name=tool_parser_name,
|
||||
enable_auto_tools=True,
|
||||
)
|
||||
tokenizer = make_mock_tokenizer(self._VOCAB)
|
||||
return parser_cls(tokenizer, [])
|
||||
|
||||
def _stream(self, tool_parser_name, request, chunks):
|
||||
parser = self._make_parser(tool_parser_name)
|
||||
content, reasoning = "", ""
|
||||
for i, chunk in enumerate(chunks):
|
||||
delta_token_ids = [
|
||||
tid for text, tid in self._VOCAB.items() if text in chunk
|
||||
]
|
||||
delta = parser.parse_delta(
|
||||
chunk,
|
||||
delta_token_ids,
|
||||
request,
|
||||
prompt_token_ids=[1],
|
||||
finished=(i == len(chunks) - 1),
|
||||
)
|
||||
if delta and delta.content:
|
||||
content += delta.content
|
||||
if delta and delta.reasoning:
|
||||
reasoning += delta.reasoning
|
||||
return content, reasoning
|
||||
|
||||
@pytest.mark.parametrize("tool_parser_name", _PARSERS)
|
||||
def test_think_block_passes_through_both_paths(
|
||||
self, mock_request, tool_parser_name
|
||||
):
|
||||
chunks = ["<think>", "reasoning here", "</think>", "the answer"]
|
||||
text = "".join(chunks)
|
||||
|
||||
reasoning, content, tool_calls = self._make_parser(tool_parser_name).parse(
|
||||
text, mock_request, enable_auto_tools=True
|
||||
)
|
||||
streamed_content, streamed_reasoning = self._stream(
|
||||
tool_parser_name, mock_request, chunks
|
||||
)
|
||||
|
||||
assert not tool_calls
|
||||
assert reasoning is None
|
||||
assert streamed_reasoning == ""
|
||||
assert content == streamed_content == text
|
||||
|
||||
def test_think_block_preserved_alongside_tool_call(self, mock_request):
|
||||
"""The tools-called branch also returns the tool parser's content;
|
||||
a think block preceding a promoted tool call must survive in it."""
|
||||
think = "<think>plan</think>"
|
||||
text = think + (
|
||||
"<tool_call>\n"
|
||||
"<function=get_weather>\n"
|
||||
"<parameter=city>Tokyo</parameter>\n"
|
||||
"</function>\n"
|
||||
"</tool_call>"
|
||||
)
|
||||
_, content, tool_calls = self._make_parser("qwen3_coder").parse(
|
||||
text, mock_request, enable_auto_tools=True
|
||||
)
|
||||
assert tool_calls is not None and len(tool_calls) == 1
|
||||
assert tool_calls[0].name == "get_weather"
|
||||
assert content == think
|
||||
|
||||
Reference in New Issue
Block a user