Add pydocstyle to the ruff rules (#52136)

Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
Signed-off-by: Harry Mellor <hej.mellor@gmail.com>
Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Harry Mellor
2026-09-16 16:44:29 +00:00
committed by GitHub
co-authored by Claude Opus 5
parent 804e5377aa
commit 8c1557a79c
2002 changed files with 15024 additions and 14269 deletions
+2 -1
View File
@@ -1,7 +1,8 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Reasoning extraction, reasoning-end gating and parser selection for the
unified Cohere Command parser."""
unified Cohere Command parser.
"""
from __future__ import annotations
+6 -3
View File
@@ -76,7 +76,8 @@ def token_deltas(
) -> list[tuple[str, list[int]]]:
"""Split ``text`` into ``(delta_text, delta_token_ids)`` steps of
``chunk_size`` tokens, buffering incomplete multi-byte sequences (trailing
U+FFFD) into the next step as real streaming does."""
U+FFFD) into the next step as real streaming does.
"""
ids = tokenizer.encode(text)
deltas: list[tuple[str, list[int]]] = []
prev = ""
@@ -100,7 +101,8 @@ def make_parser(
model_config=None,
) -> Parser:
"""Resolve the unified Cohere parser through ``ParserManager`` by registry
name (``cohere_command3`` / ``cohere_command4``)."""
name (``cohere_command3`` / ``cohere_command4``).
"""
cls = ParserManager.get_parser(name, name, enable_auto_tools=True)
assert cls is not None
return cls(
@@ -117,7 +119,8 @@ def drive_parser(
deltas: list[tuple[str, list[int]]],
) -> list[DeltaMessage]:
"""Feed ``deltas`` through ``parse_delta`` (flushing on the last one) and
return the non-``None`` messages in order."""
return the non-``None`` messages in order.
"""
out: list[DeltaMessage] = []
for i, (text, ids) in enumerate(deltas):
delta = parser.parse_delta(text, ids, request, finished=i == len(deltas) - 1)
+1
View File
@@ -30,6 +30,7 @@ def make_mock_tokenizer(
The returned mock supports get_vocab(), encode(), and decode().
decode() maps known token IDs back to their text and falls back to
chr(id) for ASCII IDs or ``<id>`` for others.
"""
id_to_text = {v: k for k, v in vocab.items()}
tokenizer = MagicMock()
+1
View File
@@ -183,6 +183,7 @@ def replay_streaming(
Returns:
List of ``DeltaMessage`` results from each ``parse_delta()`` call.
"""
if chunk_size is None:
chunk_size = len(tokens)
+2 -1
View File
@@ -289,7 +289,8 @@ class TestStreaming:
class TestMissingFunctionCallsWrapper:
"""An invoke without the ``<|DSML|function_calls>`` wrapper is still a
tool call (V3.2 counterpart of #48931)."""
tool call (V3.2 counterpart of #48931).
"""
def test_non_streaming(self, mock_tokenizer, mock_request):
text = _invoke("fn", _param("k", "true", "v"))
+12 -6
View File
@@ -334,7 +334,8 @@ class TestMissingToolCallsWrapper:
):
"""Text after a tool block is dropped whether or not the closing
wrapper is present, so a missing ``</|DSML|tool_calls>`` does not
change what the client sees."""
change what the client sees.
"""
text = "pre\n" + self._ORPHAN
if trailing_end:
text += DSML_TOOL_END
@@ -915,7 +916,8 @@ class TestParallelUnwrapping:
class TestStreamingWrapperConsistency:
"""Streamed arg deltas must stay consistent with final extraction
when wrapper params like 'arguments' are unwrapped."""
when wrapper params like 'arguments' are unwrapped.
"""
def test_streaming_wrapper_unwrap_consistency(self, mock_tokenizer, mock_request):
tool = _make_tool("get_weather", {"location": {"type": "string"}})
@@ -1196,7 +1198,8 @@ class TestDelegatingParserLargeDelta:
)
def test_eos_not_leaked_when_reasoning_never_ends(self, chunk_size):
"""EOS must not leak into reasoning_content when the model never
emits </think> (generation ends while still in REASONING state)."""
emits </think> (generation ends while still in REASONING state).
"""
eos_text = "<|end▁of▁sentence|>"
eos_id = 128801
vocab = {
@@ -1285,7 +1288,8 @@ class TestMalformedDsmlNoise:
self, mock_tokenizer, mock_request, wrapper
):
"""Like the real wrapper, a corrupted one inside ``<think>`` closes
the reasoning block and starts the tool call."""
the reasoning block and starts the tool call.
"""
text = f"thinking{wrapper}\n{self._INNER}"
parser = self._parser(mock_tokenizer, thinking=True)
reasoning, content = parser.extract_reasoning(text, mock_request)
@@ -1329,7 +1333,8 @@ class TestMalformedDsmlNoise:
prose mention of the real wrapper: the opener starts a tool block and
the rest of the message is dropped. That is pre-existing behavior of
``<|DSML|tool_calls>``; the variants must not differ from it either
way."""
way.
"""
template = "The opener looks like {} and then params follow. Done."
text = template.format(wrapper)
reference = template.format(DSML_TOOL_START)
@@ -1353,7 +1358,8 @@ class TestMalformedDsmlNoise:
@pytest.mark.parametrize("chunk_size", [1, 3, None], ids=lambda c: f"chunk={c}")
def test_delegating_parser_corrupted_wrapper(self, wrapper, chunk_size):
"""The serving-layer shape: reasoning and tool adapters on separate
engines, with the corrupted opener arriving as plain text."""
engines, with the corrupted opener arriving as plain text.
"""
tokens = _dsv4_tokens(
reasoning="Checking the weather.",
tool_name="get_weather",
@@ -174,7 +174,8 @@ _TOOL_CALL_SAMPLES = [
)
def test_delegating_parse_tool_choice_none(parser_cls, parser_name, sample):
"""Non-streaming parse() with tool_choice='none' via DelegatingParser
must not leak special tokens into content."""
must not leak special tokens into content.
"""
tokenizer = make_mock_tokenizer(sample)
validated_tools = (
_TOOLS_VALIDATOR.validate_python(sample.tools) if sample.tools else None
+38 -19
View File
@@ -275,7 +275,8 @@ class TestStreaming:
def test_unmatched_close_brace_does_not_poison_depth(self):
"""A stray } in malformed JSON must not kill streaming for
all subsequent content."""
all subsequent content.
"""
engine = StreamingParserEngine(_hermes_config(), tokenizer=None)
engine.feed("<tool_call>", [])
@@ -291,7 +292,8 @@ class TestStreaming:
def test_json_args_no_premature_close_brace(self):
"""Closing braces of the top-level JSON shouldn't be streamed
until confirmed by the end tag."""
until confirmed by the end tag.
"""
engine = StreamingParserEngine(_hermes_config(), tokenizer=None)
engine.feed("<tool_call>", [])
@@ -346,7 +348,8 @@ class TestLexerBufferFlush:
def test_buffered_prefix_emitted_in_current_state(self):
"""Text buffered by the lexer (e.g. '<') must be emitted as
REASONING_CHUNK before THINK_END transitions to CONTENT."""
REASONING_CHUNK before THINK_END transitions to CONTENT.
"""
engine = StreamingParserEngine(_think_config(), _make_think_tokenizer())
events = engine.feed("<think>", [_START_ID])
@@ -384,11 +387,13 @@ class TestLexerBufferFlush:
class TestTokenIdFiltering:
"""When token IDs are available, lex-matched terminals that also
have token_id_terminal entries should be demoted to content."""
have token_id_terminal entries should be demoted to content.
"""
def test_lex_matched_terminal_demoted_after_token_ids_seen(self):
"""After receiving token IDs, text that matches a token-ID
terminal should be treated as content, not trigger a transition."""
terminal should be treated as content, not trigger a transition.
"""
engine = StreamingParserEngine(_hermes_config(), _make_hermes_tokenizer())
# First feed with a non-special token ID to set _ever_had_token_ids
@@ -409,7 +414,8 @@ class TestTokenIdFiltering:
def test_scanner_matched_terminal_bypasses_filter(self):
"""PreLexedTerminals from the scanner bypass the filter and
still trigger state transitions."""
still trigger state transitions.
"""
engine = StreamingParserEngine(_hermes_config(), _make_hermes_tokenizer())
events = engine.feed("<tool_call>", [_TOOL_START_ID])
@@ -422,7 +428,8 @@ class TestTokenIdFiltering:
def test_no_filtering_without_token_ids(self):
"""When no token IDs are ever provided (non-streaming),
text matching still triggers transitions."""
text matching still triggers transitions.
"""
engine = StreamingParserEngine(_hermes_config(), _make_hermes_tokenizer())
events = engine.feed('<tool_call>{"name": "f"}</tool_call>', [])
@@ -434,7 +441,8 @@ class TestTokenIdFiltering:
def test_mixed_text_then_real_tool_call(self):
"""Text mentioning tool syntax followed by a real special-token
tool call."""
tool call.
"""
engine = StreamingParserEngine(_hermes_config(), _make_hermes_tokenizer())
events1 = engine.feed("Mention <tool_call> in text. ", [1, 2, 3, 4])
@@ -494,7 +502,8 @@ class TestReasoningTokenCounts:
def _func_prefix_config() -> ParserEngineConfig:
"""Config mixing token-ID terminals (TOOL_START/END) with
text-only terminals (FUNC_PREFIX) and fallback transitions."""
text-only terminals (FUNC_PREFIX) and fallback transitions.
"""
return ParserEngineConfig(
name="func_prefix_test",
terminals={
@@ -563,11 +572,13 @@ def _make_func_prefix_tokenizer():
class TestTextOnlyFallbackFiltering:
"""When token IDs are available, transitions marked
skip_in_token_id_mode should be skipped."""
skip_in_token_id_mode should be skipped.
"""
def test_func_prefix_in_prose_demoted_in_strict_mode(self):
"""<function=get_time> in prose should NOT trigger a tool call
when strict mode is active."""
when strict mode is active.
"""
engine = StreamingParserEngine(
_func_prefix_config(), _make_func_prefix_tokenizer()
)
@@ -584,7 +595,8 @@ class TestTextOnlyFallbackFiltering:
def test_normal_flow_after_tool_start_still_works(self):
"""TOOL_START (special token) -> FUNC_PREFIX (text) should
still parse a tool call normally in strict mode."""
still parse a tool call normally in strict mode.
"""
engine = StreamingParserEngine(
_func_prefix_config(), _make_func_prefix_tokenizer()
)
@@ -604,7 +616,8 @@ class TestTextOnlyFallbackFiltering:
def test_fallback_fires_without_token_ids(self):
"""When no token IDs are provided, fallback transitions should
still fire normally."""
still fire normally.
"""
engine = StreamingParserEngine(
_func_prefix_config(), _make_func_prefix_tokenizer()
)
@@ -618,7 +631,8 @@ class TestTextOnlyFallbackFiltering:
def test_tool_between_fallback_blocked_in_strict_mode(self):
"""The (TOOL_BETWEEN, FUNC_PREFIX) fallback should also be
blocked in strict mode."""
blocked in strict mode.
"""
engine = StreamingParserEngine(
_func_prefix_config(), _make_func_prefix_tokenizer()
)
@@ -648,7 +662,8 @@ class TestNoUnusedTokenizerAttr:
class TestArgsResetOnReentry:
"""When leaving TOOL_ARGS and later re-entering (e.g. two tool
calls), the entering-TOOL_ARGS block resets args tracking. The
redundant reset on exit was removed."""
redundant reset on exit was removed.
"""
@staticmethod
def _multi_tool_config() -> ParserEngineConfig:
@@ -723,7 +738,8 @@ class TestArgsResetOnReentry:
class TestToolPreambleFinish:
"""finish() in TOOL_PREAMBLE state emits TOOL_CALL_END when a tool
call was started (tool_index >= 0), but not when tool_index is -1."""
call was started (tool_index >= 0), but not when tool_index is -1.
"""
@staticmethod
def _preamble_with_tool_call_start_config() -> ParserEngineConfig:
@@ -782,7 +798,8 @@ class TestToolPreambleFinish:
class TestRegexTerminalInfraRemoved:
"""TerminalDef.priority, LexerShape.regex_terminals, and the regex
matching loop were removed."""
matching loop were removed.
"""
def test_terminal_def_no_priority(self):
import regex as re
@@ -807,7 +824,8 @@ class TestRegexTerminalInfraRemoved:
class TestMultiCharTerminalInArgs:
"""Regression: multi-char terminals falling through in TOOL_ARGS
must be fed char-by-char via _feed_args_text, not _feed_args_char."""
must be fed char-by-char via _feed_args_text, not _feed_args_char.
"""
@staticmethod
def _newline_config() -> ParserEngineConfig:
@@ -899,7 +917,8 @@ class TestSkipToolParsing:
def _message_header_config(reasoning_end: bool) -> ParserEngineConfig:
"""MESSAGE_HEADER-based grammar whose tool opener optionally carries
REASONING_END, mirroring the two shapes a transition table can take."""
REASONING_END, mirroring the two shapes a transition table can take.
"""
return ParserEngineConfig(
name=f"message_header_test_{reasoning_end}",
initial_state=ParserState.MESSAGE_HEADER,
@@ -1097,7 +1097,8 @@ class TestNonStreamingReasoningPlusToolCalls:
def test_both_extractions_independent(self, parser, request_obj):
"""Calling extract_reasoning then extract_tool_calls on the same
parser instance should both work (each resets the engine)."""
parser instance should both work (each resets the engine).
"""
model_output = FULL_MODEL_OUTPUT
reasoning, _ = parser.extract_reasoning(model_output, request_obj)
@@ -1111,7 +1112,8 @@ class TestNonStreamingReasoningPlusToolCalls:
class TestAdapterExtractReasoning:
"""The reasoning adapter's extract_reasoning uses skip_tool_parsing
so tool call text is preserved as content for the tool adapter."""
so tool call text is preserved as content for the tool adapter.
"""
@pytest.fixture
def adapter(self, mock_tokenizer):
@@ -1152,7 +1154,8 @@ class TestAdapterExtractReasoning:
class TestGemma4SchemaAwareTypeCoercion:
"""Verify that streaming and non-streaming produce identical
type-fixed arguments when tool schemas declare string parameters
but the model outputs bare numbers/booleans."""
but the model outputs bare numbers/booleans.
"""
@pytest.fixture
def tools(self):
@@ -1182,7 +1185,8 @@ class TestGemma4SchemaAwareTypeCoercion:
def test_streaming_string_param_not_coerced(self, parser_with_tools, mock_request):
"""A numeric value for a string-typed param must remain a string
in the streamed output, matching the non-streaming result."""
in the streamed output, matching the non-streaming result.
"""
chunks = [
"<|tool_call>",
"call:update_record{",
@@ -1213,7 +1217,8 @@ class TestGemma4SchemaAwareTypeCoercion:
def test_streaming_matches_non_streaming(self, parser_with_tools, mock_request):
"""Concatenated streaming deltas must produce the same arguments
as non-streaming extraction."""
as non-streaming extraction.
"""
text = "<|tool_call>call:update_record{zipcode:12345}<tool_call|>"
non_streaming = parser_with_tools.extract_tool_calls(text, mock_request)
@@ -1234,7 +1239,8 @@ class TestGemma4SchemaAwareTypeCoercion:
class TestGemma4SchemaCoercionBoolNumberNull:
"""Verify that _fix_arg_types coerces string values to non-string
schema types for the Gemma4 parser."""
schema types for the Gemma4 parser.
"""
@pytest.fixture
def tools(self):
@@ -1295,7 +1301,8 @@ class TestGemma4SchemaCoercionBoolNumberNull:
def test_streaming_type_stability(self, parser_with_tools, mock_request):
"""Values streamed incrementally must not cause prefix
incompatibility when types are coerced."""
incompatibility when types are coerced.
"""
text = (
"<|tool_call>call:configure{"
"enabled:true,"
@@ -1395,7 +1402,8 @@ BARE_THOUGHT_SEQUENCE.append((TOOL_CALL_END_ID, "<tool_call|>"))
class TestBareThoughtWithoutChannelOpener:
"""When the model omits <|channel> and starts with bare ``thought``,
the parser should auto-inject the channel opener so reasoning is
captured correctly."""
captured correctly.
"""
@pytest.fixture
def bare_thought_tokenizer(self):
@@ -1466,7 +1474,8 @@ class TestBareThoughtWithoutChannelOpener:
def test_bare_thought_token_at_end_of_stream(self, request_obj):
"""When the stream ends with just "thought" (no \\n), the parser
should treat it as the thought prefix token, not real reasoning."""
should treat it as the thought prefix token, not real reasoning.
"""
seq: list[tuple[int, str]] = [
(CHANNEL_START_ID, "<|channel>"),
(3000, "thought"),
+4 -2
View File
@@ -61,7 +61,8 @@ class TestIsReasoningEnd:
def test_previous_turn_tool_call_ignored(self, parser):
"""A finished tool call from an earlier turn says nothing about the
turn currently being generated."""
turn currently being generated.
"""
prompt = [THINK_S, TEXT, TOOL_S, TEXT, TOOL_E, OBSERVATION, TEXT, ASSISTANT]
assert not parser.is_reasoning_end(prompt)
@@ -88,7 +89,8 @@ class TestExtractContentIds:
def test_falls_back_to_tool_call(self, parser):
"""Without ``</think>``, content starts at the opener itself so the
tool parser still receives a well-formed call."""
tool parser still receives a well-formed call.
"""
ids = [THINK_S, TEXT, TOOL_S, 30, TOOL_E]
assert parser.extract_content_ids(ids) == [TOOL_S, 30, TOOL_E]
+20 -10
View File
@@ -70,7 +70,8 @@ _MARKERS = sorted(_TML_VOCAB, key=len, reverse=True)
def _tokenize(text: str) -> list[tuple[int, str]]:
"""Tokenize like the real stream: markers are atomic special tokens,
plain text becomes one token per character (matching the mock
tokenizer's ``chr``-based decode)."""
tokenizer's ``chr``-based decode).
"""
tokens: list[tuple[int, str]] = []
i = 0
while i < len(text):
@@ -87,7 +88,8 @@ def _tokenize(text: str) -> list[tuple[int, str]]:
def _stream(parser, request, text: str, chunk_size: int):
"""Stream production-shaped deltas: ``chunk_size`` tokens per delta,
with delta_token_ids covering every token (specials and text)."""
with delta_token_ids covering every token (specials and text).
"""
tokens = _tokenize(text)
results = []
previous_text = ""
@@ -119,7 +121,8 @@ def _stream(parser, request, text: str, chunk_size: int):
def _stream_text_only(parser, request, text: str, chunk_size: int):
"""Stream text-only deltas (no token ids), chunked at arbitrary
character boundaries — exercises the text-lexing fallback path,
including markers split across chunks."""
including markers split across chunks.
"""
results = []
previous_text = ""
for start in range(0, len(text), chunk_size):
@@ -170,7 +173,8 @@ def _stream_delegating(parser, request, text, chunk_size, prompt_token_ids):
"""Stream ``text`` through ``DelegatingParser.parse_delta``, ``chunk_size``
tokens per delta; return ``(content, reasoning, ordered tool names,
ordered tool arguments)``. Arguments arrive in fragments, so they are
concatenated per tool index."""
concatenated per tool index.
"""
tokens = _tokenize(text)
content, reasoning = "", ""
tools: dict[int, str] = {}
@@ -522,7 +526,8 @@ class TestToolCallFiltering:
"""Inkling equivalents of the generic tool-call-filtering replay tests
(Inkling is excluded from those in test_replay.py: its structural
role/kind tokens and shared block-end token don't fit the generic
reasoning/tool split model)."""
reasoning/tool split model).
"""
def test_skip_tool_parsing_round_trip(self, mock_tokenizer, mock_request):
# First pass (reasoning adapter, skip_tool_parsing): reasoning is
@@ -820,7 +825,8 @@ class TestDelegatingTwoPass:
):
"""The optional function name between ``<|message_model|>`` and the
content-kind marker is metadata: the buffered header must be
discarded on the way out, not flushed into content."""
discarded on the way out, not flushed into content.
"""
tools = [_function_tool()]
mock_request.tools = tools
content, _, names, args = _stream_delegating(
@@ -837,7 +843,8 @@ class TestDelegatingTwoPass:
def test_content_state_tool_start_streaming(self, mock_tokenizer, mock_request):
"""Same opener reached from CONTENT rather than MESSAGE_HEADER: a
text block closed with no ``<|message_model|>`` before the tool
block. Preceding text must reach content exactly once, unmarked."""
block. Preceding text must reach content exactly once, unmarked.
"""
tools = [_function_tool()]
mock_request.tools = tools
content, _, names, args = _stream_delegating(
@@ -859,7 +866,8 @@ def test_content_tool_start_emits_reasoning_end_in_reasoning_pass():
boundary (#49876), but the header-flush path
(MESSAGE_HEADER --END_MESSAGE--> CONTENT) reaches CONTENT without
one, so a tool block opening from there relies on this transition
alone to hand off to the tool pass."""
alone to hand off to the tool pass.
"""
engine = StreamingParserEngine(inkling_config(), tokenizer=None)
engine.skip_tool_parsing = True
engine.reset(initial_state=ParserState.CONTENT)
@@ -876,7 +884,8 @@ def test_skip_reasoning_parsing_inert_for_shared_markers():
markers. Inkling has none — ``<|end_message|>`` is labelled THINK_END
yet also closes text, header, and tool blocks — so the flag must be
inert: a tool block still closes through its transition and following
text returns to CONTENT instead of leaking into the argument stream."""
text returns to CONTENT instead of leaking into the argument stream.
"""
engine = StreamingParserEngine(inkling_config(), tokenizer=None)
engine.skip_reasoning_parsing = True
engine.reset(initial_state=ParserState.CONTENT)
@@ -896,7 +905,8 @@ class TestToolParserWithoutReasoningParser:
labels ``<|end_message|>`` as THINK_END; a bypass keyed on the label
would neutralize the closer of every block kind, leaking markers into
content and leaving tool calls unterminated. Structure must keep
parsing exactly as with the reasoning parser attached."""
parsing exactly as with the reasoning parser attached.
"""
GEN_PROMPT = [_TML_VOCAB[MSG_MODEL]]
+12 -6
View File
@@ -56,7 +56,8 @@ def parser():
class TestNemotronSwap:
def test_enable_thinking_false_swaps(self, parser):
"""When enable_thinking=False, model output without think tags
should have reasoning swapped to content."""
should have reasoning swapped to content.
"""
text = "The answer is 42."
request = _make_request(enable_thinking=False)
reasoning, content = parser.extract_reasoning(text, request)
@@ -73,7 +74,8 @@ class TestNemotronSwap:
def test_no_swap_when_content_exists(self, parser):
"""With enable_thinking=False but real </think> giving content,
no swap occurs."""
no swap occurs.
"""
text = "Some reasoning.</think>Actual content here."
request = _make_request(enable_thinking=False)
reasoning, content = parser.extract_reasoning(text, request)
@@ -180,11 +182,13 @@ class TestStreaming:
class TestParseDeltaTokenIdFiltering:
"""parse_delta must not trigger tool call parsing when <tool_call>
appears as regular text rather than as a special token ID."""
appears as regular text rather than as a special token ID.
"""
def test_tool_call_text_in_reasoning_is_not_parsed(self, parser):
"""Literal <tool_call> in model reasoning should be content,
not a tool call."""
not a tool call.
"""
request = _make_request()
text = (
@@ -210,7 +214,8 @@ class TestParseDeltaTokenIdFiltering:
def test_special_token_id_still_triggers_tool_call(self, parser):
"""When the scanner matches a special token ID, the tool call
must still be parsed correctly."""
must still be parsed correctly.
"""
request = _make_request()
parser.parse_delta(
@@ -250,7 +255,8 @@ class TestParseDeltaTokenIdFiltering:
def test_text_discussion_then_real_tool_call(self, parser):
"""Model discusses tool syntax in reasoning, then makes a real
tool call via special tokens."""
tool call via special tokens.
"""
request = _make_request()
r1 = parser.parse_delta(
+46 -23
View File
@@ -323,7 +323,8 @@ class TestEventsToDelta:
def test_multiple_arg_chunks_same_batch_coalesced(self):
"""Multiple events for the same tool in one batch must produce
at most one DeltaToolCall per index."""
at most one DeltaToolCall per index.
"""
engine = _make_engine()
events = [
SemanticEvent(EventType.TOOL_CALL_START, tool_index=0),
@@ -551,7 +552,8 @@ class TestPostToolContentDeferral:
"""Regression: content after TOOL_CALL_END in the same batch must not
produce a mixed DeltaMessage(content=..., tool_calls=...) — that causes
split_delta to reorder content before tool_calls, breaking the Responses
API state machine."""
API state machine.
"""
def test_text_after_tool_end_deferred(self):
engine = _make_engine()
@@ -605,7 +607,8 @@ class TestPostToolContentDeferral:
"""Deferred content from batch N must not mix with arg-continuation
tool events in batch N+1 — that creates a DeltaMessage with both
content and nameless tool_calls, which crashes the Responses API
state machine (name=None → Pydantic ValidationError)."""
state machine (name=None → Pydantic ValidationError).
"""
engine = _make_engine()
engine._content_has_nonws = True
@@ -834,7 +837,8 @@ class TestBuildExtractedResult:
class TestEngineBasedPath:
"""Tests for the _engine_based accumulation behavior in
DelegatingParser.parse_delta."""
DelegatingParser.parse_delta.
"""
def test_engine_based_true_when_both_parsers_engine(self):
r = SimpleNamespace(engine_based_streaming=True)
@@ -880,7 +884,8 @@ class TestEngineBasedPath:
class TestParseTokenIdPassthrough:
"""parse() must forward model_output_token_ids to _single_pass_parse
so that token-ID-based strict terminal matching is active."""
so that token-ID-based strict terminal matching is active.
"""
def test_literal_tool_tag_in_content_preserved_with_token_ids(self, mock_request):
engine = _make_engine(_hermes_config())
@@ -1104,7 +1109,8 @@ class TestAdapterFinishOnStreamEnd:
def test_lexer_buffer_flushed_on_finished(self):
"""Text buffered as a potential terminal prefix must be emitted
as content when the stream ends."""
as content when the stream ends.
"""
tokenizer = make_mock_tokenizer(_VOCAB)
parser = _CombinedDelegating(tokenizer)
request = _make_delegating_request()
@@ -1122,7 +1128,8 @@ class TestAdapterFinishOnStreamEnd:
def test_args_buffer_flushed_on_finished(self):
"""Pending arg buffer text must be emitted when stream ends
mid-tool-call (closing brace held back in buffer)."""
mid-tool-call (closing brace held back in buffer).
"""
tokenizer = make_mock_tokenizer(_VOCAB)
parser = _CombinedDelegating(tokenizer)
request = _make_delegating_request()
@@ -1358,7 +1365,8 @@ class TestSkipToolSpanForwarding:
class TestToolAdapterForwardsKwargs:
"""ParserEngineToolAdapter.__init__ must forward **kwargs to the
parser engine class so chat_template_kwargs reach model parsers."""
parser engine class so chat_template_kwargs reach model parsers.
"""
@pytest.mark.parametrize(
"enable_thinking,expected_state",
@@ -1388,7 +1396,8 @@ class TestToolAdapterForwardsKwargs:
class TestExtractContentIdsNoEmptyReturn:
"""extract_content_ids must return input_ids (not []) when there is
no THINK_END token ID and _reasoning_ended is True."""
no THINK_END token ID and _reasoning_ended is True.
"""
_NO_THINK_CONFIG = ParserEngineConfig(name="no_think_end", token_id_terminals={})
@@ -1786,7 +1795,8 @@ class TestDropSpecialTokens:
def test_drops_special_token_by_id_from_content(self):
"""A special token (not a configured terminal) is dropped when
it arrives as its actual token ID."""
it arrives as its actual token ID.
"""
config = ParserEngineConfig(
name="drop_content_test",
terminals={},
@@ -1821,7 +1831,8 @@ class TestDropSpecialTokens:
def test_drops_via_text_fallback_when_no_token_ids(self):
"""When no token IDs are provided, text-based lexer catches
drop tokens as a fallback."""
drop tokens as a fallback.
"""
engine = _make_engine(
vocab=_DROP_VOCAB,
special_tokens=list(_DROP_VOCAB.keys()),
@@ -1834,7 +1845,8 @@ class TestDropSpecialTokens:
def test_regular_tokens_spelling_special_survive(self):
"""Regular tokens that spell out a drop-token string survive
when token IDs prove they are not the special token."""
when token IDs prove they are not the special token.
"""
vocab = {**_DROP_VOCAB, "h": 72, "<": 73, "bos": 74, ">": 75, "w": 76}
engine = _make_engine(
vocab=vocab,
@@ -1850,7 +1862,8 @@ class TestDropSpecialTokens:
def test_configured_terminal_not_treated_as_drop(self):
"""Tokens already in config.terminals (like <think>) are handled
by the state machine, not the drop mechanism."""
by the state machine, not the drop mechanism.
"""
engine = _make_engine(
vocab=_DROP_VOCAB,
special_tokens=list(_DROP_VOCAB.keys()),
@@ -1905,7 +1918,8 @@ class TestDropSpecialTokens:
def test_drops_applied_with_skip_tool_parsing(self):
"""Drop tokens are always dropped, even with skip_tool_parsing.
DROP_TERMINALs have no transitions by construction, so no parser
pass can use them."""
pass can use them.
"""
for initial_state in (ParserState.REASONING, ParserState.CONTENT):
engine = _make_engine(
vocab=_DROP_VOCAB,
@@ -1921,7 +1935,8 @@ class TestDropSpecialTokens:
def test_transitions_unaffected_by_drop_in_reasoning_with_skip_tool_parsing(self):
"""With skip_tool_parsing in REASONING state, drop tokens are
removed but configured terminals still fire their transitions."""
removed but configured terminals still fire their transitions.
"""
engine = _make_engine(
vocab=_DROP_VOCAB,
special_tokens=list(_DROP_VOCAB.keys()),
@@ -1957,7 +1972,8 @@ class TestDropSpecialTokens:
def test_mixed_configured_and_drop_terminals(self):
"""Configured terminals trigger transitions while drop terminals
are silently removed in the same stream."""
are silently removed in the same stream.
"""
engine = _make_engine(
vocab=_DROP_VOCAB,
special_tokens=list(_DROP_VOCAB.keys()),
@@ -1981,7 +1997,8 @@ class TestTruncatedToolOpenerStreamParity:
"""Regression tests for #47137: when generation terminates inside a
``<tool_call>`` opener that has not been promoted to a tool call
(via ``max_tokens`` or a ``stop`` string), the non-streaming path
must drop the incomplete markup, matching the streaming path."""
must drop the incomplete markup, matching the streaming path.
"""
_QWEN3_VOCAB = {
"<tool_call>": 100,
@@ -2027,7 +2044,8 @@ class TestTruncatedToolOpenerStreamParity:
)
def test_truncated_opener_dropped_in_both_paths(self, mock_request, chunks):
"""A cutoff inside the opener yields no content and no tool calls
in both the non-streaming and streaming paths."""
in both the non-streaming and streaming paths.
"""
text = "".join(chunks)
_, content, tool_calls = self._make_parser().parse(
@@ -2040,7 +2058,8 @@ class TestTruncatedToolOpenerStreamParity:
def test_content_before_truncated_opener_preserved(self, mock_request):
"""Only the incomplete markup is dropped; content generated before
the opener is returned identically by both paths."""
the opener is returned identically by both paths.
"""
chunks = ["Checking the weather. ", "<tool_call>", "\n", "<function"]
text = "".join(chunks)
@@ -2054,7 +2073,8 @@ class TestTruncatedToolOpenerStreamParity:
def test_content_around_unpromoted_tool_block_stays_ordered(self, mock_request):
"""Text surrounding a complete-but-unparsable tool block keeps its
original order in the non-streaming path, matching streaming."""
original order in the non-streaming path, matching streaming.
"""
chunks = ["A ", "<tool_call>", "garbage", "</tool_call>", " B"]
text = "".join(chunks)
@@ -2068,7 +2088,8 @@ class TestTruncatedToolOpenerStreamParity:
def test_complete_tool_call_still_promoted(self, mock_request):
"""Sanity check: a complete tool call still parses in the
non-streaming path after the truncation fix."""
non-streaming path after the truncation fix.
"""
text = (
"<tool_call>\n"
"<function=get_weather>\n"
@@ -2088,7 +2109,8 @@ class TestThinkMarkupWithoutReasoningParser:
"""With no reasoning parser configured, reasoning markup is plain
content: engine-backed tool parsers must return it verbatim in both
the non-streaming and streaming paths, not consume the markers or
reclassify the block as reasoning and drop it."""
reclassify the block as reasoning and drop it.
"""
_VOCAB = {
"<think>": 90,
@@ -2154,7 +2176,8 @@ class TestThinkMarkupWithoutReasoningParser:
def test_think_block_preserved_alongside_tool_call(self, mock_request):
"""The tools-called branch also returns the tool parser's content;
a think block preceding a promoted tool call must survive in it."""
a think block preceding a promoted tool call must survive in it.
"""
think = "<think>plan</think>"
text = think + (
"<tool_call>\n"
+10 -5
View File
@@ -318,7 +318,8 @@ class TestStreaming:
def test_streaming_args_arrive_incrementally(self, parser, mock_request):
"""Arguments must stream as intermediate deltas, not batch at
tool-end."""
tool-end.
"""
chunks = [
"<tool_call>\n",
"<function=get_weather>\n",
@@ -523,7 +524,8 @@ class TestStreaming:
def test_streaming_split_tool_call_tag(self, parser, mock_request):
"""<tool_call> arrives as a single special token; the rest of
the content is split into fine-grained chunks."""
the content is split into fine-grained chunks.
"""
chunks = [
"<tool_call>\n",
"<function=test>\n",
@@ -700,7 +702,8 @@ class TestArgConverter:
class TestSchemaAwareTypeCoercion:
"""Verify that _fix_arg_types corrects miscoerced values using the
tool schema."""
tool schema.
"""
@pytest.fixture
def tools(self):
@@ -842,7 +845,8 @@ class TestAnyOfTypeCoercion:
def test_anyof_string_param_not_coerced(self, parser_with_anyof, mock_request):
"""A param with anyOf including 'string' must not be coerced
to integer."""
to integer.
"""
text = (
"<tool_call>\n"
"<function=set_config>\n"
@@ -858,7 +862,8 @@ class TestAnyOfTypeCoercion:
class TestSchemaCoercionBoolNumberNull:
"""Verify that _fix_arg_types coerces string values to non-string
schema types using coerce_to_schema_type."""
schema types using coerce_to_schema_type.
"""
@pytest.fixture
def tools(self):
+12 -6
View File
@@ -276,7 +276,8 @@ class TestIsReasoningEndTurnBoundaries:
def test_empty_boundary_config_keeps_global_walk(self):
"""Configs without turn_boundary_tokens keep the pre-existing
behavior: any </think> in the sequence ends reasoning."""
behavior: any </think> in the sequence ends reasoning.
"""
cfg = qwen3_config(turn_boundary_tokens=frozenset())
parser = Qwen3Parser(
make_mock_tokenizer(_QWEN3_VOCAB), parser_engine_config=cfg
@@ -288,7 +289,8 @@ class TestIsReasoningEndTurnBoundaries:
def test_boundary_tokens_missing_from_vocab_keep_global_walk(self):
"""Boundary tokens absent from the vocabulary resolve to nothing,
so subclasses reusing the default on a non-ChatML vocab keep the
pre-existing behavior."""
pre-existing behavior.
"""
vocab = {
token: token_id
for token, token_id in _QWEN3_VOCAB.items()
@@ -299,7 +301,8 @@ class TestIsReasoningEndTurnBoundaries:
def test_boundary_with_thinking_disabled_is_end(self, mock_tokenizer):
"""With thinking disabled (initial state CONTENT) a walk that stops at a turn
boundary must report reasoning as ended: the model never emits a marker."""
boundary must report reasoning as ended: the model never emits a marker.
"""
parser = Qwen3Parser(
mock_tokenizer, chat_template_kwargs={"enable_thinking": False}
)
@@ -307,7 +310,8 @@ class TestIsReasoningEndTurnBoundaries:
def test_boundary_with_thinking_enabled_not_end(self, parser):
"""With thinking disabled (initial state REASONING) a walk that stops at a turn
boundary must keep reasoning open"""
boundary must keep reasoning open
"""
assert not parser.is_reasoning_end([_IM_START_ID, _TEXT_ID])
@@ -668,7 +672,8 @@ class TestTrailingWhitespaceStripping:
class TestWhitespaceStrippingDisabled:
"""When strip_trailing_reasoning_whitespace is False,
trailing whitespace in reasoning must be preserved."""
trailing whitespace in reasoning must be preserved.
"""
@pytest.fixture
def parser_no_strip(self):
@@ -726,7 +731,8 @@ class TestThinkingDisabled:
def test_thinking_disabled_streaming_content_only(self, mock_tokenizer):
"""Plain text with thinking disabled must stream as content, not
reasoning. Before the fix, the REASONING initial state caused all
output to be emitted as reasoning chunks."""
output to be emitted as reasoning chunks.
"""
p = Qwen3Parser(
mock_tokenizer,
chat_template_kwargs={"enable_thinking": False},
+8 -4
View File
@@ -440,7 +440,8 @@ class TestToolCallFilteringReplay:
)
class TestToolCallFilteringNonStreaming:
"""Non-streaming parse() with tool_choice='none' must suppress tool
calls and not leak special tokens into content."""
calls and not leak special tokens into content.
"""
def test_parse(self, parser_cls, sample, think_end, tool_start):
tokenizer = make_mock_tokenizer(sample)
@@ -483,7 +484,8 @@ _WS_TOOL_SAMPLES = [(t[0], t[1]) for t in _TOOL_CALL_SAMPLES if "whitespace" in
)
class TestToolChoiceNoneStreamingParity:
"""Streaming and non-streaming must return the same content
when tool_choice='none' suppresses tool calls."""
when tool_choice='none' suppresses tool calls.
"""
def test_content_matches(self, parser_cls, sample):
tokenizer = make_mock_tokenizer(sample)
@@ -541,7 +543,8 @@ def _inject_drop_tokens(sample):
class TestDropTokenReplay:
"""Verify unconfigured special tokens are silently dropped across
all parsers and chunk sizes."""
all parsers and chunk sizes.
"""
@pytest.mark.parametrize(
"parser_info",
@@ -598,7 +601,8 @@ class TestDropTokenNonStreaming:
class TestAdapterReferences:
"""Verify make_adapters sets reasoning/tool parser class refs on parser engine
parser classes so the serving layer finds them and calls adjust_request."""
parser classes so the serving layer finds them and calls adjust_request.
"""
@pytest.mark.parametrize(
"parser_name",
+4 -2
View File
@@ -129,7 +129,8 @@ def test_single_tool_call(tool_parser, mock_request):
def test_malformed_function_end_does_not_drop_siblings(tool_parser, mock_request):
"""Regression for #46314: a malformed ``</function>`` with no closing ``>``
on the header must not discard the other, well-formed calls."""
on the header must not discard the other, well-formed calls.
"""
text = (
f"{TOOL_CALL_START}\n<function=broken</function>\n{TOOL_CALL_END}"
f"{TOOL_CALL_START}\n<function=get_weather>\n"
@@ -171,7 +172,8 @@ def test_reasoning_then_tool_call(parser):
def test_streaming_think_end_and_tool_call_same_delta(parser):
"""``</seed:think>`` and ``<seed:tool_call>`` arriving in one delta must
not leak the terminal tokens into the reasoning text."""
not leak the terminal tokens into the reasoning text.
"""
reasoning, content = simulate_reasoning_streaming(
parser,
[
@@ -146,7 +146,8 @@ class TestUfffdReasoningTransition:
def test_byte_fallback_tokenizer_produces_ufffd(self):
"""Validate the fixture: decode() returns U+FFFD for isolated
byte-fallback token IDs, proving the old code path would leak."""
byte-fallback token IDs, proving the old code path would leak.
"""
vocab = dict(_MODEL_CONFIGS[0].values[0])
tokenizer = ByteFallbackMockTokenizer(vocab, _SHARED_TOKENS, _SHARED_UFFFD_IDS)
assert tokenizer.decode([200]) == "�"
+2 -1
View File
@@ -992,7 +992,8 @@ def _inkling_block(
) -> None:
"""Append one Inkling content block; blocks after the first start with
the ``<|message_model|>`` role token (the first block continues the
generation prompt directly)."""
generation prompt directly).
"""
if segs:
segs.append(("<|message_model|>", True))
segs.append((kind_token, True))
+11 -7
View File
@@ -492,7 +492,8 @@ def test_extract_tool_calls_pre_v11_regex_fallback(
):
"""The regex fallback path finds valid JSON via regex when the primary
raw_decode fails on leading junk. It should re-serialize arguments
and return a valid tool call."""
and return a valid tool call.
"""
model_output = (
'[TOOL_CALLS] junk [{"name": "add", "arguments":{"a": 1, "b": 2}}] trail'
)
@@ -2003,7 +2004,8 @@ def test_streaming_pre_v11_parallel_calls_batched_deltas(
mistral_pre_v11_tool_parser, mistral_pre_v11_tokenizer, chunk_size, driver
):
"""A batched delta spanning the boundary between two parallel calls must
keep them on distinct indices (the bug collapsed both onto index 0)."""
keep them on distinct indices (the bug collapsed both onto index 0).
"""
model_output = (
'[TOOL_CALLS] [{"name": "add", "arguments": {"a": 3.5, "b": 4}}, '
'{"name": "get_current_weather", "arguments": '
@@ -2045,7 +2047,6 @@ def test_content_tool_calls_transition_emits_reasoning_end(
reasoning_encoding,
):
"""(CONTENT, TOOL_CALLS) must emit REASONING_END when reasoning is enabled."""
cfg = mistral_config(reasoning_encoding=reasoning_encoding)
engine = StreamingParserEngine(
config=cfg, tokenizer=mistral_tokenizer, vocab=mistral_tokenizer.get_vocab()
@@ -2122,7 +2123,8 @@ def test_reasoning_active_no_think_block_no_leak(
chunk_size,
):
"""Tool call without a preceding think block must not leak as content
when the reasoning parser is active."""
when the reasoning parser is active.
"""
accumulated_content = ""
function_names: list[str] = []
function_args_strs: list[str] = []
@@ -2219,11 +2221,12 @@ def test_adjust_request_pre_v11_required_clears_response_format(
mistral_pre_v11_tool_parser: MistralToolParser,
response_format: dict,
) -> None:
"""required + response_format must inject the tool schema and clear
"""Required + response_format must inject the tool schema and clear
response_format so the tool schema is the sole structured-output
constraint, matching the base ToolParser (otherwise the request either
hits the "multiple constraints" engine error or, with no constraint,
rambles to finish_reason='length')."""
rambles to finish_reason='length').
"""
request = _make_request(tool_choice="required", response_format=response_format)
result = mistral_pre_v11_tool_parser.adjust_request(request)
@@ -2240,7 +2243,8 @@ def test_adjust_request_non_mistral_tokenizer_required_injects_schema(
) -> None:
"""The guided-schema injection must also fire for non-Mistral (e.g. HF-mode)
tokenizers driving the Mistral tool parser, mirroring the base ToolParser so
required + response_format does not fall back to an unconstrained ramble."""
required + response_format does not fall back to an unconstrained ramble.
"""
request = _make_request(
tool_choice="required", response_format={"type": "json_object"}
)
+24 -12
View File
@@ -229,7 +229,8 @@ def _boundary_chunks(tokenizer, parser, end_token_id=None):
def test_parse_delta_reasoning_not_dropped_on_boundary(tokenizer, request_obj):
"""Regression: reasoning must not be lost when a multi-token delta
spans the reasoning/tool-call boundary."""
spans the reasoning/tool-call boundary.
"""
parser = make_parser(tokenizer, reasoning=True, tool=True)
chunks = _boundary_chunks(tokenizer, parser)
results = stream_chunks(parser, tokenizer, chunks, request_obj)
@@ -247,7 +248,8 @@ def test_parse_delta_reasoning_not_dropped_on_boundary(tokenizer, request_obj):
def test_parse_delta_reasoning_boundary_no_tool_parser(tokenizer, request_obj):
"""When no tool parser is active, boundary-spanning chunks must still
preserve reasoning and pass post-</think> text as content."""
preserve reasoning and pass post-</think> text as content.
"""
parser = make_parser(tokenizer, reasoning=True, tool=False)
chunks = _boundary_chunks(tokenizer, parser)
results = stream_chunks(parser, tokenizer, chunks, request_obj)
@@ -261,7 +263,8 @@ def test_parse_delta_reasoning_boundary_no_tool_parser(tokenizer, request_obj):
def test_parse_delta_reasoning_only_no_think_leak(tokenizer, request_obj):
"""Regression: </think> must not leak into content when streaming
token-by-token with reasoning=True, tool=False."""
token-by-token with reasoning=True, tool=False.
"""
parser = make_parser(tokenizer, reasoning=True, tool=False)
results = stream_text(
parser, tokenizer, MODEL_OUTPUT, request_obj, prompt_token_ids=[]
@@ -303,7 +306,8 @@ def test_parse_delta_reasoning_only_thinking_disabled(tokenizer, request_obj):
def test_parse_delta_finished_no_flush_without_tool_call_delta(tokenizer, request_obj):
"""When finished=True but the final parse_delta produces no
tool-call delta, unstreamed args are not flushed."""
tool-call delta, unstreamed args are not flushed.
"""
parser = make_parser(tokenizer, reasoning=False, tool=True)
results = stream_text(
@@ -326,7 +330,8 @@ def test_parse_delta_finished_no_flush_without_tool_call_delta(tokenizer, reques
def test_parse_delta_finished_no_extra_args_when_fully_streamed(tokenizer, request_obj):
"""When all args have been streamed, finished=True must not
produce extra or duplicate arguments."""
produce extra or duplicate arguments.
"""
parser = make_parser(tokenizer, reasoning=False, tool=True)
results = stream_text(
parser, tokenizer, MODEL_OUTPUT, request_obj, prompt_token_ids=[]
@@ -346,7 +351,8 @@ def test_parse_delta_finished_no_extra_args_when_fully_streamed(tokenizer, reque
def test_parse_delta_finished_appends_remaining_args(tokenizer, request_obj):
"""When finished=True and the tool parser has unstreamed args,
parse_delta appends the remaining arguments to the tool-call delta."""
parse_delta appends the remaining arguments to the tool-call delta.
"""
parser = make_parser(tokenizer, reasoning=False, tool=True)
token_ids = tokenizer.encode(MODEL_OUTPUT, add_special_tokens=False)
@@ -493,7 +499,8 @@ def test_engine_reasoning_hermes_tool_token_by_token(tokenizer, request_obj):
"""Qwen3 engine reasoning + Hermes tool parser, token-by-token.
Sanity check that the mixed engine/non-engine configuration works
when tokens arrive one at a time (no deferred content)."""
when tokens arrive one at a time (no deferred content).
"""
parser = Qwen3ReasoningHermesToolParser(tokenizer)
assert parser._reasoning_parser.engine_based_streaming is True
@@ -520,7 +527,8 @@ def test_engine_reasoning_hermes_tool_boundary(tokenizer, request_obj):
When </think> and <tool_call> are in the same chunk with aligned
text and token IDs, the engine processes both terminals and returns
the <tool_call> text as content."""
the <tool_call> text as content.
"""
parser = Qwen3ReasoningHermesToolParser(tokenizer)
end_token_id = parser._reasoning_parser._parser_engine._reasoning_end_token_id
chunks = _boundary_chunks(tokenizer, parser, end_token_id=end_token_id)
@@ -551,7 +559,8 @@ def test_engine_reasoning_hermes_tool_text_holdback(tokenizer, request_obj):
Without the fix, finish_streaming() is never called at the
reasoning->tool transition when _engine_based is False, so the '<'
is lost and the Hermes parser sees 'tool_call>...' instead of
'<tool_call>...'."""
'<tool_call>...'.
"""
parser = Qwen3ReasoningHermesToolParser(tokenizer)
vocab = tokenizer.get_vocab()
think_end_id = vocab["</think>"]
@@ -642,7 +651,8 @@ def test_engine_reasoning_no_tool_batched_content_passthrough(tokenizer, request
batched delta carries ``</think>`` plus the following content
(as happens with stream_interval > 1). The post-``</think>`` text
must be emitted as content -- not dropped, not reclassified as
reasoning -- and the ``</think>`` marker must not leak either way."""
reasoning -- and the ``</think>`` marker must not leak either way.
"""
parser = Qwen3ReasoningNoToolParser(tokenizer)
assert parser._reasoning_parser.engine_based_streaming is True
assert parser._tool_parser is None
@@ -670,7 +680,8 @@ def _decode_stream_deltas(tokenizer, groups):
This mirrors how vLLM's detokenizer feeds ``parse_delta`` in
production: byte-level UTF-8 hold-back means a character whose bytes
span multiple tokens is only surfaced once complete (a naive
per-token ``decode`` would instead emit U+FFFD replacement chars)."""
per-token ``decode`` would instead emit U+FFFD replacement chars).
"""
from tokenizers.decoders import DecodeStream
stream = DecodeStream(skip_special_tokens=False)
@@ -702,7 +713,8 @@ def test_engine_reasoning_hermes_tool_multibyte_holdback(tokenizer, request_obj)
``DecodeStream``. Its UTF-8 hold-back yields the correct
character round-trip (a naive per-token decode would corrupt it),
verifying the boundary stays byte-safe for multi-token
characters."""
characters.
"""
parser = Qwen3ReasoningHermesToolParser(tokenizer)
vocab = tokenizer.get_vocab()
think_end_id = vocab["</think>"]