Add pydocstyle to the ruff rules (#52136)
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com> Signed-off-by: Harry Mellor <hej.mellor@gmail.com> Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5
parent
804e5377aa
commit
8c1557a79c
@@ -1,7 +1,8 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Reasoning extraction, reasoning-end gating and parser selection for the
|
||||
unified Cohere Command parser."""
|
||||
unified Cohere Command parser.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
|
||||
@@ -76,7 +76,8 @@ def token_deltas(
|
||||
) -> list[tuple[str, list[int]]]:
|
||||
"""Split ``text`` into ``(delta_text, delta_token_ids)`` steps of
|
||||
``chunk_size`` tokens, buffering incomplete multi-byte sequences (trailing
|
||||
U+FFFD) into the next step as real streaming does."""
|
||||
U+FFFD) into the next step as real streaming does.
|
||||
"""
|
||||
ids = tokenizer.encode(text)
|
||||
deltas: list[tuple[str, list[int]]] = []
|
||||
prev = ""
|
||||
@@ -100,7 +101,8 @@ def make_parser(
|
||||
model_config=None,
|
||||
) -> Parser:
|
||||
"""Resolve the unified Cohere parser through ``ParserManager`` by registry
|
||||
name (``cohere_command3`` / ``cohere_command4``)."""
|
||||
name (``cohere_command3`` / ``cohere_command4``).
|
||||
"""
|
||||
cls = ParserManager.get_parser(name, name, enable_auto_tools=True)
|
||||
assert cls is not None
|
||||
return cls(
|
||||
@@ -117,7 +119,8 @@ def drive_parser(
|
||||
deltas: list[tuple[str, list[int]]],
|
||||
) -> list[DeltaMessage]:
|
||||
"""Feed ``deltas`` through ``parse_delta`` (flushing on the last one) and
|
||||
return the non-``None`` messages in order."""
|
||||
return the non-``None`` messages in order.
|
||||
"""
|
||||
out: list[DeltaMessage] = []
|
||||
for i, (text, ids) in enumerate(deltas):
|
||||
delta = parser.parse_delta(text, ids, request, finished=i == len(deltas) - 1)
|
||||
|
||||
@@ -30,6 +30,7 @@ def make_mock_tokenizer(
|
||||
The returned mock supports get_vocab(), encode(), and decode().
|
||||
decode() maps known token IDs back to their text and falls back to
|
||||
chr(id) for ASCII IDs or ``<id>`` for others.
|
||||
|
||||
"""
|
||||
id_to_text = {v: k for k, v in vocab.items()}
|
||||
tokenizer = MagicMock()
|
||||
|
||||
@@ -183,6 +183,7 @@ def replay_streaming(
|
||||
|
||||
Returns:
|
||||
List of ``DeltaMessage`` results from each ``parse_delta()`` call.
|
||||
|
||||
"""
|
||||
if chunk_size is None:
|
||||
chunk_size = len(tokens)
|
||||
|
||||
@@ -289,7 +289,8 @@ class TestStreaming:
|
||||
|
||||
class TestMissingFunctionCallsWrapper:
|
||||
"""An invoke without the ``<|DSML|function_calls>`` wrapper is still a
|
||||
tool call (V3.2 counterpart of #48931)."""
|
||||
tool call (V3.2 counterpart of #48931).
|
||||
"""
|
||||
|
||||
def test_non_streaming(self, mock_tokenizer, mock_request):
|
||||
text = _invoke("fn", _param("k", "true", "v"))
|
||||
|
||||
@@ -334,7 +334,8 @@ class TestMissingToolCallsWrapper:
|
||||
):
|
||||
"""Text after a tool block is dropped whether or not the closing
|
||||
wrapper is present, so a missing ``</|DSML|tool_calls>`` does not
|
||||
change what the client sees."""
|
||||
change what the client sees.
|
||||
"""
|
||||
text = "pre\n" + self._ORPHAN
|
||||
if trailing_end:
|
||||
text += DSML_TOOL_END
|
||||
@@ -915,7 +916,8 @@ class TestParallelUnwrapping:
|
||||
|
||||
class TestStreamingWrapperConsistency:
|
||||
"""Streamed arg deltas must stay consistent with final extraction
|
||||
when wrapper params like 'arguments' are unwrapped."""
|
||||
when wrapper params like 'arguments' are unwrapped.
|
||||
"""
|
||||
|
||||
def test_streaming_wrapper_unwrap_consistency(self, mock_tokenizer, mock_request):
|
||||
tool = _make_tool("get_weather", {"location": {"type": "string"}})
|
||||
@@ -1196,7 +1198,8 @@ class TestDelegatingParserLargeDelta:
|
||||
)
|
||||
def test_eos_not_leaked_when_reasoning_never_ends(self, chunk_size):
|
||||
"""EOS must not leak into reasoning_content when the model never
|
||||
emits </think> (generation ends while still in REASONING state)."""
|
||||
emits </think> (generation ends while still in REASONING state).
|
||||
"""
|
||||
eos_text = "<|end▁of▁sentence|>"
|
||||
eos_id = 128801
|
||||
vocab = {
|
||||
@@ -1285,7 +1288,8 @@ class TestMalformedDsmlNoise:
|
||||
self, mock_tokenizer, mock_request, wrapper
|
||||
):
|
||||
"""Like the real wrapper, a corrupted one inside ``<think>`` closes
|
||||
the reasoning block and starts the tool call."""
|
||||
the reasoning block and starts the tool call.
|
||||
"""
|
||||
text = f"thinking{wrapper}\n{self._INNER}"
|
||||
parser = self._parser(mock_tokenizer, thinking=True)
|
||||
reasoning, content = parser.extract_reasoning(text, mock_request)
|
||||
@@ -1329,7 +1333,8 @@ class TestMalformedDsmlNoise:
|
||||
prose mention of the real wrapper: the opener starts a tool block and
|
||||
the rest of the message is dropped. That is pre-existing behavior of
|
||||
``<|DSML|tool_calls>``; the variants must not differ from it either
|
||||
way."""
|
||||
way.
|
||||
"""
|
||||
template = "The opener looks like {} and then params follow. Done."
|
||||
text = template.format(wrapper)
|
||||
reference = template.format(DSML_TOOL_START)
|
||||
@@ -1353,7 +1358,8 @@ class TestMalformedDsmlNoise:
|
||||
@pytest.mark.parametrize("chunk_size", [1, 3, None], ids=lambda c: f"chunk={c}")
|
||||
def test_delegating_parser_corrupted_wrapper(self, wrapper, chunk_size):
|
||||
"""The serving-layer shape: reasoning and tool adapters on separate
|
||||
engines, with the corrupted opener arriving as plain text."""
|
||||
engines, with the corrupted opener arriving as plain text.
|
||||
"""
|
||||
tokens = _dsv4_tokens(
|
||||
reasoning="Checking the weather.",
|
||||
tool_name="get_weather",
|
||||
|
||||
@@ -174,7 +174,8 @@ _TOOL_CALL_SAMPLES = [
|
||||
)
|
||||
def test_delegating_parse_tool_choice_none(parser_cls, parser_name, sample):
|
||||
"""Non-streaming parse() with tool_choice='none' via DelegatingParser
|
||||
must not leak special tokens into content."""
|
||||
must not leak special tokens into content.
|
||||
"""
|
||||
tokenizer = make_mock_tokenizer(sample)
|
||||
validated_tools = (
|
||||
_TOOLS_VALIDATOR.validate_python(sample.tools) if sample.tools else None
|
||||
|
||||
@@ -275,7 +275,8 @@ class TestStreaming:
|
||||
|
||||
def test_unmatched_close_brace_does_not_poison_depth(self):
|
||||
"""A stray } in malformed JSON must not kill streaming for
|
||||
all subsequent content."""
|
||||
all subsequent content.
|
||||
"""
|
||||
engine = StreamingParserEngine(_hermes_config(), tokenizer=None)
|
||||
engine.feed("<tool_call>", [])
|
||||
|
||||
@@ -291,7 +292,8 @@ class TestStreaming:
|
||||
|
||||
def test_json_args_no_premature_close_brace(self):
|
||||
"""Closing braces of the top-level JSON shouldn't be streamed
|
||||
until confirmed by the end tag."""
|
||||
until confirmed by the end tag.
|
||||
"""
|
||||
engine = StreamingParserEngine(_hermes_config(), tokenizer=None)
|
||||
|
||||
engine.feed("<tool_call>", [])
|
||||
@@ -346,7 +348,8 @@ class TestLexerBufferFlush:
|
||||
|
||||
def test_buffered_prefix_emitted_in_current_state(self):
|
||||
"""Text buffered by the lexer (e.g. '<') must be emitted as
|
||||
REASONING_CHUNK before THINK_END transitions to CONTENT."""
|
||||
REASONING_CHUNK before THINK_END transitions to CONTENT.
|
||||
"""
|
||||
engine = StreamingParserEngine(_think_config(), _make_think_tokenizer())
|
||||
|
||||
events = engine.feed("<think>", [_START_ID])
|
||||
@@ -384,11 +387,13 @@ class TestLexerBufferFlush:
|
||||
|
||||
class TestTokenIdFiltering:
|
||||
"""When token IDs are available, lex-matched terminals that also
|
||||
have token_id_terminal entries should be demoted to content."""
|
||||
have token_id_terminal entries should be demoted to content.
|
||||
"""
|
||||
|
||||
def test_lex_matched_terminal_demoted_after_token_ids_seen(self):
|
||||
"""After receiving token IDs, text that matches a token-ID
|
||||
terminal should be treated as content, not trigger a transition."""
|
||||
terminal should be treated as content, not trigger a transition.
|
||||
"""
|
||||
engine = StreamingParserEngine(_hermes_config(), _make_hermes_tokenizer())
|
||||
|
||||
# First feed with a non-special token ID to set _ever_had_token_ids
|
||||
@@ -409,7 +414,8 @@ class TestTokenIdFiltering:
|
||||
|
||||
def test_scanner_matched_terminal_bypasses_filter(self):
|
||||
"""PreLexedTerminals from the scanner bypass the filter and
|
||||
still trigger state transitions."""
|
||||
still trigger state transitions.
|
||||
"""
|
||||
engine = StreamingParserEngine(_hermes_config(), _make_hermes_tokenizer())
|
||||
|
||||
events = engine.feed("<tool_call>", [_TOOL_START_ID])
|
||||
@@ -422,7 +428,8 @@ class TestTokenIdFiltering:
|
||||
|
||||
def test_no_filtering_without_token_ids(self):
|
||||
"""When no token IDs are ever provided (non-streaming),
|
||||
text matching still triggers transitions."""
|
||||
text matching still triggers transitions.
|
||||
"""
|
||||
engine = StreamingParserEngine(_hermes_config(), _make_hermes_tokenizer())
|
||||
|
||||
events = engine.feed('<tool_call>{"name": "f"}</tool_call>', [])
|
||||
@@ -434,7 +441,8 @@ class TestTokenIdFiltering:
|
||||
|
||||
def test_mixed_text_then_real_tool_call(self):
|
||||
"""Text mentioning tool syntax followed by a real special-token
|
||||
tool call."""
|
||||
tool call.
|
||||
"""
|
||||
engine = StreamingParserEngine(_hermes_config(), _make_hermes_tokenizer())
|
||||
|
||||
events1 = engine.feed("Mention <tool_call> in text. ", [1, 2, 3, 4])
|
||||
@@ -494,7 +502,8 @@ class TestReasoningTokenCounts:
|
||||
|
||||
def _func_prefix_config() -> ParserEngineConfig:
|
||||
"""Config mixing token-ID terminals (TOOL_START/END) with
|
||||
text-only terminals (FUNC_PREFIX) and fallback transitions."""
|
||||
text-only terminals (FUNC_PREFIX) and fallback transitions.
|
||||
"""
|
||||
return ParserEngineConfig(
|
||||
name="func_prefix_test",
|
||||
terminals={
|
||||
@@ -563,11 +572,13 @@ def _make_func_prefix_tokenizer():
|
||||
|
||||
class TestTextOnlyFallbackFiltering:
|
||||
"""When token IDs are available, transitions marked
|
||||
skip_in_token_id_mode should be skipped."""
|
||||
skip_in_token_id_mode should be skipped.
|
||||
"""
|
||||
|
||||
def test_func_prefix_in_prose_demoted_in_strict_mode(self):
|
||||
"""<function=get_time> in prose should NOT trigger a tool call
|
||||
when strict mode is active."""
|
||||
when strict mode is active.
|
||||
"""
|
||||
engine = StreamingParserEngine(
|
||||
_func_prefix_config(), _make_func_prefix_tokenizer()
|
||||
)
|
||||
@@ -584,7 +595,8 @@ class TestTextOnlyFallbackFiltering:
|
||||
|
||||
def test_normal_flow_after_tool_start_still_works(self):
|
||||
"""TOOL_START (special token) -> FUNC_PREFIX (text) should
|
||||
still parse a tool call normally in strict mode."""
|
||||
still parse a tool call normally in strict mode.
|
||||
"""
|
||||
engine = StreamingParserEngine(
|
||||
_func_prefix_config(), _make_func_prefix_tokenizer()
|
||||
)
|
||||
@@ -604,7 +616,8 @@ class TestTextOnlyFallbackFiltering:
|
||||
|
||||
def test_fallback_fires_without_token_ids(self):
|
||||
"""When no token IDs are provided, fallback transitions should
|
||||
still fire normally."""
|
||||
still fire normally.
|
||||
"""
|
||||
engine = StreamingParserEngine(
|
||||
_func_prefix_config(), _make_func_prefix_tokenizer()
|
||||
)
|
||||
@@ -618,7 +631,8 @@ class TestTextOnlyFallbackFiltering:
|
||||
|
||||
def test_tool_between_fallback_blocked_in_strict_mode(self):
|
||||
"""The (TOOL_BETWEEN, FUNC_PREFIX) fallback should also be
|
||||
blocked in strict mode."""
|
||||
blocked in strict mode.
|
||||
"""
|
||||
engine = StreamingParserEngine(
|
||||
_func_prefix_config(), _make_func_prefix_tokenizer()
|
||||
)
|
||||
@@ -648,7 +662,8 @@ class TestNoUnusedTokenizerAttr:
|
||||
class TestArgsResetOnReentry:
|
||||
"""When leaving TOOL_ARGS and later re-entering (e.g. two tool
|
||||
calls), the entering-TOOL_ARGS block resets args tracking. The
|
||||
redundant reset on exit was removed."""
|
||||
redundant reset on exit was removed.
|
||||
"""
|
||||
|
||||
@staticmethod
|
||||
def _multi_tool_config() -> ParserEngineConfig:
|
||||
@@ -723,7 +738,8 @@ class TestArgsResetOnReentry:
|
||||
|
||||
class TestToolPreambleFinish:
|
||||
"""finish() in TOOL_PREAMBLE state emits TOOL_CALL_END when a tool
|
||||
call was started (tool_index >= 0), but not when tool_index is -1."""
|
||||
call was started (tool_index >= 0), but not when tool_index is -1.
|
||||
"""
|
||||
|
||||
@staticmethod
|
||||
def _preamble_with_tool_call_start_config() -> ParserEngineConfig:
|
||||
@@ -782,7 +798,8 @@ class TestToolPreambleFinish:
|
||||
|
||||
class TestRegexTerminalInfraRemoved:
|
||||
"""TerminalDef.priority, LexerShape.regex_terminals, and the regex
|
||||
matching loop were removed."""
|
||||
matching loop were removed.
|
||||
"""
|
||||
|
||||
def test_terminal_def_no_priority(self):
|
||||
import regex as re
|
||||
@@ -807,7 +824,8 @@ class TestRegexTerminalInfraRemoved:
|
||||
|
||||
class TestMultiCharTerminalInArgs:
|
||||
"""Regression: multi-char terminals falling through in TOOL_ARGS
|
||||
must be fed char-by-char via _feed_args_text, not _feed_args_char."""
|
||||
must be fed char-by-char via _feed_args_text, not _feed_args_char.
|
||||
"""
|
||||
|
||||
@staticmethod
|
||||
def _newline_config() -> ParserEngineConfig:
|
||||
@@ -899,7 +917,8 @@ class TestSkipToolParsing:
|
||||
|
||||
def _message_header_config(reasoning_end: bool) -> ParserEngineConfig:
|
||||
"""MESSAGE_HEADER-based grammar whose tool opener optionally carries
|
||||
REASONING_END, mirroring the two shapes a transition table can take."""
|
||||
REASONING_END, mirroring the two shapes a transition table can take.
|
||||
"""
|
||||
return ParserEngineConfig(
|
||||
name=f"message_header_test_{reasoning_end}",
|
||||
initial_state=ParserState.MESSAGE_HEADER,
|
||||
|
||||
@@ -1097,7 +1097,8 @@ class TestNonStreamingReasoningPlusToolCalls:
|
||||
|
||||
def test_both_extractions_independent(self, parser, request_obj):
|
||||
"""Calling extract_reasoning then extract_tool_calls on the same
|
||||
parser instance should both work (each resets the engine)."""
|
||||
parser instance should both work (each resets the engine).
|
||||
"""
|
||||
model_output = FULL_MODEL_OUTPUT
|
||||
|
||||
reasoning, _ = parser.extract_reasoning(model_output, request_obj)
|
||||
@@ -1111,7 +1112,8 @@ class TestNonStreamingReasoningPlusToolCalls:
|
||||
|
||||
class TestAdapterExtractReasoning:
|
||||
"""The reasoning adapter's extract_reasoning uses skip_tool_parsing
|
||||
so tool call text is preserved as content for the tool adapter."""
|
||||
so tool call text is preserved as content for the tool adapter.
|
||||
"""
|
||||
|
||||
@pytest.fixture
|
||||
def adapter(self, mock_tokenizer):
|
||||
@@ -1152,7 +1154,8 @@ class TestAdapterExtractReasoning:
|
||||
class TestGemma4SchemaAwareTypeCoercion:
|
||||
"""Verify that streaming and non-streaming produce identical
|
||||
type-fixed arguments when tool schemas declare string parameters
|
||||
but the model outputs bare numbers/booleans."""
|
||||
but the model outputs bare numbers/booleans.
|
||||
"""
|
||||
|
||||
@pytest.fixture
|
||||
def tools(self):
|
||||
@@ -1182,7 +1185,8 @@ class TestGemma4SchemaAwareTypeCoercion:
|
||||
|
||||
def test_streaming_string_param_not_coerced(self, parser_with_tools, mock_request):
|
||||
"""A numeric value for a string-typed param must remain a string
|
||||
in the streamed output, matching the non-streaming result."""
|
||||
in the streamed output, matching the non-streaming result.
|
||||
"""
|
||||
chunks = [
|
||||
"<|tool_call>",
|
||||
"call:update_record{",
|
||||
@@ -1213,7 +1217,8 @@ class TestGemma4SchemaAwareTypeCoercion:
|
||||
|
||||
def test_streaming_matches_non_streaming(self, parser_with_tools, mock_request):
|
||||
"""Concatenated streaming deltas must produce the same arguments
|
||||
as non-streaming extraction."""
|
||||
as non-streaming extraction.
|
||||
"""
|
||||
text = "<|tool_call>call:update_record{zipcode:12345}<tool_call|>"
|
||||
|
||||
non_streaming = parser_with_tools.extract_tool_calls(text, mock_request)
|
||||
@@ -1234,7 +1239,8 @@ class TestGemma4SchemaAwareTypeCoercion:
|
||||
|
||||
class TestGemma4SchemaCoercionBoolNumberNull:
|
||||
"""Verify that _fix_arg_types coerces string values to non-string
|
||||
schema types for the Gemma4 parser."""
|
||||
schema types for the Gemma4 parser.
|
||||
"""
|
||||
|
||||
@pytest.fixture
|
||||
def tools(self):
|
||||
@@ -1295,7 +1301,8 @@ class TestGemma4SchemaCoercionBoolNumberNull:
|
||||
|
||||
def test_streaming_type_stability(self, parser_with_tools, mock_request):
|
||||
"""Values streamed incrementally must not cause prefix
|
||||
incompatibility when types are coerced."""
|
||||
incompatibility when types are coerced.
|
||||
"""
|
||||
text = (
|
||||
"<|tool_call>call:configure{"
|
||||
"enabled:true,"
|
||||
@@ -1395,7 +1402,8 @@ BARE_THOUGHT_SEQUENCE.append((TOOL_CALL_END_ID, "<tool_call|>"))
|
||||
class TestBareThoughtWithoutChannelOpener:
|
||||
"""When the model omits <|channel> and starts with bare ``thought``,
|
||||
the parser should auto-inject the channel opener so reasoning is
|
||||
captured correctly."""
|
||||
captured correctly.
|
||||
"""
|
||||
|
||||
@pytest.fixture
|
||||
def bare_thought_tokenizer(self):
|
||||
@@ -1466,7 +1474,8 @@ class TestBareThoughtWithoutChannelOpener:
|
||||
|
||||
def test_bare_thought_token_at_end_of_stream(self, request_obj):
|
||||
"""When the stream ends with just "thought" (no \\n), the parser
|
||||
should treat it as the thought prefix token, not real reasoning."""
|
||||
should treat it as the thought prefix token, not real reasoning.
|
||||
"""
|
||||
seq: list[tuple[int, str]] = [
|
||||
(CHANNEL_START_ID, "<|channel>"),
|
||||
(3000, "thought"),
|
||||
|
||||
@@ -61,7 +61,8 @@ class TestIsReasoningEnd:
|
||||
|
||||
def test_previous_turn_tool_call_ignored(self, parser):
|
||||
"""A finished tool call from an earlier turn says nothing about the
|
||||
turn currently being generated."""
|
||||
turn currently being generated.
|
||||
"""
|
||||
prompt = [THINK_S, TEXT, TOOL_S, TEXT, TOOL_E, OBSERVATION, TEXT, ASSISTANT]
|
||||
assert not parser.is_reasoning_end(prompt)
|
||||
|
||||
@@ -88,7 +89,8 @@ class TestExtractContentIds:
|
||||
|
||||
def test_falls_back_to_tool_call(self, parser):
|
||||
"""Without ``</think>``, content starts at the opener itself so the
|
||||
tool parser still receives a well-formed call."""
|
||||
tool parser still receives a well-formed call.
|
||||
"""
|
||||
ids = [THINK_S, TEXT, TOOL_S, 30, TOOL_E]
|
||||
assert parser.extract_content_ids(ids) == [TOOL_S, 30, TOOL_E]
|
||||
|
||||
|
||||
@@ -70,7 +70,8 @@ _MARKERS = sorted(_TML_VOCAB, key=len, reverse=True)
|
||||
def _tokenize(text: str) -> list[tuple[int, str]]:
|
||||
"""Tokenize like the real stream: markers are atomic special tokens,
|
||||
plain text becomes one token per character (matching the mock
|
||||
tokenizer's ``chr``-based decode)."""
|
||||
tokenizer's ``chr``-based decode).
|
||||
"""
|
||||
tokens: list[tuple[int, str]] = []
|
||||
i = 0
|
||||
while i < len(text):
|
||||
@@ -87,7 +88,8 @@ def _tokenize(text: str) -> list[tuple[int, str]]:
|
||||
|
||||
def _stream(parser, request, text: str, chunk_size: int):
|
||||
"""Stream production-shaped deltas: ``chunk_size`` tokens per delta,
|
||||
with delta_token_ids covering every token (specials and text)."""
|
||||
with delta_token_ids covering every token (specials and text).
|
||||
"""
|
||||
tokens = _tokenize(text)
|
||||
results = []
|
||||
previous_text = ""
|
||||
@@ -119,7 +121,8 @@ def _stream(parser, request, text: str, chunk_size: int):
|
||||
def _stream_text_only(parser, request, text: str, chunk_size: int):
|
||||
"""Stream text-only deltas (no token ids), chunked at arbitrary
|
||||
character boundaries — exercises the text-lexing fallback path,
|
||||
including markers split across chunks."""
|
||||
including markers split across chunks.
|
||||
"""
|
||||
results = []
|
||||
previous_text = ""
|
||||
for start in range(0, len(text), chunk_size):
|
||||
@@ -170,7 +173,8 @@ def _stream_delegating(parser, request, text, chunk_size, prompt_token_ids):
|
||||
"""Stream ``text`` through ``DelegatingParser.parse_delta``, ``chunk_size``
|
||||
tokens per delta; return ``(content, reasoning, ordered tool names,
|
||||
ordered tool arguments)``. Arguments arrive in fragments, so they are
|
||||
concatenated per tool index."""
|
||||
concatenated per tool index.
|
||||
"""
|
||||
tokens = _tokenize(text)
|
||||
content, reasoning = "", ""
|
||||
tools: dict[int, str] = {}
|
||||
@@ -522,7 +526,8 @@ class TestToolCallFiltering:
|
||||
"""Inkling equivalents of the generic tool-call-filtering replay tests
|
||||
(Inkling is excluded from those in test_replay.py: its structural
|
||||
role/kind tokens and shared block-end token don't fit the generic
|
||||
reasoning/tool split model)."""
|
||||
reasoning/tool split model).
|
||||
"""
|
||||
|
||||
def test_skip_tool_parsing_round_trip(self, mock_tokenizer, mock_request):
|
||||
# First pass (reasoning adapter, skip_tool_parsing): reasoning is
|
||||
@@ -820,7 +825,8 @@ class TestDelegatingTwoPass:
|
||||
):
|
||||
"""The optional function name between ``<|message_model|>`` and the
|
||||
content-kind marker is metadata: the buffered header must be
|
||||
discarded on the way out, not flushed into content."""
|
||||
discarded on the way out, not flushed into content.
|
||||
"""
|
||||
tools = [_function_tool()]
|
||||
mock_request.tools = tools
|
||||
content, _, names, args = _stream_delegating(
|
||||
@@ -837,7 +843,8 @@ class TestDelegatingTwoPass:
|
||||
def test_content_state_tool_start_streaming(self, mock_tokenizer, mock_request):
|
||||
"""Same opener reached from CONTENT rather than MESSAGE_HEADER: a
|
||||
text block closed with no ``<|message_model|>`` before the tool
|
||||
block. Preceding text must reach content exactly once, unmarked."""
|
||||
block. Preceding text must reach content exactly once, unmarked.
|
||||
"""
|
||||
tools = [_function_tool()]
|
||||
mock_request.tools = tools
|
||||
content, _, names, args = _stream_delegating(
|
||||
@@ -859,7 +866,8 @@ def test_content_tool_start_emits_reasoning_end_in_reasoning_pass():
|
||||
boundary (#49876), but the header-flush path
|
||||
(MESSAGE_HEADER --END_MESSAGE--> CONTENT) reaches CONTENT without
|
||||
one, so a tool block opening from there relies on this transition
|
||||
alone to hand off to the tool pass."""
|
||||
alone to hand off to the tool pass.
|
||||
"""
|
||||
engine = StreamingParserEngine(inkling_config(), tokenizer=None)
|
||||
engine.skip_tool_parsing = True
|
||||
engine.reset(initial_state=ParserState.CONTENT)
|
||||
@@ -876,7 +884,8 @@ def test_skip_reasoning_parsing_inert_for_shared_markers():
|
||||
markers. Inkling has none — ``<|end_message|>`` is labelled THINK_END
|
||||
yet also closes text, header, and tool blocks — so the flag must be
|
||||
inert: a tool block still closes through its transition and following
|
||||
text returns to CONTENT instead of leaking into the argument stream."""
|
||||
text returns to CONTENT instead of leaking into the argument stream.
|
||||
"""
|
||||
engine = StreamingParserEngine(inkling_config(), tokenizer=None)
|
||||
engine.skip_reasoning_parsing = True
|
||||
engine.reset(initial_state=ParserState.CONTENT)
|
||||
@@ -896,7 +905,8 @@ class TestToolParserWithoutReasoningParser:
|
||||
labels ``<|end_message|>`` as THINK_END; a bypass keyed on the label
|
||||
would neutralize the closer of every block kind, leaking markers into
|
||||
content and leaving tool calls unterminated. Structure must keep
|
||||
parsing exactly as with the reasoning parser attached."""
|
||||
parsing exactly as with the reasoning parser attached.
|
||||
"""
|
||||
|
||||
GEN_PROMPT = [_TML_VOCAB[MSG_MODEL]]
|
||||
|
||||
|
||||
@@ -56,7 +56,8 @@ def parser():
|
||||
class TestNemotronSwap:
|
||||
def test_enable_thinking_false_swaps(self, parser):
|
||||
"""When enable_thinking=False, model output without think tags
|
||||
should have reasoning swapped to content."""
|
||||
should have reasoning swapped to content.
|
||||
"""
|
||||
text = "The answer is 42."
|
||||
request = _make_request(enable_thinking=False)
|
||||
reasoning, content = parser.extract_reasoning(text, request)
|
||||
@@ -73,7 +74,8 @@ class TestNemotronSwap:
|
||||
|
||||
def test_no_swap_when_content_exists(self, parser):
|
||||
"""With enable_thinking=False but real </think> giving content,
|
||||
no swap occurs."""
|
||||
no swap occurs.
|
||||
"""
|
||||
text = "Some reasoning.</think>Actual content here."
|
||||
request = _make_request(enable_thinking=False)
|
||||
reasoning, content = parser.extract_reasoning(text, request)
|
||||
@@ -180,11 +182,13 @@ class TestStreaming:
|
||||
|
||||
class TestParseDeltaTokenIdFiltering:
|
||||
"""parse_delta must not trigger tool call parsing when <tool_call>
|
||||
appears as regular text rather than as a special token ID."""
|
||||
appears as regular text rather than as a special token ID.
|
||||
"""
|
||||
|
||||
def test_tool_call_text_in_reasoning_is_not_parsed(self, parser):
|
||||
"""Literal <tool_call> in model reasoning should be content,
|
||||
not a tool call."""
|
||||
not a tool call.
|
||||
"""
|
||||
request = _make_request()
|
||||
|
||||
text = (
|
||||
@@ -210,7 +214,8 @@ class TestParseDeltaTokenIdFiltering:
|
||||
|
||||
def test_special_token_id_still_triggers_tool_call(self, parser):
|
||||
"""When the scanner matches a special token ID, the tool call
|
||||
must still be parsed correctly."""
|
||||
must still be parsed correctly.
|
||||
"""
|
||||
request = _make_request()
|
||||
|
||||
parser.parse_delta(
|
||||
@@ -250,7 +255,8 @@ class TestParseDeltaTokenIdFiltering:
|
||||
|
||||
def test_text_discussion_then_real_tool_call(self, parser):
|
||||
"""Model discusses tool syntax in reasoning, then makes a real
|
||||
tool call via special tokens."""
|
||||
tool call via special tokens.
|
||||
"""
|
||||
request = _make_request()
|
||||
|
||||
r1 = parser.parse_delta(
|
||||
|
||||
@@ -323,7 +323,8 @@ class TestEventsToDelta:
|
||||
|
||||
def test_multiple_arg_chunks_same_batch_coalesced(self):
|
||||
"""Multiple events for the same tool in one batch must produce
|
||||
at most one DeltaToolCall per index."""
|
||||
at most one DeltaToolCall per index.
|
||||
"""
|
||||
engine = _make_engine()
|
||||
events = [
|
||||
SemanticEvent(EventType.TOOL_CALL_START, tool_index=0),
|
||||
@@ -551,7 +552,8 @@ class TestPostToolContentDeferral:
|
||||
"""Regression: content after TOOL_CALL_END in the same batch must not
|
||||
produce a mixed DeltaMessage(content=..., tool_calls=...) — that causes
|
||||
split_delta to reorder content before tool_calls, breaking the Responses
|
||||
API state machine."""
|
||||
API state machine.
|
||||
"""
|
||||
|
||||
def test_text_after_tool_end_deferred(self):
|
||||
engine = _make_engine()
|
||||
@@ -605,7 +607,8 @@ class TestPostToolContentDeferral:
|
||||
"""Deferred content from batch N must not mix with arg-continuation
|
||||
tool events in batch N+1 — that creates a DeltaMessage with both
|
||||
content and nameless tool_calls, which crashes the Responses API
|
||||
state machine (name=None → Pydantic ValidationError)."""
|
||||
state machine (name=None → Pydantic ValidationError).
|
||||
"""
|
||||
engine = _make_engine()
|
||||
engine._content_has_nonws = True
|
||||
|
||||
@@ -834,7 +837,8 @@ class TestBuildExtractedResult:
|
||||
|
||||
class TestEngineBasedPath:
|
||||
"""Tests for the _engine_based accumulation behavior in
|
||||
DelegatingParser.parse_delta."""
|
||||
DelegatingParser.parse_delta.
|
||||
"""
|
||||
|
||||
def test_engine_based_true_when_both_parsers_engine(self):
|
||||
r = SimpleNamespace(engine_based_streaming=True)
|
||||
@@ -880,7 +884,8 @@ class TestEngineBasedPath:
|
||||
|
||||
class TestParseTokenIdPassthrough:
|
||||
"""parse() must forward model_output_token_ids to _single_pass_parse
|
||||
so that token-ID-based strict terminal matching is active."""
|
||||
so that token-ID-based strict terminal matching is active.
|
||||
"""
|
||||
|
||||
def test_literal_tool_tag_in_content_preserved_with_token_ids(self, mock_request):
|
||||
engine = _make_engine(_hermes_config())
|
||||
@@ -1104,7 +1109,8 @@ class TestAdapterFinishOnStreamEnd:
|
||||
|
||||
def test_lexer_buffer_flushed_on_finished(self):
|
||||
"""Text buffered as a potential terminal prefix must be emitted
|
||||
as content when the stream ends."""
|
||||
as content when the stream ends.
|
||||
"""
|
||||
tokenizer = make_mock_tokenizer(_VOCAB)
|
||||
parser = _CombinedDelegating(tokenizer)
|
||||
request = _make_delegating_request()
|
||||
@@ -1122,7 +1128,8 @@ class TestAdapterFinishOnStreamEnd:
|
||||
|
||||
def test_args_buffer_flushed_on_finished(self):
|
||||
"""Pending arg buffer text must be emitted when stream ends
|
||||
mid-tool-call (closing brace held back in buffer)."""
|
||||
mid-tool-call (closing brace held back in buffer).
|
||||
"""
|
||||
tokenizer = make_mock_tokenizer(_VOCAB)
|
||||
parser = _CombinedDelegating(tokenizer)
|
||||
request = _make_delegating_request()
|
||||
@@ -1358,7 +1365,8 @@ class TestSkipToolSpanForwarding:
|
||||
|
||||
class TestToolAdapterForwardsKwargs:
|
||||
"""ParserEngineToolAdapter.__init__ must forward **kwargs to the
|
||||
parser engine class so chat_template_kwargs reach model parsers."""
|
||||
parser engine class so chat_template_kwargs reach model parsers.
|
||||
"""
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"enable_thinking,expected_state",
|
||||
@@ -1388,7 +1396,8 @@ class TestToolAdapterForwardsKwargs:
|
||||
|
||||
class TestExtractContentIdsNoEmptyReturn:
|
||||
"""extract_content_ids must return input_ids (not []) when there is
|
||||
no THINK_END token ID and _reasoning_ended is True."""
|
||||
no THINK_END token ID and _reasoning_ended is True.
|
||||
"""
|
||||
|
||||
_NO_THINK_CONFIG = ParserEngineConfig(name="no_think_end", token_id_terminals={})
|
||||
|
||||
@@ -1786,7 +1795,8 @@ class TestDropSpecialTokens:
|
||||
|
||||
def test_drops_special_token_by_id_from_content(self):
|
||||
"""A special token (not a configured terminal) is dropped when
|
||||
it arrives as its actual token ID."""
|
||||
it arrives as its actual token ID.
|
||||
"""
|
||||
config = ParserEngineConfig(
|
||||
name="drop_content_test",
|
||||
terminals={},
|
||||
@@ -1821,7 +1831,8 @@ class TestDropSpecialTokens:
|
||||
|
||||
def test_drops_via_text_fallback_when_no_token_ids(self):
|
||||
"""When no token IDs are provided, text-based lexer catches
|
||||
drop tokens as a fallback."""
|
||||
drop tokens as a fallback.
|
||||
"""
|
||||
engine = _make_engine(
|
||||
vocab=_DROP_VOCAB,
|
||||
special_tokens=list(_DROP_VOCAB.keys()),
|
||||
@@ -1834,7 +1845,8 @@ class TestDropSpecialTokens:
|
||||
|
||||
def test_regular_tokens_spelling_special_survive(self):
|
||||
"""Regular tokens that spell out a drop-token string survive
|
||||
when token IDs prove they are not the special token."""
|
||||
when token IDs prove they are not the special token.
|
||||
"""
|
||||
vocab = {**_DROP_VOCAB, "h": 72, "<": 73, "bos": 74, ">": 75, "w": 76}
|
||||
engine = _make_engine(
|
||||
vocab=vocab,
|
||||
@@ -1850,7 +1862,8 @@ class TestDropSpecialTokens:
|
||||
|
||||
def test_configured_terminal_not_treated_as_drop(self):
|
||||
"""Tokens already in config.terminals (like <think>) are handled
|
||||
by the state machine, not the drop mechanism."""
|
||||
by the state machine, not the drop mechanism.
|
||||
"""
|
||||
engine = _make_engine(
|
||||
vocab=_DROP_VOCAB,
|
||||
special_tokens=list(_DROP_VOCAB.keys()),
|
||||
@@ -1905,7 +1918,8 @@ class TestDropSpecialTokens:
|
||||
def test_drops_applied_with_skip_tool_parsing(self):
|
||||
"""Drop tokens are always dropped, even with skip_tool_parsing.
|
||||
DROP_TERMINALs have no transitions by construction, so no parser
|
||||
pass can use them."""
|
||||
pass can use them.
|
||||
"""
|
||||
for initial_state in (ParserState.REASONING, ParserState.CONTENT):
|
||||
engine = _make_engine(
|
||||
vocab=_DROP_VOCAB,
|
||||
@@ -1921,7 +1935,8 @@ class TestDropSpecialTokens:
|
||||
|
||||
def test_transitions_unaffected_by_drop_in_reasoning_with_skip_tool_parsing(self):
|
||||
"""With skip_tool_parsing in REASONING state, drop tokens are
|
||||
removed but configured terminals still fire their transitions."""
|
||||
removed but configured terminals still fire their transitions.
|
||||
"""
|
||||
engine = _make_engine(
|
||||
vocab=_DROP_VOCAB,
|
||||
special_tokens=list(_DROP_VOCAB.keys()),
|
||||
@@ -1957,7 +1972,8 @@ class TestDropSpecialTokens:
|
||||
|
||||
def test_mixed_configured_and_drop_terminals(self):
|
||||
"""Configured terminals trigger transitions while drop terminals
|
||||
are silently removed in the same stream."""
|
||||
are silently removed in the same stream.
|
||||
"""
|
||||
engine = _make_engine(
|
||||
vocab=_DROP_VOCAB,
|
||||
special_tokens=list(_DROP_VOCAB.keys()),
|
||||
@@ -1981,7 +1997,8 @@ class TestTruncatedToolOpenerStreamParity:
|
||||
"""Regression tests for #47137: when generation terminates inside a
|
||||
``<tool_call>`` opener that has not been promoted to a tool call
|
||||
(via ``max_tokens`` or a ``stop`` string), the non-streaming path
|
||||
must drop the incomplete markup, matching the streaming path."""
|
||||
must drop the incomplete markup, matching the streaming path.
|
||||
"""
|
||||
|
||||
_QWEN3_VOCAB = {
|
||||
"<tool_call>": 100,
|
||||
@@ -2027,7 +2044,8 @@ class TestTruncatedToolOpenerStreamParity:
|
||||
)
|
||||
def test_truncated_opener_dropped_in_both_paths(self, mock_request, chunks):
|
||||
"""A cutoff inside the opener yields no content and no tool calls
|
||||
in both the non-streaming and streaming paths."""
|
||||
in both the non-streaming and streaming paths.
|
||||
"""
|
||||
text = "".join(chunks)
|
||||
|
||||
_, content, tool_calls = self._make_parser().parse(
|
||||
@@ -2040,7 +2058,8 @@ class TestTruncatedToolOpenerStreamParity:
|
||||
|
||||
def test_content_before_truncated_opener_preserved(self, mock_request):
|
||||
"""Only the incomplete markup is dropped; content generated before
|
||||
the opener is returned identically by both paths."""
|
||||
the opener is returned identically by both paths.
|
||||
"""
|
||||
chunks = ["Checking the weather. ", "<tool_call>", "\n", "<function"]
|
||||
text = "".join(chunks)
|
||||
|
||||
@@ -2054,7 +2073,8 @@ class TestTruncatedToolOpenerStreamParity:
|
||||
|
||||
def test_content_around_unpromoted_tool_block_stays_ordered(self, mock_request):
|
||||
"""Text surrounding a complete-but-unparsable tool block keeps its
|
||||
original order in the non-streaming path, matching streaming."""
|
||||
original order in the non-streaming path, matching streaming.
|
||||
"""
|
||||
chunks = ["A ", "<tool_call>", "garbage", "</tool_call>", " B"]
|
||||
text = "".join(chunks)
|
||||
|
||||
@@ -2068,7 +2088,8 @@ class TestTruncatedToolOpenerStreamParity:
|
||||
|
||||
def test_complete_tool_call_still_promoted(self, mock_request):
|
||||
"""Sanity check: a complete tool call still parses in the
|
||||
non-streaming path after the truncation fix."""
|
||||
non-streaming path after the truncation fix.
|
||||
"""
|
||||
text = (
|
||||
"<tool_call>\n"
|
||||
"<function=get_weather>\n"
|
||||
@@ -2088,7 +2109,8 @@ class TestThinkMarkupWithoutReasoningParser:
|
||||
"""With no reasoning parser configured, reasoning markup is plain
|
||||
content: engine-backed tool parsers must return it verbatim in both
|
||||
the non-streaming and streaming paths, not consume the markers or
|
||||
reclassify the block as reasoning and drop it."""
|
||||
reclassify the block as reasoning and drop it.
|
||||
"""
|
||||
|
||||
_VOCAB = {
|
||||
"<think>": 90,
|
||||
@@ -2154,7 +2176,8 @@ class TestThinkMarkupWithoutReasoningParser:
|
||||
|
||||
def test_think_block_preserved_alongside_tool_call(self, mock_request):
|
||||
"""The tools-called branch also returns the tool parser's content;
|
||||
a think block preceding a promoted tool call must survive in it."""
|
||||
a think block preceding a promoted tool call must survive in it.
|
||||
"""
|
||||
think = "<think>plan</think>"
|
||||
text = think + (
|
||||
"<tool_call>\n"
|
||||
|
||||
@@ -318,7 +318,8 @@ class TestStreaming:
|
||||
|
||||
def test_streaming_args_arrive_incrementally(self, parser, mock_request):
|
||||
"""Arguments must stream as intermediate deltas, not batch at
|
||||
tool-end."""
|
||||
tool-end.
|
||||
"""
|
||||
chunks = [
|
||||
"<tool_call>\n",
|
||||
"<function=get_weather>\n",
|
||||
@@ -523,7 +524,8 @@ class TestStreaming:
|
||||
|
||||
def test_streaming_split_tool_call_tag(self, parser, mock_request):
|
||||
"""<tool_call> arrives as a single special token; the rest of
|
||||
the content is split into fine-grained chunks."""
|
||||
the content is split into fine-grained chunks.
|
||||
"""
|
||||
chunks = [
|
||||
"<tool_call>\n",
|
||||
"<function=test>\n",
|
||||
@@ -700,7 +702,8 @@ class TestArgConverter:
|
||||
|
||||
class TestSchemaAwareTypeCoercion:
|
||||
"""Verify that _fix_arg_types corrects miscoerced values using the
|
||||
tool schema."""
|
||||
tool schema.
|
||||
"""
|
||||
|
||||
@pytest.fixture
|
||||
def tools(self):
|
||||
@@ -842,7 +845,8 @@ class TestAnyOfTypeCoercion:
|
||||
|
||||
def test_anyof_string_param_not_coerced(self, parser_with_anyof, mock_request):
|
||||
"""A param with anyOf including 'string' must not be coerced
|
||||
to integer."""
|
||||
to integer.
|
||||
"""
|
||||
text = (
|
||||
"<tool_call>\n"
|
||||
"<function=set_config>\n"
|
||||
@@ -858,7 +862,8 @@ class TestAnyOfTypeCoercion:
|
||||
|
||||
class TestSchemaCoercionBoolNumberNull:
|
||||
"""Verify that _fix_arg_types coerces string values to non-string
|
||||
schema types using coerce_to_schema_type."""
|
||||
schema types using coerce_to_schema_type.
|
||||
"""
|
||||
|
||||
@pytest.fixture
|
||||
def tools(self):
|
||||
|
||||
@@ -276,7 +276,8 @@ class TestIsReasoningEndTurnBoundaries:
|
||||
|
||||
def test_empty_boundary_config_keeps_global_walk(self):
|
||||
"""Configs without turn_boundary_tokens keep the pre-existing
|
||||
behavior: any </think> in the sequence ends reasoning."""
|
||||
behavior: any </think> in the sequence ends reasoning.
|
||||
"""
|
||||
cfg = qwen3_config(turn_boundary_tokens=frozenset())
|
||||
parser = Qwen3Parser(
|
||||
make_mock_tokenizer(_QWEN3_VOCAB), parser_engine_config=cfg
|
||||
@@ -288,7 +289,8 @@ class TestIsReasoningEndTurnBoundaries:
|
||||
def test_boundary_tokens_missing_from_vocab_keep_global_walk(self):
|
||||
"""Boundary tokens absent from the vocabulary resolve to nothing,
|
||||
so subclasses reusing the default on a non-ChatML vocab keep the
|
||||
pre-existing behavior."""
|
||||
pre-existing behavior.
|
||||
"""
|
||||
vocab = {
|
||||
token: token_id
|
||||
for token, token_id in _QWEN3_VOCAB.items()
|
||||
@@ -299,7 +301,8 @@ class TestIsReasoningEndTurnBoundaries:
|
||||
|
||||
def test_boundary_with_thinking_disabled_is_end(self, mock_tokenizer):
|
||||
"""With thinking disabled (initial state CONTENT) a walk that stops at a turn
|
||||
boundary must report reasoning as ended: the model never emits a marker."""
|
||||
boundary must report reasoning as ended: the model never emits a marker.
|
||||
"""
|
||||
parser = Qwen3Parser(
|
||||
mock_tokenizer, chat_template_kwargs={"enable_thinking": False}
|
||||
)
|
||||
@@ -307,7 +310,8 @@ class TestIsReasoningEndTurnBoundaries:
|
||||
|
||||
def test_boundary_with_thinking_enabled_not_end(self, parser):
|
||||
"""With thinking disabled (initial state REASONING) a walk that stops at a turn
|
||||
boundary must keep reasoning open"""
|
||||
boundary must keep reasoning open
|
||||
"""
|
||||
assert not parser.is_reasoning_end([_IM_START_ID, _TEXT_ID])
|
||||
|
||||
|
||||
@@ -668,7 +672,8 @@ class TestTrailingWhitespaceStripping:
|
||||
|
||||
class TestWhitespaceStrippingDisabled:
|
||||
"""When strip_trailing_reasoning_whitespace is False,
|
||||
trailing whitespace in reasoning must be preserved."""
|
||||
trailing whitespace in reasoning must be preserved.
|
||||
"""
|
||||
|
||||
@pytest.fixture
|
||||
def parser_no_strip(self):
|
||||
@@ -726,7 +731,8 @@ class TestThinkingDisabled:
|
||||
def test_thinking_disabled_streaming_content_only(self, mock_tokenizer):
|
||||
"""Plain text with thinking disabled must stream as content, not
|
||||
reasoning. Before the fix, the REASONING initial state caused all
|
||||
output to be emitted as reasoning chunks."""
|
||||
output to be emitted as reasoning chunks.
|
||||
"""
|
||||
p = Qwen3Parser(
|
||||
mock_tokenizer,
|
||||
chat_template_kwargs={"enable_thinking": False},
|
||||
|
||||
@@ -440,7 +440,8 @@ class TestToolCallFilteringReplay:
|
||||
)
|
||||
class TestToolCallFilteringNonStreaming:
|
||||
"""Non-streaming parse() with tool_choice='none' must suppress tool
|
||||
calls and not leak special tokens into content."""
|
||||
calls and not leak special tokens into content.
|
||||
"""
|
||||
|
||||
def test_parse(self, parser_cls, sample, think_end, tool_start):
|
||||
tokenizer = make_mock_tokenizer(sample)
|
||||
@@ -483,7 +484,8 @@ _WS_TOOL_SAMPLES = [(t[0], t[1]) for t in _TOOL_CALL_SAMPLES if "whitespace" in
|
||||
)
|
||||
class TestToolChoiceNoneStreamingParity:
|
||||
"""Streaming and non-streaming must return the same content
|
||||
when tool_choice='none' suppresses tool calls."""
|
||||
when tool_choice='none' suppresses tool calls.
|
||||
"""
|
||||
|
||||
def test_content_matches(self, parser_cls, sample):
|
||||
tokenizer = make_mock_tokenizer(sample)
|
||||
@@ -541,7 +543,8 @@ def _inject_drop_tokens(sample):
|
||||
|
||||
class TestDropTokenReplay:
|
||||
"""Verify unconfigured special tokens are silently dropped across
|
||||
all parsers and chunk sizes."""
|
||||
all parsers and chunk sizes.
|
||||
"""
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"parser_info",
|
||||
@@ -598,7 +601,8 @@ class TestDropTokenNonStreaming:
|
||||
|
||||
class TestAdapterReferences:
|
||||
"""Verify make_adapters sets reasoning/tool parser class refs on parser engine
|
||||
parser classes so the serving layer finds them and calls adjust_request."""
|
||||
parser classes so the serving layer finds them and calls adjust_request.
|
||||
"""
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"parser_name",
|
||||
|
||||
@@ -129,7 +129,8 @@ def test_single_tool_call(tool_parser, mock_request):
|
||||
|
||||
def test_malformed_function_end_does_not_drop_siblings(tool_parser, mock_request):
|
||||
"""Regression for #46314: a malformed ``</function>`` with no closing ``>``
|
||||
on the header must not discard the other, well-formed calls."""
|
||||
on the header must not discard the other, well-formed calls.
|
||||
"""
|
||||
text = (
|
||||
f"{TOOL_CALL_START}\n<function=broken</function>\n{TOOL_CALL_END}"
|
||||
f"{TOOL_CALL_START}\n<function=get_weather>\n"
|
||||
@@ -171,7 +172,8 @@ def test_reasoning_then_tool_call(parser):
|
||||
|
||||
def test_streaming_think_end_and_tool_call_same_delta(parser):
|
||||
"""``</seed:think>`` and ``<seed:tool_call>`` arriving in one delta must
|
||||
not leak the terminal tokens into the reasoning text."""
|
||||
not leak the terminal tokens into the reasoning text.
|
||||
"""
|
||||
reasoning, content = simulate_reasoning_streaming(
|
||||
parser,
|
||||
[
|
||||
|
||||
@@ -146,7 +146,8 @@ class TestUfffdReasoningTransition:
|
||||
|
||||
def test_byte_fallback_tokenizer_produces_ufffd(self):
|
||||
"""Validate the fixture: decode() returns U+FFFD for isolated
|
||||
byte-fallback token IDs, proving the old code path would leak."""
|
||||
byte-fallback token IDs, proving the old code path would leak.
|
||||
"""
|
||||
vocab = dict(_MODEL_CONFIGS[0].values[0])
|
||||
tokenizer = ByteFallbackMockTokenizer(vocab, _SHARED_TOKENS, _SHARED_UFFFD_IDS)
|
||||
assert tokenizer.decode([200]) == "�"
|
||||
|
||||
@@ -992,7 +992,8 @@ def _inkling_block(
|
||||
) -> None:
|
||||
"""Append one Inkling content block; blocks after the first start with
|
||||
the ``<|message_model|>`` role token (the first block continues the
|
||||
generation prompt directly)."""
|
||||
generation prompt directly).
|
||||
"""
|
||||
if segs:
|
||||
segs.append(("<|message_model|>", True))
|
||||
segs.append((kind_token, True))
|
||||
|
||||
@@ -492,7 +492,8 @@ def test_extract_tool_calls_pre_v11_regex_fallback(
|
||||
):
|
||||
"""The regex fallback path finds valid JSON via regex when the primary
|
||||
raw_decode fails on leading junk. It should re-serialize arguments
|
||||
and return a valid tool call."""
|
||||
and return a valid tool call.
|
||||
"""
|
||||
model_output = (
|
||||
'[TOOL_CALLS] junk [{"name": "add", "arguments":{"a": 1, "b": 2}}] trail'
|
||||
)
|
||||
@@ -2003,7 +2004,8 @@ def test_streaming_pre_v11_parallel_calls_batched_deltas(
|
||||
mistral_pre_v11_tool_parser, mistral_pre_v11_tokenizer, chunk_size, driver
|
||||
):
|
||||
"""A batched delta spanning the boundary between two parallel calls must
|
||||
keep them on distinct indices (the bug collapsed both onto index 0)."""
|
||||
keep them on distinct indices (the bug collapsed both onto index 0).
|
||||
"""
|
||||
model_output = (
|
||||
'[TOOL_CALLS] [{"name": "add", "arguments": {"a": 3.5, "b": 4}}, '
|
||||
'{"name": "get_current_weather", "arguments": '
|
||||
@@ -2045,7 +2047,6 @@ def test_content_tool_calls_transition_emits_reasoning_end(
|
||||
reasoning_encoding,
|
||||
):
|
||||
"""(CONTENT, TOOL_CALLS) must emit REASONING_END when reasoning is enabled."""
|
||||
|
||||
cfg = mistral_config(reasoning_encoding=reasoning_encoding)
|
||||
engine = StreamingParserEngine(
|
||||
config=cfg, tokenizer=mistral_tokenizer, vocab=mistral_tokenizer.get_vocab()
|
||||
@@ -2122,7 +2123,8 @@ def test_reasoning_active_no_think_block_no_leak(
|
||||
chunk_size,
|
||||
):
|
||||
"""Tool call without a preceding think block must not leak as content
|
||||
when the reasoning parser is active."""
|
||||
when the reasoning parser is active.
|
||||
"""
|
||||
accumulated_content = ""
|
||||
function_names: list[str] = []
|
||||
function_args_strs: list[str] = []
|
||||
@@ -2219,11 +2221,12 @@ def test_adjust_request_pre_v11_required_clears_response_format(
|
||||
mistral_pre_v11_tool_parser: MistralToolParser,
|
||||
response_format: dict,
|
||||
) -> None:
|
||||
"""required + response_format must inject the tool schema and clear
|
||||
"""Required + response_format must inject the tool schema and clear
|
||||
response_format so the tool schema is the sole structured-output
|
||||
constraint, matching the base ToolParser (otherwise the request either
|
||||
hits the "multiple constraints" engine error or, with no constraint,
|
||||
rambles to finish_reason='length')."""
|
||||
rambles to finish_reason='length').
|
||||
"""
|
||||
request = _make_request(tool_choice="required", response_format=response_format)
|
||||
result = mistral_pre_v11_tool_parser.adjust_request(request)
|
||||
|
||||
@@ -2240,7 +2243,8 @@ def test_adjust_request_non_mistral_tokenizer_required_injects_schema(
|
||||
) -> None:
|
||||
"""The guided-schema injection must also fire for non-Mistral (e.g. HF-mode)
|
||||
tokenizers driving the Mistral tool parser, mirroring the base ToolParser so
|
||||
required + response_format does not fall back to an unconstrained ramble."""
|
||||
required + response_format does not fall back to an unconstrained ramble.
|
||||
"""
|
||||
request = _make_request(
|
||||
tool_choice="required", response_format={"type": "json_object"}
|
||||
)
|
||||
|
||||
@@ -229,7 +229,8 @@ def _boundary_chunks(tokenizer, parser, end_token_id=None):
|
||||
|
||||
def test_parse_delta_reasoning_not_dropped_on_boundary(tokenizer, request_obj):
|
||||
"""Regression: reasoning must not be lost when a multi-token delta
|
||||
spans the reasoning/tool-call boundary."""
|
||||
spans the reasoning/tool-call boundary.
|
||||
"""
|
||||
parser = make_parser(tokenizer, reasoning=True, tool=True)
|
||||
chunks = _boundary_chunks(tokenizer, parser)
|
||||
results = stream_chunks(parser, tokenizer, chunks, request_obj)
|
||||
@@ -247,7 +248,8 @@ def test_parse_delta_reasoning_not_dropped_on_boundary(tokenizer, request_obj):
|
||||
|
||||
def test_parse_delta_reasoning_boundary_no_tool_parser(tokenizer, request_obj):
|
||||
"""When no tool parser is active, boundary-spanning chunks must still
|
||||
preserve reasoning and pass post-</think> text as content."""
|
||||
preserve reasoning and pass post-</think> text as content.
|
||||
"""
|
||||
parser = make_parser(tokenizer, reasoning=True, tool=False)
|
||||
chunks = _boundary_chunks(tokenizer, parser)
|
||||
results = stream_chunks(parser, tokenizer, chunks, request_obj)
|
||||
@@ -261,7 +263,8 @@ def test_parse_delta_reasoning_boundary_no_tool_parser(tokenizer, request_obj):
|
||||
|
||||
def test_parse_delta_reasoning_only_no_think_leak(tokenizer, request_obj):
|
||||
"""Regression: </think> must not leak into content when streaming
|
||||
token-by-token with reasoning=True, tool=False."""
|
||||
token-by-token with reasoning=True, tool=False.
|
||||
"""
|
||||
parser = make_parser(tokenizer, reasoning=True, tool=False)
|
||||
results = stream_text(
|
||||
parser, tokenizer, MODEL_OUTPUT, request_obj, prompt_token_ids=[]
|
||||
@@ -303,7 +306,8 @@ def test_parse_delta_reasoning_only_thinking_disabled(tokenizer, request_obj):
|
||||
|
||||
def test_parse_delta_finished_no_flush_without_tool_call_delta(tokenizer, request_obj):
|
||||
"""When finished=True but the final parse_delta produces no
|
||||
tool-call delta, unstreamed args are not flushed."""
|
||||
tool-call delta, unstreamed args are not flushed.
|
||||
"""
|
||||
parser = make_parser(tokenizer, reasoning=False, tool=True)
|
||||
|
||||
results = stream_text(
|
||||
@@ -326,7 +330,8 @@ def test_parse_delta_finished_no_flush_without_tool_call_delta(tokenizer, reques
|
||||
|
||||
def test_parse_delta_finished_no_extra_args_when_fully_streamed(tokenizer, request_obj):
|
||||
"""When all args have been streamed, finished=True must not
|
||||
produce extra or duplicate arguments."""
|
||||
produce extra or duplicate arguments.
|
||||
"""
|
||||
parser = make_parser(tokenizer, reasoning=False, tool=True)
|
||||
results = stream_text(
|
||||
parser, tokenizer, MODEL_OUTPUT, request_obj, prompt_token_ids=[]
|
||||
@@ -346,7 +351,8 @@ def test_parse_delta_finished_no_extra_args_when_fully_streamed(tokenizer, reque
|
||||
|
||||
def test_parse_delta_finished_appends_remaining_args(tokenizer, request_obj):
|
||||
"""When finished=True and the tool parser has unstreamed args,
|
||||
parse_delta appends the remaining arguments to the tool-call delta."""
|
||||
parse_delta appends the remaining arguments to the tool-call delta.
|
||||
"""
|
||||
parser = make_parser(tokenizer, reasoning=False, tool=True)
|
||||
token_ids = tokenizer.encode(MODEL_OUTPUT, add_special_tokens=False)
|
||||
|
||||
@@ -493,7 +499,8 @@ def test_engine_reasoning_hermes_tool_token_by_token(tokenizer, request_obj):
|
||||
"""Qwen3 engine reasoning + Hermes tool parser, token-by-token.
|
||||
|
||||
Sanity check that the mixed engine/non-engine configuration works
|
||||
when tokens arrive one at a time (no deferred content)."""
|
||||
when tokens arrive one at a time (no deferred content).
|
||||
"""
|
||||
parser = Qwen3ReasoningHermesToolParser(tokenizer)
|
||||
|
||||
assert parser._reasoning_parser.engine_based_streaming is True
|
||||
@@ -520,7 +527,8 @@ def test_engine_reasoning_hermes_tool_boundary(tokenizer, request_obj):
|
||||
|
||||
When </think> and <tool_call> are in the same chunk with aligned
|
||||
text and token IDs, the engine processes both terminals and returns
|
||||
the <tool_call> text as content."""
|
||||
the <tool_call> text as content.
|
||||
"""
|
||||
parser = Qwen3ReasoningHermesToolParser(tokenizer)
|
||||
end_token_id = parser._reasoning_parser._parser_engine._reasoning_end_token_id
|
||||
chunks = _boundary_chunks(tokenizer, parser, end_token_id=end_token_id)
|
||||
@@ -551,7 +559,8 @@ def test_engine_reasoning_hermes_tool_text_holdback(tokenizer, request_obj):
|
||||
Without the fix, finish_streaming() is never called at the
|
||||
reasoning->tool transition when _engine_based is False, so the '<'
|
||||
is lost and the Hermes parser sees 'tool_call>...' instead of
|
||||
'<tool_call>...'."""
|
||||
'<tool_call>...'.
|
||||
"""
|
||||
parser = Qwen3ReasoningHermesToolParser(tokenizer)
|
||||
vocab = tokenizer.get_vocab()
|
||||
think_end_id = vocab["</think>"]
|
||||
@@ -642,7 +651,8 @@ def test_engine_reasoning_no_tool_batched_content_passthrough(tokenizer, request
|
||||
batched delta carries ``</think>`` plus the following content
|
||||
(as happens with stream_interval > 1). The post-``</think>`` text
|
||||
must be emitted as content -- not dropped, not reclassified as
|
||||
reasoning -- and the ``</think>`` marker must not leak either way."""
|
||||
reasoning -- and the ``</think>`` marker must not leak either way.
|
||||
"""
|
||||
parser = Qwen3ReasoningNoToolParser(tokenizer)
|
||||
assert parser._reasoning_parser.engine_based_streaming is True
|
||||
assert parser._tool_parser is None
|
||||
@@ -670,7 +680,8 @@ def _decode_stream_deltas(tokenizer, groups):
|
||||
This mirrors how vLLM's detokenizer feeds ``parse_delta`` in
|
||||
production: byte-level UTF-8 hold-back means a character whose bytes
|
||||
span multiple tokens is only surfaced once complete (a naive
|
||||
per-token ``decode`` would instead emit U+FFFD replacement chars)."""
|
||||
per-token ``decode`` would instead emit U+FFFD replacement chars).
|
||||
"""
|
||||
from tokenizers.decoders import DecodeStream
|
||||
|
||||
stream = DecodeStream(skip_special_tokens=False)
|
||||
@@ -702,7 +713,8 @@ def test_engine_reasoning_hermes_tool_multibyte_holdback(tokenizer, request_obj)
|
||||
``DecodeStream``. Its UTF-8 hold-back yields the correct
|
||||
character round-trip (a naive per-token decode would corrupt it),
|
||||
verifying the boundary stays byte-safe for multi-token
|
||||
characters."""
|
||||
characters.
|
||||
"""
|
||||
parser = Qwen3ReasoningHermesToolParser(tokenizer)
|
||||
vocab = tokenizer.get_vocab()
|
||||
think_end_id = vocab["</think>"]
|
||||
|
||||
Reference in New Issue
Block a user