[CI] Ignore ruff D209, rejoin the docstrings it split, and silence incompatible-rule warnings (#57212)

Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Harry Mellor
2026-09-16 10:57:24 -07:00
committed by GitHub
co-authored by Claude Opus 5
parent 1085b64425
commit 651a88c09a
693 changed files with 2088 additions and 4171 deletions
+12 -24
View File
@@ -229,8 +229,7 @@ def _boundary_chunks(tokenizer, parser, end_token_id=None):
def test_parse_delta_reasoning_not_dropped_on_boundary(tokenizer, request_obj):
"""Regression: reasoning must not be lost when a multi-token delta
spans the reasoning/tool-call boundary.
"""
spans the reasoning/tool-call boundary."""
parser = make_parser(tokenizer, reasoning=True, tool=True)
chunks = _boundary_chunks(tokenizer, parser)
results = stream_chunks(parser, tokenizer, chunks, request_obj)
@@ -248,8 +247,7 @@ def test_parse_delta_reasoning_not_dropped_on_boundary(tokenizer, request_obj):
def test_parse_delta_reasoning_boundary_no_tool_parser(tokenizer, request_obj):
"""When no tool parser is active, boundary-spanning chunks must still
preserve reasoning and pass post-</think> text as content.
"""
preserve reasoning and pass post-</think> text as content."""
parser = make_parser(tokenizer, reasoning=True, tool=False)
chunks = _boundary_chunks(tokenizer, parser)
results = stream_chunks(parser, tokenizer, chunks, request_obj)
@@ -263,8 +261,7 @@ def test_parse_delta_reasoning_boundary_no_tool_parser(tokenizer, request_obj):
def test_parse_delta_reasoning_only_no_think_leak(tokenizer, request_obj):
"""Regression: </think> must not leak into content when streaming
token-by-token with reasoning=True, tool=False.
"""
token-by-token with reasoning=True, tool=False."""
parser = make_parser(tokenizer, reasoning=True, tool=False)
results = stream_text(
parser, tokenizer, MODEL_OUTPUT, request_obj, prompt_token_ids=[]
@@ -306,8 +303,7 @@ def test_parse_delta_reasoning_only_thinking_disabled(tokenizer, request_obj):
def test_parse_delta_finished_no_flush_without_tool_call_delta(tokenizer, request_obj):
"""When finished=True but the final parse_delta produces no
tool-call delta, unstreamed args are not flushed.
"""
tool-call delta, unstreamed args are not flushed."""
parser = make_parser(tokenizer, reasoning=False, tool=True)
results = stream_text(
@@ -330,8 +326,7 @@ def test_parse_delta_finished_no_flush_without_tool_call_delta(tokenizer, reques
def test_parse_delta_finished_no_extra_args_when_fully_streamed(tokenizer, request_obj):
"""When all args have been streamed, finished=True must not
produce extra or duplicate arguments.
"""
produce extra or duplicate arguments."""
parser = make_parser(tokenizer, reasoning=False, tool=True)
results = stream_text(
parser, tokenizer, MODEL_OUTPUT, request_obj, prompt_token_ids=[]
@@ -351,8 +346,7 @@ def test_parse_delta_finished_no_extra_args_when_fully_streamed(tokenizer, reque
def test_parse_delta_finished_appends_remaining_args(tokenizer, request_obj):
"""When finished=True and the tool parser has unstreamed args,
parse_delta appends the remaining arguments to the tool-call delta.
"""
parse_delta appends the remaining arguments to the tool-call delta."""
parser = make_parser(tokenizer, reasoning=False, tool=True)
token_ids = tokenizer.encode(MODEL_OUTPUT, add_special_tokens=False)
@@ -499,8 +493,7 @@ def test_engine_reasoning_hermes_tool_token_by_token(tokenizer, request_obj):
"""Qwen3 engine reasoning + Hermes tool parser, token-by-token.
Sanity check that the mixed engine/non-engine configuration works
when tokens arrive one at a time (no deferred content).
"""
when tokens arrive one at a time (no deferred content)."""
parser = Qwen3ReasoningHermesToolParser(tokenizer)
assert parser._reasoning_parser.engine_based_streaming is True
@@ -527,8 +520,7 @@ def test_engine_reasoning_hermes_tool_boundary(tokenizer, request_obj):
When </think> and <tool_call> are in the same chunk with aligned
text and token IDs, the engine processes both terminals and returns
the <tool_call> text as content.
"""
the <tool_call> text as content."""
parser = Qwen3ReasoningHermesToolParser(tokenizer)
end_token_id = parser._reasoning_parser._parser_engine._reasoning_end_token_id
chunks = _boundary_chunks(tokenizer, parser, end_token_id=end_token_id)
@@ -559,8 +551,7 @@ def test_engine_reasoning_hermes_tool_text_holdback(tokenizer, request_obj):
Without the fix, finish_streaming() is never called at the
reasoning->tool transition when _engine_based is False, so the '<'
is lost and the Hermes parser sees 'tool_call>...' instead of
'<tool_call>...'.
"""
'<tool_call>...'."""
parser = Qwen3ReasoningHermesToolParser(tokenizer)
vocab = tokenizer.get_vocab()
think_end_id = vocab["</think>"]
@@ -651,8 +642,7 @@ def test_engine_reasoning_no_tool_batched_content_passthrough(tokenizer, request
batched delta carries ``</think>`` plus the following content
(as happens with stream_interval > 1). The post-``</think>`` text
must be emitted as content -- not dropped, not reclassified as
reasoning -- and the ``</think>`` marker must not leak either way.
"""
reasoning -- and the ``</think>`` marker must not leak either way."""
parser = Qwen3ReasoningNoToolParser(tokenizer)
assert parser._reasoning_parser.engine_based_streaming is True
assert parser._tool_parser is None
@@ -680,8 +670,7 @@ def _decode_stream_deltas(tokenizer, groups):
This mirrors how vLLM's detokenizer feeds ``parse_delta`` in
production: byte-level UTF-8 hold-back means a character whose bytes
span multiple tokens is only surfaced once complete (a naive
per-token ``decode`` would instead emit U+FFFD replacement chars).
"""
per-token ``decode`` would instead emit U+FFFD replacement chars)."""
from tokenizers.decoders import DecodeStream
stream = DecodeStream(skip_special_tokens=False)
@@ -713,8 +702,7 @@ def test_engine_reasoning_hermes_tool_multibyte_holdback(tokenizer, request_obj)
``DecodeStream``. Its UTF-8 hold-back yields the correct
character round-trip (a naive per-token decode would corrupt it),
verifying the boundary stays byte-safe for multi-token
characters.
"""
characters."""
parser = Qwen3ReasoningHermesToolParser(tokenizer)
vocab = tokenizer.get_vocab()
think_end_id = vocab["</think>"]