[Bugfix] Fix the DSML token leakage in DSV4/3.2 (#40806)

Signed-off-by: chaunceyjiang <chaunceyjiang@gmail.com>
Signed-off-by: sfeng33 <4florafeng@gmail.com>
Co-authored-by: sfeng33 <4florafeng@gmail.com>
Co-authored-by: Windswithyou 1694599440@qq.com
This commit is contained in:
Chauncey
2026-04-26 08:58:50 +08:00
committed by GitHub
co-authored by sfeng33 Windswithyou 1694599440@qq.com
parent 12a3f6454b
commit c798593f0d
2 changed files with 76 additions and 23 deletions
@@ -484,6 +484,58 @@ class TestExtractToolCallsStreaming:
# Should have no tool call deltas yet
assert all(not d.tool_calls for d in deltas)
def test_no_marker_leak_chunked(self, parser):
"""Chunked streaming must NOT leak DSML start-marker fragments
as content (GitHub #40801)."""
full_text = build_tool_call("fn", {"k": "v"})
deltas = self._stream_chunked(parser, full_text, chunk_size=5)
content = "".join(d.content for d in deltas if d.content is not None)
assert content == ""
args_str = self._reconstruct_args(deltas)
assert json.loads(args_str) == {"k": "v"}
def test_no_marker_leak_with_prefix_chunked(self, parser):
"""Content before a tool call must not include start-marker
fragments when chunked (GitHub #40801)."""
full_text = "Hello!" + build_tool_call("fn", {"a": "b"})
deltas = self._stream_chunked(parser, full_text, chunk_size=5)
content = "".join(d.content for d in deltas if d.content is not None)
assert content == "Hello!"
assert "DSML" not in content
assert "<|" not in content
args_str = self._reconstruct_args(deltas)
assert json.loads(args_str) == {"a": "b"}
def test_no_marker_leak_char_by_char(self, parser):
"""Character-by-character streaming must not leak marker
fragments (GitHub #40801)."""
full_text = build_tool_call("fn", {"k": "v"})
deltas = self._stream_chunked(parser, full_text, chunk_size=1)
content = "".join(d.content for d in deltas if d.content is not None)
assert content == ""
args_str = self._reconstruct_args(deltas)
assert json.loads(args_str) == {"k": "v"}
def test_no_marker_leak_all_split_points(self, parser):
"""Start token split at every possible boundary must not
leak (GitHub #40801)."""
for chunk_size in range(1, len(FC_START) + 2):
p = make_parser()
full_text = build_tool_call("fn", {"k": "v"})
deltas = self._stream_chunked(p, full_text, chunk_size=chunk_size)
content = "".join(d.content for d in deltas if d.content is not None)
assert content == "", (
f"Leaked content {content!r} at chunk_size={chunk_size}"
)
def test_false_partial_marker_emitted(self, parser):
"""Text ending with a prefix of the start token that turns out
NOT to be a marker must still be emitted as content."""
full_text = "<|DSM some regular text"
deltas = self._stream_chunked(parser, full_text, chunk_size=3)
content = "".join(d.content for d in deltas if d.content is not None)
assert content == full_text
class TestDelimiterPreservation:
"""Regression: fast detokenization skipping DSML delimiters (PR #33964)."""