[Bugfix][Parser] Fix special tokens (EOS/BOS) leaking into reasoning content (#48748)
Signed-off-by: Ben Browning <56071+bbrowning@users.noreply.github.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
b44311b6ef
commit
3de4b2bf3c
@@ -920,3 +920,45 @@ class TestDelegatingParserLargeDelta:
|
|||||||
assert output.tool_calls[0]["name"] == "get_weather"
|
assert output.tool_calls[0]["name"] == "get_weather"
|
||||||
args = json.loads(output.tool_calls[0]["arguments"])
|
args = json.loads(output.tool_calls[0]["arguments"])
|
||||||
assert args == {"location": "Berlin"}
|
assert args == {"location": "Berlin"}
|
||||||
|
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
"chunk_size",
|
||||||
|
[1, 2, 3, 5, None],
|
||||||
|
ids=lambda c: f"chunk={c}",
|
||||||
|
)
|
||||||
|
def test_eos_not_leaked_when_reasoning_never_ends(self, chunk_size):
|
||||||
|
"""EOS must not leak into reasoning_content when the model never
|
||||||
|
emits </think> (generation ends while still in REASONING state)."""
|
||||||
|
eos_text = "<|end▁of▁sentence|>"
|
||||||
|
eos_id = 128801
|
||||||
|
vocab = {
|
||||||
|
**_DSV4_FULL_VOCAB,
|
||||||
|
eos_text: eos_id,
|
||||||
|
}
|
||||||
|
|
||||||
|
reasoning_text = "Good morning! How can I help you today?"
|
||||||
|
tokens: list[tuple[int, str]] = []
|
||||||
|
tid = 100
|
||||||
|
for word in reasoning_text.split(" "):
|
||||||
|
prefix = " " if tokens else ""
|
||||||
|
tokens.append((tid, prefix + word))
|
||||||
|
tid += 1
|
||||||
|
tokens.append((eos_id, eos_text))
|
||||||
|
|
||||||
|
tokenizer = MockTokenizer(vocab=vocab, tokens=tokens)
|
||||||
|
parser = _DeepSeekV4Delegating(
|
||||||
|
tokenizer,
|
||||||
|
chat_template_kwargs={"thinking": True},
|
||||||
|
)
|
||||||
|
deltas = replay_streaming(
|
||||||
|
parser,
|
||||||
|
tokens,
|
||||||
|
chunk_size=chunk_size,
|
||||||
|
finished_on_last=True,
|
||||||
|
)
|
||||||
|
output = collect_output(deltas)
|
||||||
|
|
||||||
|
assert reasoning_text in output.reasoning
|
||||||
|
assert eos_text not in output.reasoning
|
||||||
|
assert output.content == ""
|
||||||
|
assert output.tool_calls == []
|
||||||
|
|||||||
@@ -1624,19 +1624,41 @@ class TestDropSpecialTokens:
|
|||||||
assert delta is not None
|
assert delta is not None
|
||||||
assert "<bos>" in delta.reasoning
|
assert "<bos>" in delta.reasoning
|
||||||
|
|
||||||
def test_drops_suppressed_with_skip_tool_parsing(self):
|
def test_drops_applied_with_skip_tool_parsing(self):
|
||||||
"""When skip_tool_parsing is active, drop tokens are preserved
|
"""Drop tokens are always dropped, even with skip_tool_parsing.
|
||||||
as content so a later tool-call pass can see them."""
|
DROP_TERMINALs have no transitions by construction, so no parser
|
||||||
|
pass can use them."""
|
||||||
|
for initial_state in (ParserState.REASONING, ParserState.CONTENT):
|
||||||
|
engine = _make_engine(
|
||||||
|
vocab=_DROP_VOCAB,
|
||||||
|
special_tokens=list(_DROP_VOCAB.keys()),
|
||||||
|
)
|
||||||
|
engine._engine.skip_tool_parsing = True
|
||||||
|
engine._engine.reset(initial_state=initial_state)
|
||||||
|
events = engine._engine.feed("hello<bos>world", [72, 204, 73])
|
||||||
|
delta = engine._events_to_delta(events)
|
||||||
|
assert delta is not None
|
||||||
|
output = (delta.reasoning or "") + (delta.content or "")
|
||||||
|
assert "<bos>" not in output, f"<bos> leaked in state {initial_state}"
|
||||||
|
|
||||||
|
def test_transitions_unaffected_by_drop_in_reasoning_with_skip_tool_parsing(self):
|
||||||
|
"""With skip_tool_parsing in REASONING state, drop tokens are
|
||||||
|
removed but configured terminals still fire their transitions."""
|
||||||
engine = _make_engine(
|
engine = _make_engine(
|
||||||
vocab=_DROP_VOCAB,
|
vocab=_DROP_VOCAB,
|
||||||
special_tokens=list(_DROP_VOCAB.keys()),
|
special_tokens=list(_DROP_VOCAB.keys()),
|
||||||
)
|
)
|
||||||
engine._engine.skip_tool_parsing = True
|
engine._engine.skip_tool_parsing = True
|
||||||
engine._engine.reset()
|
engine._engine.reset()
|
||||||
events = engine._engine.feed("hello<bos>world", [72, 204, 73])
|
events = engine._engine.feed("thought<bos></think>answer", [72, 204, 201, 73])
|
||||||
delta = engine._events_to_delta(events)
|
types = [e.type for e in events]
|
||||||
assert delta is not None
|
assert EventType.REASONING_CHUNK in types
|
||||||
assert "<bos>" in delta.reasoning
|
assert EventType.REASONING_END in types
|
||||||
|
assert EventType.TEXT_CHUNK in types
|
||||||
|
reasoning_text = "".join(
|
||||||
|
e.value for e in events if e.type == EventType.REASONING_CHUNK
|
||||||
|
)
|
||||||
|
assert "<bos>" not in reasoning_text
|
||||||
|
|
||||||
def test_drops_in_tool_args_state(self):
|
def test_drops_in_tool_args_state(self):
|
||||||
"""Drop tokens in TOOL_ARGS state are silently discarded."""
|
"""Drop tokens in TOOL_ARGS state are silently discarded."""
|
||||||
|
|||||||
@@ -304,14 +304,7 @@ class StreamingParserEngine:
|
|||||||
transition = self.config.transitions.get(key)
|
transition = self.config.transitions.get(key)
|
||||||
|
|
||||||
if transition is None:
|
if transition is None:
|
||||||
if (
|
if self._has_drops and terminal == DROP_TERMINAL:
|
||||||
self._has_drops
|
|
||||||
and terminal == DROP_TERMINAL
|
|
||||||
# Preserve drop tokens when skip_tool_parsing is active so
|
|
||||||
# the reasoning pass doesn't silently remove tokens that a
|
|
||||||
# later tool-call pass might need to see.
|
|
||||||
and not self.skip_tool_parsing
|
|
||||||
):
|
|
||||||
return []
|
return []
|
||||||
return self._emit_for_state(value)
|
return self._emit_for_state(value)
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user