From 859e4d436ba0fb0da8a655a80d5c4fab12adc82e Mon Sep 17 00:00:00 2001 From: Ben Browning Date: Fri, 19 Jun 2026 18:09:28 -0400 Subject: [PATCH] [Bugfix][Parser] Fix U+FFFD leak at reasoning-to-content transition in engine parsers (#46159) Signed-off-by: Ben Browning --- tests/parser/engine/replay_harness.py | 3 + tests/parser/engine/test_delegating_replay.py | 3 +- .../engine/test_ufffd_reasoning_transition.py | 181 ++++++++++++++++++ vllm/parser/abstract_parser.py | 7 +- vllm/parser/engine/parser_engine.py | 4 +- 5 files changed, 191 insertions(+), 7 deletions(-) create mode 100644 tests/parser/engine/test_ufffd_reasoning_transition.py diff --git a/tests/parser/engine/replay_harness.py b/tests/parser/engine/replay_harness.py index fac643390b..9abd460f76 100644 --- a/tests/parser/engine/replay_harness.py +++ b/tests/parser/engine/replay_harness.py @@ -96,6 +96,9 @@ class MockTokenizer: return "".join(parts) +CHUNK_SIZES = [1, 2, 3, 5, 11, 23, None] + + def make_mock_tokenizer(sample: Sample) -> MockTokenizer: """Build a mock tokenizer from a sample's vocab and token data.""" return MockTokenizer( diff --git a/tests/parser/engine/test_delegating_replay.py b/tests/parser/engine/test_delegating_replay.py index 7460ab21ec..5d5d6b3247 100644 --- a/tests/parser/engine/test_delegating_replay.py +++ b/tests/parser/engine/test_delegating_replay.py @@ -19,6 +19,7 @@ import pytest from pydantic import TypeAdapter from tests.parser.engine.replay_harness import ( + CHUNK_SIZES, MockTokenizer, assert_parse_output, collect_output, @@ -113,8 +114,6 @@ _PAIRINGS = _discover_pairings() _ALL_SAMPLES = [(p.parser_cls, s) for p in _PAIRINGS for s in p.samples] -CHUNK_SIZES = [1, 2, 3, 5, 11, 23, None] - @pytest.mark.parametrize("chunk_size", CHUNK_SIZES, ids=lambda c: f"chunk={c}") @pytest.mark.parametrize( diff --git a/tests/parser/engine/test_ufffd_reasoning_transition.py b/tests/parser/engine/test_ufffd_reasoning_transition.py new file mode 100644 index 0000000000..ffd2ead95d --- /dev/null +++ b/tests/parser/engine/test_ufffd_reasoning_transition.py @@ -0,0 +1,181 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Regression test for U+FFFD leak at reasoning→content transition. + +When byte-fallback tokens span the reasoning/content boundary, +decoding isolated content-side token IDs via tokenizer.decode() +produces U+FFFD (Unicode replacement character). The fix flushes +the reasoning parser's engine lexer instead. + +Reproduces the bug at various chunk sizes and validates that the +fix prevents U+FFFD from leaking into streamed content. +""" + +from __future__ import annotations + +import pytest + +from tests.parser.engine.replay_harness import ( + CHUNK_SIZES, + MockTokenizer, + collect_output, + replay_streaming, +) +from vllm.parser.abstract_parser import DelegatingParser +from vllm.parser.engine.registered_adapters import ( + Glm47MoeParserReasoningAdapter, + Glm47MoeParserToolAdapter, + Qwen3ParserReasoningAdapter, + Qwen3ParserToolAdapter, +) + + +class ByteFallbackMockTokenizer(MockTokenizer): + """MockTokenizer that returns U+FFFD for specified token IDs. + + Simulates byte-fallback tokenizer behavior where isolated + partial-byte tokens decode to the Unicode replacement character. + """ + + def __init__( + self, + vocab: dict[str, int], + tokens: list[tuple[int, str]], + ufffd_token_ids: set[int], + ) -> None: + super().__init__(vocab, tokens) + self._ufffd_token_ids = frozenset(ufffd_token_ids) + + def decode(self, ids: list[int], skip_special_tokens: bool = False) -> str: + parts: list[str] = [] + for tid in ids: + if skip_special_tokens and tid in self._special_ids: + continue + if tid in self._ufffd_token_ids: + parts.append("�") + else: + text = self._token_decode_map.get(tid, f"?{tid}?") + parts.append(text) + return "".join(parts) + + +# ── Model-specific DelegatingParser subclasses ─────────────────────── + + +class _Glm47Delegating(DelegatingParser): + reasoning_parser_cls = Glm47MoeParserReasoningAdapter + tool_parser_cls = Glm47MoeParserToolAdapter + + +class _Qwen3Delegating(DelegatingParser): + reasoning_parser_cls = Qwen3ParserReasoningAdapter + tool_parser_cls = Qwen3ParserToolAdapter + + +# ── Shared test data ───────────────────────────────────────────────── + +_SHARED_TOKENS: list[tuple[int, str]] = [ + (100, "Let me"), + (101, " think"), + (102, " about"), + (103, " Samsung."), + (51, ""), + (200, "삼성"), + (201, "전자의"), + (202, " 주가를"), + (203, " 분석합니다."), +] + +_SHARED_UFFFD_IDS: set[int] = {200} + +EXPECTED_REASONING = "Let me think about Samsung." +EXPECTED_CONTENT = "삼성전자의 주가를 분석합니다." + +_MODEL_CONFIGS = [ + pytest.param( + { + "": 50, + "": 51, + "": 60, + "": 61, + "": 62, + "": 63, + "": 64, + "": 65, + }, + _Glm47Delegating, + id="glm47", + ), + pytest.param( + { + "": 50, + "": 51, + "": 60, + "": 61, + }, + _Qwen3Delegating, + id="qwen3", + ), +] + + +# ── Tests ──────────────────────────────────────────────────────────── + + +class TestUfffdReasoningTransition: + """U+FFFD must not appear at the reasoning→content transition.""" + + @pytest.mark.parametrize("vocab,delegating_cls", _MODEL_CONFIGS) + @pytest.mark.parametrize("chunk_size", CHUNK_SIZES, ids=lambda c: f"chunk={c}") + def test_no_ufffd(self, chunk_size, vocab, delegating_cls): + tokenizer = ByteFallbackMockTokenizer(vocab, _SHARED_TOKENS, _SHARED_UFFFD_IDS) + parser = delegating_cls(tokenizer) + deltas = replay_streaming( + parser, + _SHARED_TOKENS, + chunk_size=chunk_size, + finished_on_last=True, + ) + output = collect_output(deltas) + + assert "�" not in output.content, ( + f"U+FFFD leaked into content: {output.content!r}" + ) + assert output.content == EXPECTED_CONTENT + assert output.reasoning == EXPECTED_REASONING + + def test_byte_fallback_tokenizer_produces_ufffd(self): + """Validate the fixture: decode() returns U+FFFD for isolated + byte-fallback token IDs, proving the old code path would leak.""" + vocab = dict(_MODEL_CONFIGS[0].values[0]) + tokenizer = ByteFallbackMockTokenizer(vocab, _SHARED_TOKENS, _SHARED_UFFFD_IDS) + assert tokenizer.decode([200]) == "�" + + @pytest.mark.parametrize("chunk_size", CHUNK_SIZES, ids=lambda c: f"chunk={c}") + def test_multiple_ufffd_tokens_at_boundary(self, chunk_size): + """Multiple consecutive byte-fallback tokens at the boundary.""" + tokens: list[tuple[int, str]] = [ + (100, "Reasoning."), + (51, ""), + (200, "삼"), + (201, "성"), + (202, "전자"), + ] + ufffd_ids: set[int] = {200, 201} + vocab = dict(_MODEL_CONFIGS[0].values[0]) + + tokenizer = ByteFallbackMockTokenizer(vocab, tokens, ufffd_ids) + parser = _Glm47Delegating(tokenizer) + deltas = replay_streaming( + parser, + tokens, + chunk_size=chunk_size, + finished_on_last=True, + ) + output = collect_output(deltas) + + assert "�" not in output.content, ( + f"U+FFFD leaked into content: {output.content!r}" + ) + assert output.content == "삼성전자" + assert output.reasoning == "Reasoning." diff --git a/vllm/parser/abstract_parser.py b/vllm/parser/abstract_parser.py index 915d401f7b..11fca8e43a 100644 --- a/vllm/parser/abstract_parser.py +++ b/vllm/parser/abstract_parser.py @@ -794,11 +794,10 @@ class DelegatingParser(Parser): reasoning_transitioned = True current_token_ids = self.extract_content_ids(delta_token_ids) if self._engine_based: + flush_delta = reasoning_parser.finish_streaming() # type: ignore[union-attr, attr-defined] current_text = ( - self.model_tokenizer.decode(current_token_ids) - if current_token_ids - else "" - ) + (delta_message.content if delta_message else None) or "" + ) + ((flush_delta.content if flush_delta else None) or "") if delta_message and self._tool_parser is not None: delta_message.content = None else: diff --git a/vllm/parser/engine/parser_engine.py b/vllm/parser/engine/parser_engine.py index dafb26fc48..ba838d31a0 100644 --- a/vllm/parser/engine/parser_engine.py +++ b/vllm/parser/engine/parser_engine.py @@ -172,7 +172,9 @@ class ParserEngine(Parser): def finish_streaming(self) -> DeltaMessage | None: events = self._engine.finish() - return self._events_to_delta(events) if events else None + if events or self._deferred_content: + return self._events_to_delta(events, finished=True) + return None def _reset(self, initial_state: ParserState | None = None) -> None: self._engine.reset(initial_state=initial_state)