diff --git a/tests/parser/engine/replay_harness.py b/tests/parser/engine/replay_harness.py
index fac643390b..9abd460f76 100644
--- a/tests/parser/engine/replay_harness.py
+++ b/tests/parser/engine/replay_harness.py
@@ -96,6 +96,9 @@ class MockTokenizer:
return "".join(parts)
+CHUNK_SIZES = [1, 2, 3, 5, 11, 23, None]
+
+
def make_mock_tokenizer(sample: Sample) -> MockTokenizer:
"""Build a mock tokenizer from a sample's vocab and token data."""
return MockTokenizer(
diff --git a/tests/parser/engine/test_delegating_replay.py b/tests/parser/engine/test_delegating_replay.py
index 7460ab21ec..5d5d6b3247 100644
--- a/tests/parser/engine/test_delegating_replay.py
+++ b/tests/parser/engine/test_delegating_replay.py
@@ -19,6 +19,7 @@ import pytest
from pydantic import TypeAdapter
from tests.parser.engine.replay_harness import (
+ CHUNK_SIZES,
MockTokenizer,
assert_parse_output,
collect_output,
@@ -113,8 +114,6 @@ _PAIRINGS = _discover_pairings()
_ALL_SAMPLES = [(p.parser_cls, s) for p in _PAIRINGS for s in p.samples]
-CHUNK_SIZES = [1, 2, 3, 5, 11, 23, None]
-
@pytest.mark.parametrize("chunk_size", CHUNK_SIZES, ids=lambda c: f"chunk={c}")
@pytest.mark.parametrize(
diff --git a/tests/parser/engine/test_ufffd_reasoning_transition.py b/tests/parser/engine/test_ufffd_reasoning_transition.py
new file mode 100644
index 0000000000..ffd2ead95d
--- /dev/null
+++ b/tests/parser/engine/test_ufffd_reasoning_transition.py
@@ -0,0 +1,181 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+"""Regression test for U+FFFD leak at reasoning→content transition.
+
+When byte-fallback tokens span the reasoning/content boundary,
+decoding isolated content-side token IDs via tokenizer.decode()
+produces U+FFFD (Unicode replacement character). The fix flushes
+the reasoning parser's engine lexer instead.
+
+Reproduces the bug at various chunk sizes and validates that the
+fix prevents U+FFFD from leaking into streamed content.
+"""
+
+from __future__ import annotations
+
+import pytest
+
+from tests.parser.engine.replay_harness import (
+ CHUNK_SIZES,
+ MockTokenizer,
+ collect_output,
+ replay_streaming,
+)
+from vllm.parser.abstract_parser import DelegatingParser
+from vllm.parser.engine.registered_adapters import (
+ Glm47MoeParserReasoningAdapter,
+ Glm47MoeParserToolAdapter,
+ Qwen3ParserReasoningAdapter,
+ Qwen3ParserToolAdapter,
+)
+
+
+class ByteFallbackMockTokenizer(MockTokenizer):
+ """MockTokenizer that returns U+FFFD for specified token IDs.
+
+ Simulates byte-fallback tokenizer behavior where isolated
+ partial-byte tokens decode to the Unicode replacement character.
+ """
+
+ def __init__(
+ self,
+ vocab: dict[str, int],
+ tokens: list[tuple[int, str]],
+ ufffd_token_ids: set[int],
+ ) -> None:
+ super().__init__(vocab, tokens)
+ self._ufffd_token_ids = frozenset(ufffd_token_ids)
+
+ def decode(self, ids: list[int], skip_special_tokens: bool = False) -> str:
+ parts: list[str] = []
+ for tid in ids:
+ if skip_special_tokens and tid in self._special_ids:
+ continue
+ if tid in self._ufffd_token_ids:
+ parts.append("�")
+ else:
+ text = self._token_decode_map.get(tid, f"?{tid}?")
+ parts.append(text)
+ return "".join(parts)
+
+
+# ── Model-specific DelegatingParser subclasses ───────────────────────
+
+
+class _Glm47Delegating(DelegatingParser):
+ reasoning_parser_cls = Glm47MoeParserReasoningAdapter
+ tool_parser_cls = Glm47MoeParserToolAdapter
+
+
+class _Qwen3Delegating(DelegatingParser):
+ reasoning_parser_cls = Qwen3ParserReasoningAdapter
+ tool_parser_cls = Qwen3ParserToolAdapter
+
+
+# ── Shared test data ─────────────────────────────────────────────────
+
+_SHARED_TOKENS: list[tuple[int, str]] = [
+ (100, "Let me"),
+ (101, " think"),
+ (102, " about"),
+ (103, " Samsung."),
+ (51, ""),
+ (200, "삼성"),
+ (201, "전자의"),
+ (202, " 주가를"),
+ (203, " 분석합니다."),
+]
+
+_SHARED_UFFFD_IDS: set[int] = {200}
+
+EXPECTED_REASONING = "Let me think about Samsung."
+EXPECTED_CONTENT = "삼성전자의 주가를 분석합니다."
+
+_MODEL_CONFIGS = [
+ pytest.param(
+ {
+ "": 50,
+ "": 51,
+ "": 60,
+ "": 61,
+ "": 62,
+ "": 63,
+ "": 64,
+ "": 65,
+ },
+ _Glm47Delegating,
+ id="glm47",
+ ),
+ pytest.param(
+ {
+ "": 50,
+ "": 51,
+ "": 60,
+ "": 61,
+ },
+ _Qwen3Delegating,
+ id="qwen3",
+ ),
+]
+
+
+# ── Tests ────────────────────────────────────────────────────────────
+
+
+class TestUfffdReasoningTransition:
+ """U+FFFD must not appear at the reasoning→content transition."""
+
+ @pytest.mark.parametrize("vocab,delegating_cls", _MODEL_CONFIGS)
+ @pytest.mark.parametrize("chunk_size", CHUNK_SIZES, ids=lambda c: f"chunk={c}")
+ def test_no_ufffd(self, chunk_size, vocab, delegating_cls):
+ tokenizer = ByteFallbackMockTokenizer(vocab, _SHARED_TOKENS, _SHARED_UFFFD_IDS)
+ parser = delegating_cls(tokenizer)
+ deltas = replay_streaming(
+ parser,
+ _SHARED_TOKENS,
+ chunk_size=chunk_size,
+ finished_on_last=True,
+ )
+ output = collect_output(deltas)
+
+ assert "�" not in output.content, (
+ f"U+FFFD leaked into content: {output.content!r}"
+ )
+ assert output.content == EXPECTED_CONTENT
+ assert output.reasoning == EXPECTED_REASONING
+
+ def test_byte_fallback_tokenizer_produces_ufffd(self):
+ """Validate the fixture: decode() returns U+FFFD for isolated
+ byte-fallback token IDs, proving the old code path would leak."""
+ vocab = dict(_MODEL_CONFIGS[0].values[0])
+ tokenizer = ByteFallbackMockTokenizer(vocab, _SHARED_TOKENS, _SHARED_UFFFD_IDS)
+ assert tokenizer.decode([200]) == "�"
+
+ @pytest.mark.parametrize("chunk_size", CHUNK_SIZES, ids=lambda c: f"chunk={c}")
+ def test_multiple_ufffd_tokens_at_boundary(self, chunk_size):
+ """Multiple consecutive byte-fallback tokens at the boundary."""
+ tokens: list[tuple[int, str]] = [
+ (100, "Reasoning."),
+ (51, ""),
+ (200, "삼"),
+ (201, "성"),
+ (202, "전자"),
+ ]
+ ufffd_ids: set[int] = {200, 201}
+ vocab = dict(_MODEL_CONFIGS[0].values[0])
+
+ tokenizer = ByteFallbackMockTokenizer(vocab, tokens, ufffd_ids)
+ parser = _Glm47Delegating(tokenizer)
+ deltas = replay_streaming(
+ parser,
+ tokens,
+ chunk_size=chunk_size,
+ finished_on_last=True,
+ )
+ output = collect_output(deltas)
+
+ assert "�" not in output.content, (
+ f"U+FFFD leaked into content: {output.content!r}"
+ )
+ assert output.content == "삼성전자"
+ assert output.reasoning == "Reasoning."
diff --git a/vllm/parser/abstract_parser.py b/vllm/parser/abstract_parser.py
index 915d401f7b..11fca8e43a 100644
--- a/vllm/parser/abstract_parser.py
+++ b/vllm/parser/abstract_parser.py
@@ -794,11 +794,10 @@ class DelegatingParser(Parser):
reasoning_transitioned = True
current_token_ids = self.extract_content_ids(delta_token_ids)
if self._engine_based:
+ flush_delta = reasoning_parser.finish_streaming() # type: ignore[union-attr, attr-defined]
current_text = (
- self.model_tokenizer.decode(current_token_ids)
- if current_token_ids
- else ""
- )
+ (delta_message.content if delta_message else None) or ""
+ ) + ((flush_delta.content if flush_delta else None) or "")
if delta_message and self._tool_parser is not None:
delta_message.content = None
else:
diff --git a/vllm/parser/engine/parser_engine.py b/vllm/parser/engine/parser_engine.py
index dafb26fc48..ba838d31a0 100644
--- a/vllm/parser/engine/parser_engine.py
+++ b/vllm/parser/engine/parser_engine.py
@@ -172,7 +172,9 @@ class ParserEngine(Parser):
def finish_streaming(self) -> DeltaMessage | None:
events = self._engine.finish()
- return self._events_to_delta(events) if events else None
+ if events or self._deferred_content:
+ return self._events_to_delta(events, finished=True)
+ return None
def _reset(self, initial_state: ParserState | None = None) -> None:
self._engine.reset(initial_state=initial_state)