From 8d10c38b7304aa42432f4117f0d197ff46258ecb Mon Sep 17 00:00:00 2001
From: Kamran <79936503+spa5k@users.noreply.github.com>
Date: Fri, 9 Oct 2026 03:12:56 +0530
Subject: [PATCH] [Perf] Reduce fallback decode CPU by 69% at 32K (#60497)

Signed-off-by: spa5k <79936503+spa5k@users.noreply.github.com>
Co-authored-by: Codex <codex@openai.com>
---
 tests/tokenizers_/test_detokenize.py | 48 +++++++++++++++++++++++++++-
 vllm/tokenizers/detokenizer_utils.py | 20 ++++++++----
 2 files changed, 61 insertions(+), 7 deletions(-)

diff --git a/tests/tokenizers_/test_detokenize.py b/tests/tokenizers_/test_detokenize.py
index d0117e4711..1dd2a000a6 100644
--- a/tests/tokenizers_/test_detokenize.py
+++ b/tests/tokenizers_/test_detokenize.py
@@ -8,7 +8,11 @@ import pytest
 from transformers import AutoTokenizer, PythonBackend, TokenizersBackend
 
 from vllm.sampling_params import SamplingParams
-from vllm.tokenizers.detokenizer_utils import convert_ids_list_to_tokens
+from vllm.tokenizers.detokenizer_utils import (
+    convert_ids_list_to_tokens,
+    convert_prompt_ids_to_tokens,
+    detokenize_incrementally,
+)
 from vllm.tokenizers.mistral import MistralTokenizer
 from vllm.tokenizers.protocol import TokenizerLike
 from vllm.v1.engine import EngineCoreRequest
@@ -98,6 +102,48 @@ def tokenizer(tokenizer_name):
     )
 
 
+@pytest.mark.parametrize(
+    "tokenizer_name", ["openai-community/gpt2", "mistralai/Pixtral-12B-2409"]
+)
+def test_slow_decode_long_history_preserves_text_and_token_ids(tokenizer):
+    truth = "A map.  我很感谢你的热情 🙂🌶️\n" * 32
+    ids = tokenizer.encode(truth, add_special_tokens=False)
+    text, output_ids = _run_incremental_decode(
+        tokenizer, ids, skip_special_tokens=True, starting_index=0, fast=False
+    )
+    assert text == tokenizer.decode(ids, skip_special_tokens=True)
+    assert output_ids == ids
+
+
+@pytest.mark.parametrize(
+    "tokenizer_name", ["openai-community/gpt2", "mistralai/Pixtral-12B-2409"]
+)
+def test_first_incremental_decode_returns_the_prompt_window(tokenizer):
+    ids = tokenizer.encode("A map in a library. " * 8, add_special_tokens=False)
+    prompt_tokens, _, _ = convert_prompt_ids_to_tokens(
+        tokenizer, ids[:-1], skip_special_tokens=True
+    )
+    new_token = tokenizer.convert_ids_to_tokens([ids[-1]], skip_special_tokens=True)
+    tokens, _, _, _ = detokenize_incrementally(
+        tokenizer, ids, None, 0, 0, skip_special_tokens=True
+    )
+    assert tokens == prompt_tokens + new_token
+
+
+@pytest.mark.parametrize(
+    "tokenizer_name", ["openai-community/gpt2", "mistralai/Pixtral-12B-2409"]
+)
+def test_incremental_decode_past_the_window_emits_no_text(tokenizer):
+    ids = tokenizer.encode("A map in a library.", add_special_tokens=False)
+    tokens = tokenizer.convert_ids_to_tokens(ids[:-1], skip_special_tokens=True)
+    offset = len(tokens) + 1
+    _, text, prefix_offset, read_offset = detokenize_incrementally(
+        tokenizer, ids, tokens, offset, offset, skip_special_tokens=True
+    )
+    assert text == ""
+    assert (prefix_offset, read_offset) == (offset, offset)
+
+
 @pytest.mark.parametrize("tokenizer_name", ["mistralai/Pixtral-12B-2409"])
 @pytest.mark.parametrize(
     "truth",
diff --git a/vllm/tokenizers/detokenizer_utils.py b/vllm/tokenizers/detokenizer_utils.py
index ddaf512442..fc2ce4d44c 100644
--- a/vllm/tokenizers/detokenizer_utils.py
+++ b/vllm/tokenizers/detokenizer_utils.py
@@ -230,7 +230,13 @@ def detokenize_incrementally(
             _replace_none_with_empty(new_tokens)  # type: ignore[arg-type]
     else:
         new_tokens = [""]
-    output_tokens = prev_tokens + new_tokens
+    output_token_count = len(prev_tokens) + len(new_tokens)
+    window_start = (
+        0 if is_first_iter else min(prefix_offset, read_offset, len(prev_tokens))
+    )
+    output_tokens = (
+        prev_tokens[window_start:] if window_start else prev_tokens
+    ) + new_tokens
 
     # If this is the first iteration, return all tokens.
     if is_first_iter:
@@ -241,19 +247,21 @@ def detokenize_incrementally(
     # surrounding ids.
     if tokenizer.is_fast or not tokenizer.get_added_vocab():
         prefix_text = tokenizer.convert_tokens_to_string(
-            output_tokens[prefix_offset:read_offset]
+            output_tokens[prefix_offset - window_start : read_offset - window_start]
+        )
+        new_text = tokenizer.convert_tokens_to_string(
+            output_tokens[prefix_offset - window_start :]
         )
-        new_text = tokenizer.convert_tokens_to_string(output_tokens[prefix_offset:])
     else:
         prefix_text = _convert_tokens_to_string_with_added_encoders(
             tokenizer,
-            output_tokens[prefix_offset:read_offset],
+            output_tokens[prefix_offset - window_start : read_offset - window_start],
             skip_special_tokens=skip_special_tokens,
             spaces_between_special_tokens=spaces_between_special_tokens,
         )
         new_text = _convert_tokens_to_string_with_added_encoders(
             tokenizer,
-            output_tokens[prefix_offset:],
+            output_tokens[prefix_offset - window_start :],
             skip_special_tokens=skip_special_tokens,
             spaces_between_special_tokens=spaces_between_special_tokens,
         )
@@ -266,4 +274,4 @@ def detokenize_incrementally(
         return new_tokens, "", prefix_offset, read_offset
 
     new_text = new_text[len(prefix_text) :]
-    return new_tokens, new_text, read_offset, len(output_tokens)
+    return new_tokens, new_text, read_offset, output_token_count