[Perf] Reduce fallback decode CPU by 69% at 32K (#60497)

Signed-off-by: spa5k <79936503+spa5k@users.noreply.github.com>
Co-authored-by: Codex <codex@openai.com>
This commit is contained in:
Kamran
2026-10-08 21:42:56 +00:00
committed by GitHub
co-authored by Codex
parent 240785b82c
commit 8d10c38b73
2 changed files with 61 additions and 7 deletions
+47 -1
View File
@@ -8,7 +8,11 @@ import pytest
from transformers import AutoTokenizer, PythonBackend, TokenizersBackend
from vllm.sampling_params import SamplingParams
from vllm.tokenizers.detokenizer_utils import convert_ids_list_to_tokens
from vllm.tokenizers.detokenizer_utils import (
convert_ids_list_to_tokens,
convert_prompt_ids_to_tokens,
detokenize_incrementally,
)
from vllm.tokenizers.mistral import MistralTokenizer
from vllm.tokenizers.protocol import TokenizerLike
from vllm.v1.engine import EngineCoreRequest
@@ -98,6 +102,48 @@ def tokenizer(tokenizer_name):
)
@pytest.mark.parametrize(
"tokenizer_name", ["openai-community/gpt2", "mistralai/Pixtral-12B-2409"]
)
def test_slow_decode_long_history_preserves_text_and_token_ids(tokenizer):
truth = "A map. 我很感谢你的热情 🙂🌶️\n" * 32
ids = tokenizer.encode(truth, add_special_tokens=False)
text, output_ids = _run_incremental_decode(
tokenizer, ids, skip_special_tokens=True, starting_index=0, fast=False
)
assert text == tokenizer.decode(ids, skip_special_tokens=True)
assert output_ids == ids
@pytest.mark.parametrize(
"tokenizer_name", ["openai-community/gpt2", "mistralai/Pixtral-12B-2409"]
)
def test_first_incremental_decode_returns_the_prompt_window(tokenizer):
ids = tokenizer.encode("A map in a library. " * 8, add_special_tokens=False)
prompt_tokens, _, _ = convert_prompt_ids_to_tokens(
tokenizer, ids[:-1], skip_special_tokens=True
)
new_token = tokenizer.convert_ids_to_tokens([ids[-1]], skip_special_tokens=True)
tokens, _, _, _ = detokenize_incrementally(
tokenizer, ids, None, 0, 0, skip_special_tokens=True
)
assert tokens == prompt_tokens + new_token
@pytest.mark.parametrize(
"tokenizer_name", ["openai-community/gpt2", "mistralai/Pixtral-12B-2409"]
)
def test_incremental_decode_past_the_window_emits_no_text(tokenizer):
ids = tokenizer.encode("A map in a library.", add_special_tokens=False)
tokens = tokenizer.convert_ids_to_tokens(ids[:-1], skip_special_tokens=True)
offset = len(tokens) + 1
_, text, prefix_offset, read_offset = detokenize_incrementally(
tokenizer, ids, tokens, offset, offset, skip_special_tokens=True
)
assert text == ""
assert (prefix_offset, read_offset) == (offset, offset)
@pytest.mark.parametrize("tokenizer_name", ["mistralai/Pixtral-12B-2409"])
@pytest.mark.parametrize(
"truth",