[Perf] Reduce fallback decode CPU by 69% at 32K (#60497)
Signed-off-by: spa5k <79936503+spa5k@users.noreply.github.com> Co-authored-by: Codex <codex@openai.com>
This commit is contained in:
@@ -8,7 +8,11 @@ import pytest
|
||||
from transformers import AutoTokenizer, PythonBackend, TokenizersBackend
|
||||
|
||||
from vllm.sampling_params import SamplingParams
|
||||
from vllm.tokenizers.detokenizer_utils import convert_ids_list_to_tokens
|
||||
from vllm.tokenizers.detokenizer_utils import (
|
||||
convert_ids_list_to_tokens,
|
||||
convert_prompt_ids_to_tokens,
|
||||
detokenize_incrementally,
|
||||
)
|
||||
from vllm.tokenizers.mistral import MistralTokenizer
|
||||
from vllm.tokenizers.protocol import TokenizerLike
|
||||
from vllm.v1.engine import EngineCoreRequest
|
||||
@@ -98,6 +102,48 @@ def tokenizer(tokenizer_name):
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"tokenizer_name", ["openai-community/gpt2", "mistralai/Pixtral-12B-2409"]
|
||||
)
|
||||
def test_slow_decode_long_history_preserves_text_and_token_ids(tokenizer):
|
||||
truth = "A map. 我很感谢你的热情 🙂🌶️\n" * 32
|
||||
ids = tokenizer.encode(truth, add_special_tokens=False)
|
||||
text, output_ids = _run_incremental_decode(
|
||||
tokenizer, ids, skip_special_tokens=True, starting_index=0, fast=False
|
||||
)
|
||||
assert text == tokenizer.decode(ids, skip_special_tokens=True)
|
||||
assert output_ids == ids
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"tokenizer_name", ["openai-community/gpt2", "mistralai/Pixtral-12B-2409"]
|
||||
)
|
||||
def test_first_incremental_decode_returns_the_prompt_window(tokenizer):
|
||||
ids = tokenizer.encode("A map in a library. " * 8, add_special_tokens=False)
|
||||
prompt_tokens, _, _ = convert_prompt_ids_to_tokens(
|
||||
tokenizer, ids[:-1], skip_special_tokens=True
|
||||
)
|
||||
new_token = tokenizer.convert_ids_to_tokens([ids[-1]], skip_special_tokens=True)
|
||||
tokens, _, _, _ = detokenize_incrementally(
|
||||
tokenizer, ids, None, 0, 0, skip_special_tokens=True
|
||||
)
|
||||
assert tokens == prompt_tokens + new_token
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"tokenizer_name", ["openai-community/gpt2", "mistralai/Pixtral-12B-2409"]
|
||||
)
|
||||
def test_incremental_decode_past_the_window_emits_no_text(tokenizer):
|
||||
ids = tokenizer.encode("A map in a library.", add_special_tokens=False)
|
||||
tokens = tokenizer.convert_ids_to_tokens(ids[:-1], skip_special_tokens=True)
|
||||
offset = len(tokens) + 1
|
||||
_, text, prefix_offset, read_offset = detokenize_incrementally(
|
||||
tokenizer, ids, tokens, offset, offset, skip_special_tokens=True
|
||||
)
|
||||
assert text == ""
|
||||
assert (prefix_offset, read_offset) == (offset, offset)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("tokenizer_name", ["mistralai/Pixtral-12B-2409"])
|
||||
@pytest.mark.parametrize(
|
||||
"truth",
|
||||
|
||||
Reference in New Issue
Block a user