[Misc] Some minor simplifications to detokenization logic (#3670)
Some simplifications made for clarity. Also moves detokenization-related functions from tokenizer.py to detokenizer.py.
This commit is contained in:
@@ -4,8 +4,8 @@ import pytest
|
||||
from transformers import AutoTokenizer
|
||||
|
||||
from vllm.sequence import Logprob, SamplingParams, Sequence, SequenceGroup
|
||||
from vllm.transformers_utils.detokenizer import Detokenizer
|
||||
from vllm.transformers_utils.tokenizer import detokenize_incrementally
|
||||
from vllm.transformers_utils.detokenizer import (Detokenizer,
|
||||
detokenize_incrementally)
|
||||
from vllm.transformers_utils.tokenizer_group import get_tokenizer_group
|
||||
|
||||
TRUTH = [
|
||||
|
||||
Reference in New Issue
Block a user