[Misc] Refactor tokenizer interface (#29693)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
This commit is contained in:
@@ -10,7 +10,7 @@ from transformers import PreTrainedTokenizerFast
|
||||
|
||||
from vllm.logger import init_logger
|
||||
from vllm.transformers_utils.detokenizer_utils import (
|
||||
AnyTokenizer,
|
||||
TokenizerLike,
|
||||
convert_prompt_ids_to_tokens,
|
||||
detokenize_incrementally,
|
||||
)
|
||||
@@ -45,7 +45,7 @@ class IncrementalDetokenizer:
|
||||
@classmethod
|
||||
def from_new_request(
|
||||
cls,
|
||||
tokenizer: AnyTokenizer | None,
|
||||
tokenizer: TokenizerLike | None,
|
||||
request: EngineCoreRequest,
|
||||
) -> "IncrementalDetokenizer":
|
||||
assert request.sampling_params is not None
|
||||
@@ -256,7 +256,7 @@ class FastIncrementalDetokenizer(BaseIncrementalDetokenizer):
|
||||
|
||||
|
||||
class SlowIncrementalDetokenizer(BaseIncrementalDetokenizer):
|
||||
def __init__(self, tokenizer: AnyTokenizer, request: EngineCoreRequest):
|
||||
def __init__(self, tokenizer: TokenizerLike, request: EngineCoreRequest):
|
||||
super().__init__(request)
|
||||
|
||||
self.tokenizer = tokenizer
|
||||
|
||||
Reference in New Issue
Block a user