[Chore] Use tokenizer.encode and tokenizer.decode directly (#29851)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
This commit is contained in:
@@ -37,7 +37,7 @@ from vllm.inputs.data import PromptType
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.models import SupportsTranscription
|
||||
from vllm.outputs import RequestOutput
|
||||
from vllm.transformers_utils.tokenizer import get_tokenizer
|
||||
from vllm.tokenizers import get_tokenizer
|
||||
from vllm.utils.import_utils import PlaceholderModule
|
||||
|
||||
try:
|
||||
|
||||
@@ -33,7 +33,7 @@ class RenderConfig:
|
||||
`0` yields an empty list (and skips embeds).
|
||||
`-1` maps to `model_config.max_model_len`."""
|
||||
|
||||
add_special_tokens: bool | None = True
|
||||
add_special_tokens: bool = True
|
||||
"""Whether to add model-specific special tokens during tokenization."""
|
||||
|
||||
cache_salt: str | None = None
|
||||
@@ -315,7 +315,7 @@ class CompletionRenderer(BaseRenderer):
|
||||
text: str,
|
||||
max_length: int | None,
|
||||
truncate_prompt_tokens: int | None,
|
||||
add_special_tokens: bool | None,
|
||||
add_special_tokens: bool,
|
||||
cache_salt: str | None,
|
||||
) -> EngineTokensPrompt:
|
||||
"""Tokenize text input asynchronously."""
|
||||
|
||||
@@ -19,7 +19,7 @@ from vllm.inputs import TokensPrompt
|
||||
from vllm.model_executor.models.interfaces import supports_score_template
|
||||
from vllm.multimodal.inputs import MultiModalDataDict
|
||||
from vllm.outputs import PoolingRequestOutput
|
||||
from vllm.transformers_utils.tokenizer import TokenizerLike
|
||||
from vllm.tokenizers import TokenizerLike
|
||||
|
||||
ScoreContentPartParam: TypeAlias = (
|
||||
ChatCompletionContentPartImageParam | ChatCompletionContentPartImageEmbedsParam
|
||||
|
||||
Reference in New Issue
Block a user