Truncation control for embedding models (#14776)
Signed-off-by: Gabriel Marinho <gmarinho@ibm.com> Signed-off-by: Max de Bayser <mbayser@br.ibm.com> Co-authored-by: Max de Bayser <mbayser@br.ibm.com>
This commit is contained in:
co-authored by
Max de Bayser
parent
4055130a85
commit
1c2bc7ead0
@@ -55,6 +55,8 @@ def encode_tokens(
|
||||
tokenizer: AnyTokenizer,
|
||||
text: str,
|
||||
*,
|
||||
truncation: Optional[bool] = None,
|
||||
max_length: Optional[int] = None,
|
||||
add_special_tokens: Optional[bool] = None,
|
||||
) -> list[int]:
|
||||
"""
|
||||
@@ -64,10 +66,18 @@ def encode_tokens(
|
||||
:code:`add_special_tokens=None` means to use the backend's default
|
||||
settings.
|
||||
"""
|
||||
if add_special_tokens is not None:
|
||||
return tokenizer.encode(text, add_special_tokens=add_special_tokens)
|
||||
|
||||
return tokenizer.encode(text)
|
||||
kw_args: dict[str, Any] = {}
|
||||
if max_length is not None:
|
||||
kw_args["max_length"] = max_length
|
||||
|
||||
if truncation is not None:
|
||||
kw_args["truncation"] = truncation
|
||||
|
||||
if add_special_tokens is not None:
|
||||
kw_args["add_special_tokens"] = add_special_tokens
|
||||
|
||||
return tokenizer.encode(text, **kw_args)
|
||||
|
||||
|
||||
def get_cached_tokenizer(tokenizer: AnyTokenizer) -> AnyTokenizer:
|
||||
|
||||
@@ -94,6 +94,8 @@ class TokenizerBase(ABC):
|
||||
@abstractmethod
|
||||
def encode(self,
|
||||
text: str,
|
||||
truncation: Optional[bool] = None,
|
||||
max_length: Optional[int] = None,
|
||||
add_special_tokens: Optional[bool] = None) -> list[int]:
|
||||
raise NotImplementedError()
|
||||
|
||||
|
||||
@@ -45,11 +45,16 @@ class TokenizerGroup:
|
||||
|
||||
def encode(self,
|
||||
prompt: str,
|
||||
max_length: Optional[int] = None,
|
||||
truncation: Optional[bool] = None,
|
||||
lora_request: Optional[LoRARequest] = None,
|
||||
add_special_tokens: Optional[bool] = None) -> List[int]:
|
||||
|
||||
tokenizer = self.get_lora_tokenizer(lora_request)
|
||||
ret = encode_tokens(tokenizer,
|
||||
prompt,
|
||||
max_length=max_length,
|
||||
truncation=truncation,
|
||||
add_special_tokens=add_special_tokens)
|
||||
self._raise_if_input_too_long(ret, lora_request)
|
||||
return ret
|
||||
@@ -57,11 +62,15 @@ class TokenizerGroup:
|
||||
async def encode_async(
|
||||
self,
|
||||
prompt: str,
|
||||
max_length: Optional[int] = None,
|
||||
truncation: Optional[bool] = None,
|
||||
lora_request: Optional[LoRARequest] = None,
|
||||
add_special_tokens: Optional[bool] = None) -> List[int]:
|
||||
tokenizer = await self.get_lora_tokenizer_async(lora_request)
|
||||
ret = encode_tokens(tokenizer,
|
||||
prompt,
|
||||
max_length=max_length,
|
||||
truncation=truncation,
|
||||
add_special_tokens=add_special_tokens)
|
||||
self._raise_if_input_too_long(ret, lora_request)
|
||||
return ret
|
||||
|
||||
@@ -359,6 +359,8 @@ class MistralTokenizer(TokenizerBase):
|
||||
|
||||
def encode(self,
|
||||
text: str,
|
||||
truncation: Optional[bool] = None,
|
||||
max_length: Optional[int] = None,
|
||||
add_special_tokens: Optional[bool] = None) -> List[int]:
|
||||
# `encode` should only be used for prompt completion
|
||||
# it should never be used for chat_completion.
|
||||
|
||||
Reference in New Issue
Block a user