[Bugfix][Frontend] Seed derender detokenization from the prompt (#59046)
Signed-off-by: Martin Hickey <martin.hickey@ie.ibm.com>
This commit is contained in:
@@ -63,7 +63,7 @@ _CACHED_MARKER_KEY = "_vllm_space_marker_cache"
|
||||
_NOT_CACHED = "__not_computed__"
|
||||
|
||||
|
||||
def _get_leading_space_marker(tokenizer: TokenizerLike) -> str | None:
|
||||
def get_leading_space_marker(tokenizer: TokenizerLike) -> str | None:
|
||||
"""Read the space marker from the tokenizer's pre_tokenizer config.
|
||||
|
||||
Only Metaspace pre_tokenizers (used by SentencePiece-based models like
|
||||
@@ -160,7 +160,7 @@ def convert_ids_list_to_tokens(
|
||||
"""
|
||||
if not token_ids:
|
||||
return []
|
||||
marker = _get_leading_space_marker(tokenizer)
|
||||
marker = get_leading_space_marker(tokenizer)
|
||||
if marker is None:
|
||||
return [tokenizer.decode([tid]) or "" for tid in token_ids]
|
||||
raw_tokens = tokenizer.convert_ids_to_tokens(token_ids)
|
||||
|
||||
Reference in New Issue
Block a user