Bump Transformers version to 5.10.4 (#41359)
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
This commit is contained in:
@@ -5,7 +5,7 @@ from typing import _get_protocol_attrs # type: ignore
|
||||
import pytest
|
||||
from transformers import (
|
||||
PreTrainedTokenizerBase,
|
||||
PreTrainedTokenizerFast,
|
||||
TokenizersBackend,
|
||||
)
|
||||
|
||||
from vllm.tokenizers import TokenizerLike, get_tokenizer
|
||||
@@ -24,7 +24,7 @@ def _assert_tokenizer_like(tokenizer: object):
|
||||
|
||||
def test_tokenizer_like_protocol():
|
||||
tokenizer = get_tokenizer("openai-community/gpt2", use_fast=True)
|
||||
assert isinstance(tokenizer, PreTrainedTokenizerFast)
|
||||
assert isinstance(tokenizer, TokenizersBackend)
|
||||
_assert_tokenizer_like(tokenizer)
|
||||
|
||||
tokenizer = get_tokenizer(
|
||||
@@ -38,7 +38,7 @@ def test_tokenizer_like_protocol():
|
||||
assert isinstance(tokenizer, HfTokenizer)
|
||||
|
||||
# Verify it's a fast tokenizer (required for FastIncrementalDetokenizer)
|
||||
assert isinstance(tokenizer, PreTrainedTokenizerFast)
|
||||
assert isinstance(tokenizer, TokenizersBackend)
|
||||
assert "DSV32" in tokenizer.__class__.__name__
|
||||
_assert_tokenizer_like(tokenizer)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user