[BugFix] Fix tokenizer out of vocab size (#3685)
This commit is contained in:
@@ -83,6 +83,13 @@ def test_decode_streaming(tokenizer_id, truth, with_prompt,
|
||||
|
||||
assert decoded_text == generated
|
||||
|
||||
decoded_text = _run_incremental_decode(
|
||||
tokenizer, [len(tokenizer)],
|
||||
skip_special_tokens=skip_special_tokens,
|
||||
starting_index=starting_index)
|
||||
|
||||
assert decoded_text == ''
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def detokenizer(tokenizer_name: str) -> Detokenizer:
|
||||
|
||||
Reference in New Issue
Block a user