[BugFix] Fix tokenizer out of vocab size (#3685)

This commit is contained in:
Roy
2024-03-29 08:18:59 -07:00
committed by GitHub
parent d8658c8cc1
commit 6110c39dc8
4 changed files with 14 additions and 19 deletions
+7
View File
@@ -83,6 +83,13 @@ def test_decode_streaming(tokenizer_id, truth, with_prompt,
assert decoded_text == generated
decoded_text = _run_incremental_decode(
tokenizer, [len(tokenizer)],
skip_special_tokens=skip_special_tokens,
starting_index=starting_index)
assert decoded_text == ''
@pytest.fixture
def detokenizer(tokenizer_name: str) -> Detokenizer: