[Refactor] Simplify BOS/EOS token handling (#34435)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
This commit is contained in:
@@ -1123,7 +1123,7 @@ rectangle
|
||||
|
||||
# Encode all content tokens at once
|
||||
all_token_ids = step3p5_tokenizer.encode(model_output, add_special_tokens=False)
|
||||
eos_token_id = getattr(step3p5_tokenizer, "eos_token_id", None)
|
||||
eos_token_id = step3p5_tokenizer.eos_token_id
|
||||
|
||||
# Include EOS token in delta_token_ids if available
|
||||
if eos_token_id is not None:
|
||||
|
||||
Reference in New Issue
Block a user