[Bugfix] Remove tokenizer encode/decode calls from Olmo3 reasoning parser (#40855)
Signed-off-by: Yifan <yzong@redhat.com> Co-authored-by: Flora Feng <4florafeng@gmail.com>
This commit is contained in:
@@ -41,6 +41,12 @@ SIMPLE_REASONING_WITH_MULTIPLE_NEWLINES = {
|
||||
"content": "\n\n\nThis is the rest",
|
||||
}
|
||||
|
||||
SIMPLE_REASONING_WITH_TRAILING_SPACE = {
|
||||
"output": f"{START_REASONING}\nLook!\nI'm thinking... {END_REASONING}\nThis is the rest", # noqa: E501
|
||||
"reasoning": "\nLook!\nI'm thinking... ",
|
||||
"content": "\nThis is the rest",
|
||||
}
|
||||
|
||||
NO_REASONING_ONLY_END_THINK = {
|
||||
"output": f"{END_REASONING}\n\nNo thoughts, head empty!",
|
||||
"reasoning": None,
|
||||
@@ -114,6 +120,11 @@ TEST_CASES = [
|
||||
SIMPLE_REASONING_WITH_MULTIPLE_NEWLINES,
|
||||
id="simple_reasoning_with_multiple_newlines_streaming",
|
||||
),
|
||||
pytest.param(
|
||||
True, # enable streaming
|
||||
SIMPLE_REASONING_WITH_TRAILING_SPACE,
|
||||
id="simple_reasoning_with_trailing_space_streaming",
|
||||
),
|
||||
pytest.param(
|
||||
True, # enable streaming
|
||||
NO_REASONING_ONLY_END_THINK,
|
||||
@@ -127,7 +138,7 @@ TEST_CASES = [
|
||||
]
|
||||
|
||||
# Global tokenizer initialization to avoid repeated loading
|
||||
tokenizer = AutoTokenizer.from_pretrained("allenai/dolma2-tokenizer")
|
||||
tokenizer = AutoTokenizer.from_pretrained("allenai/Olmo-3-7B-Think")
|
||||
|
||||
|
||||
@pytest.mark.parametrize("streaming, param_dict", TEST_CASES)
|
||||
|
||||
Reference in New Issue
Block a user