[Bugfix] Remove tokenizer encode/decode calls from Olmo3 reasoning parser (#40855)

Signed-off-by: Yifan <yzong@redhat.com>
Co-authored-by: Flora Feng <4florafeng@gmail.com>
This commit is contained in:
yzong-rh
2026-04-27 19:36:54 -07:00
committed by GitHub
co-authored by Flora Feng
parent 03aeed802f
commit 0d4f714208
2 changed files with 42 additions and 16 deletions
+12 -1
View File
@@ -41,6 +41,12 @@ SIMPLE_REASONING_WITH_MULTIPLE_NEWLINES = {
"content": "\n\n\nThis is the rest",
}
SIMPLE_REASONING_WITH_TRAILING_SPACE = {
"output": f"{START_REASONING}\nLook!\nI'm thinking... {END_REASONING}\nThis is the rest", # noqa: E501
"reasoning": "\nLook!\nI'm thinking... ",
"content": "\nThis is the rest",
}
NO_REASONING_ONLY_END_THINK = {
"output": f"{END_REASONING}\n\nNo thoughts, head empty!",
"reasoning": None,
@@ -114,6 +120,11 @@ TEST_CASES = [
SIMPLE_REASONING_WITH_MULTIPLE_NEWLINES,
id="simple_reasoning_with_multiple_newlines_streaming",
),
pytest.param(
True, # enable streaming
SIMPLE_REASONING_WITH_TRAILING_SPACE,
id="simple_reasoning_with_trailing_space_streaming",
),
pytest.param(
True, # enable streaming
NO_REASONING_ONLY_END_THINK,
@@ -127,7 +138,7 @@ TEST_CASES = [
]
# Global tokenizer initialization to avoid repeated loading
tokenizer = AutoTokenizer.from_pretrained("allenai/dolma2-tokenizer")
tokenizer = AutoTokenizer.from_pretrained("allenai/Olmo-3-7B-Think")
@pytest.mark.parametrize("streaming, param_dict", TEST_CASES)