[Bugfix] Suppress HarmonyError Unexpected token while expecting start token 200006 (#59254)

Signed-off-by: Yifan Zong <yzong@redhat.com>
This commit is contained in:
yzong-rh
2026-09-29 16:40:15 -04:00
committed by GitHub
parent faacc13565
commit 1d331f8341
2 changed files with 17 additions and 2 deletions
+9
View File
@@ -872,6 +872,15 @@ class TestProcessChunk:
("final", "Two"),
]
def test_malformed_token_does_not_raise(self, harmony_parser):
malformed = encode_output(
"<|channel|>analysis<|message|>think<|end|><|return|>"
"<|start|>assistant<|channel|>final<|message|>answer<|return|>"
)
result = harmony_parser.process_chunk(malformed)
assert "".join(s.delta for s in result.segments if s.delta) == "thinkanswer"
class TestCountReasoningTokens:
def test_matches_process_chunk(self, harmony_parser, gpt_oss_tokenizer):