[Bugfix] Suppress HarmonyError Unexpected token while expecting start token 200006 (#59254)
Signed-off-by: Yifan Zong <yzong@redhat.com>
This commit is contained in:
@@ -872,6 +872,15 @@ class TestProcessChunk:
|
||||
("final", "Two"),
|
||||
]
|
||||
|
||||
def test_malformed_token_does_not_raise(self, harmony_parser):
|
||||
malformed = encode_output(
|
||||
"<|channel|>analysis<|message|>think<|end|><|return|>"
|
||||
"<|start|>assistant<|channel|>final<|message|>answer<|return|>"
|
||||
)
|
||||
|
||||
result = harmony_parser.process_chunk(malformed)
|
||||
assert "".join(s.delta for s in result.segments if s.delta) == "thinkanswer"
|
||||
|
||||
|
||||
class TestCountReasoningTokens:
|
||||
def test_matches_process_chunk(self, harmony_parser, gpt_oss_tokenizer):
|
||||
|
||||
Reference in New Issue
Block a user