[Bugfix] Suppress HarmonyError Unexpected token while expecting start token 200006 (#59254)
Signed-off-by: Yifan Zong <yzong@redhat.com>
This commit is contained in:
@@ -872,6 +872,15 @@ class TestProcessChunk:
|
|||||||
("final", "Two"),
|
("final", "Two"),
|
||||||
]
|
]
|
||||||
|
|
||||||
|
def test_malformed_token_does_not_raise(self, harmony_parser):
|
||||||
|
malformed = encode_output(
|
||||||
|
"<|channel|>analysis<|message|>think<|end|><|return|>"
|
||||||
|
"<|start|>assistant<|channel|>final<|message|>answer<|return|>"
|
||||||
|
)
|
||||||
|
|
||||||
|
result = harmony_parser.process_chunk(malformed)
|
||||||
|
assert "".join(s.delta for s in result.segments if s.delta) == "thinkanswer"
|
||||||
|
|
||||||
|
|
||||||
class TestCountReasoningTokens:
|
class TestCountReasoningTokens:
|
||||||
def test_matches_process_chunk(self, harmony_parser, gpt_oss_tokenizer):
|
def test_matches_process_chunk(self, harmony_parser, gpt_oss_tokenizer):
|
||||||
|
|||||||
@@ -335,7 +335,10 @@ class HarmonyParser(DelegatingParser):
|
|||||||
segments: list[Segment] = []
|
segments: list[Segment] = []
|
||||||
reasoning_token_count = 0
|
reasoning_token_count = 0
|
||||||
for token_id in token_ids:
|
for token_id in token_ids:
|
||||||
self._harmony_parser.process(token_id)
|
try:
|
||||||
|
self._harmony_parser.process(token_id)
|
||||||
|
except HarmonyError:
|
||||||
|
continue
|
||||||
channel = self._harmony_parser.current_channel
|
channel = self._harmony_parser.current_channel
|
||||||
recipient = self._normalize_recipient(
|
recipient = self._normalize_recipient(
|
||||||
self._harmony_parser.current_recipient
|
self._harmony_parser.current_recipient
|
||||||
@@ -376,7 +379,10 @@ class HarmonyParser(DelegatingParser):
|
|||||||
parser = get_streamable_parser_for_assistant()
|
parser = get_streamable_parser_for_assistant()
|
||||||
count = 0
|
count = 0
|
||||||
for token_id in token_ids:
|
for token_id in token_ids:
|
||||||
parser.process(token_id)
|
try:
|
||||||
|
parser.process(token_id)
|
||||||
|
except HarmonyError:
|
||||||
|
continue
|
||||||
recipient = self._normalize_recipient(parser.current_recipient)
|
recipient = self._normalize_recipient(parser.current_recipient)
|
||||||
if self._is_reasoning_token(token_id, parser.current_channel, recipient):
|
if self._is_reasoning_token(token_id, parser.current_channel, recipient):
|
||||||
count += 1
|
count += 1
|
||||||
|
|||||||
Reference in New Issue
Block a user