[Bugfix][Frontend] Count reasoning tokens for Harmony, DeepSeek-V3 and Step3 parsers (#58626)

Signed-off-by: Samyabrata Maji <116789799+sammaji@users.noreply.github.com>
Co-authored-by: Flora Feng <4florafeng@gmail.com>
This commit is contained in:
Samyabrata Maji
2026-09-25 00:12:29 +00:00
committed by GitHub
co-authored by Flora Feng
parent b3a87b2d04
commit 064bc747ab
6 changed files with 136 additions and 3 deletions
@@ -87,3 +87,14 @@ def test_identity_reasoning_parser_basic(tokenizer):
delta_token_ids=[],
)
assert result_none is None
@pytest.mark.parametrize("thinking,expected", [(True, 3), (False, 0)])
def test_count_reasoning_tokens_delegates(tokenizer, thinking, expected):
parser = DeepSeekV3ReasoningParser(
tokenizer, chat_template_kwargs={"thinking": thinking}
)
think = tokenizer.convert_tokens_to_ids(["<think>", "</think>"])
token_ids = [think[0], 11, 12, 13, think[1], 14]
assert parser.count_reasoning_tokens(token_ids) == expected