[Bugfix][Frontend] Count Responses reasoning tokens per tool round (#58927)

Signed-off-by: sfeng33 <4florafeng@gmail.com>
This commit is contained in:
Flora Feng
2026-09-28 00:34:38 +00:00
committed by GitHub
parent 31afa4f1bb
commit 3184226984
4 changed files with 102 additions and 64 deletions
@@ -84,6 +84,18 @@ class _ReasoningOnlyParser(DelegatingParser):
return None
class _CountingParser(_NoOpParser):
"""Parser that records the ids it counts; token id 7 is reasoning."""
def __init__(self, tokenizer, *args, **kwargs):
super().__init__(tokenizer)
self.counted_ids: list[list[int]] = []
def count_reasoning_tokens(self, token_ids: Sequence[int]) -> int:
self.counted_ids.append(list(token_ids))
return sum(token_id == 7 for token_id in token_ids)
class _StubToolParser:
"""Minimal tool parser stub that always returns a hardcoded tool call."""
@@ -339,6 +351,25 @@ def test_multi_turn_accumulation():
assert texts == ["First turn", "Second turn"]
def test_reasoning_tokens_counted_per_round():
"""Each round is counted on its own ids, not the concatenated rounds."""
ctx = _make_context(_CountingParser)
ctx.append_output(_make_request_output(token_ids=[7, 7, 1]))
ctx.append_output(_make_request_output(token_ids=[7, 2]))
assert ctx.response_parser.counted_ids == [[7, 7, 1], [7, 2]]
assert ctx.num_reasoning_tokens == 3
def test_reasoning_tokens_zero_without_parser():
ctx = _make_context(None)
ctx.append_output(_make_request_output(token_ids=[7, 7, 1]))
assert ctx.num_reasoning_tokens == 0
def test_num_init_messages_offset():
"""Initial messages are preserved and offset works correctly."""
init_messages = [MagicMock(type="message")]
@@ -960,24 +960,19 @@ class TestValidateGeneratorInput:
assert isinstance(result, ErrorResponse)
@pytest.mark.asyncio
async def test_reasoning_tokens_counted_for_text_reasoning_model(monkeypatch):
"""Ensure reasoning_tokens usage is derived from thinking token spans."""
class _Qwen3FakeTokenizer:
def __init__(self):
self._vocab = {"<think>": 1, "</think>": 2, "reason": 3, "final": 4}
class FakeTokenizer:
def __init__(self):
self._vocab = {"<think>": 1, "</think>": 2, "reason": 3, "final": 4}
def get_vocab(self):
return self._vocab
def get_vocab(self):
return self._vocab
def decode(self, token_ids):
id_to_token = {v: k for k, v in self._vocab.items()}
return "".join(id_to_token.get(token_id, "x") for token_id in token_ids)
def decode(self, token_ids):
id_to_token = {v: k for k, v in self._vocab.items()}
return "".join(id_to_token.get(token_id, "x") for token_id in token_ids)
# Force non-harmony, SimpleContext path
monkeypatch.setattr(envs, "VLLM_USE_EXPERIMENTAL_PARSER_CONTEXT", False)
def _make_qwen3_serving() -> tuple[OpenAIServingResponses, _Qwen3FakeTokenizer]:
engine_client = MagicMock()
model_config = MagicMock()
model_config.hf_config.model_type = "test"
@@ -987,21 +982,66 @@ async def test_reasoning_tokens_counted_for_text_reasoning_model(monkeypatch):
engine_client.input_processor = MagicMock()
engine_client.renderer = MagicMock()
tokenizer = FakeTokenizer()
tokenizer = _Qwen3FakeTokenizer()
engine_client.renderer.get_tokenizer.return_value = tokenizer
models = MagicMock()
serving = OpenAIServingResponses(
engine_client=engine_client,
models=models,
models=MagicMock(),
online_renderer=MagicMock(),
request_logger=None,
chat_template=None,
chat_template_content_format="auto",
reasoning_parser="qwen3",
)
return serving, tokenizer
def _make_text_request_output(text: str, token_ids: list[int]) -> RequestOutput:
return RequestOutput(
request_id="req",
prompt="hi",
prompt_token_ids=[7, 8],
prompt_logprobs=None,
outputs=[
CompletionOutput(
index=0,
text=text,
token_ids=token_ids,
cumulative_logprob=0.0,
logprobs=None,
finish_reason="stop",
stop_reason=None,
)
],
finished=True,
num_cached_tokens=0,
num_cache_creation_tokens=3,
)
async def _run_full_generator(serving, request, context, tokenizer):
async def dummy_result_generator():
yield None
return await serving.responses_full_generator(
request=request,
sampling_params=SamplingParams(max_tokens=16),
result_generator=dummy_result_generator(),
context=context,
model_name="test-model",
tokenizer=tokenizer,
request_metadata=RequestResponseMetadata(request_id="req"),
)
@pytest.mark.asyncio
async def test_reasoning_tokens_counted_for_text_reasoning_model(monkeypatch):
"""Ensure reasoning_tokens usage is derived from thinking token spans."""
# Force non-harmony, SimpleContext path
monkeypatch.setattr(envs, "VLLM_USE_EXPERIMENTAL_PARSER_CONTEXT", False)
serving, tokenizer = _make_qwen3_serving()
request = ResponsesRequest(input="hi", tools=[], stream=False)
response_parser = serving._make_response_parser(
request,
@@ -1011,43 +1051,12 @@ async def test_reasoning_tokens_counted_for_text_reasoning_model(monkeypatch):
# Build a SimpleContext with thinking tokens in the output.
context = SimpleContext(response_parser=response_parser)
token_ids = [1, 10, 2, 20] # <think> 10 </think> 20 -> reasoning token count = 1
completion = CompletionOutput(
index=0,
text="<think>reason</think>final",
token_ids=token_ids,
cumulative_logprob=0.0,
logprobs=None,
finish_reason="stop",
stop_reason=None,
# <think> 10 </think> 20 -> reasoning token count = 1
context.append_output(
_make_text_request_output("<think>reason</think>final", [1, 10, 2, 20])
)
req_output = RequestOutput(
request_id="req",
prompt="hi",
prompt_token_ids=[7, 8],
prompt_logprobs=None,
outputs=[completion],
finished=True,
num_cached_tokens=0,
num_cache_creation_tokens=3,
)
context.append_output(req_output)
async def dummy_result_generator():
yield None
sampling_params = SamplingParams(max_tokens=16)
metadata = RequestResponseMetadata(request_id="req")
response = await serving.responses_full_generator(
request=request,
sampling_params=sampling_params,
result_generator=dummy_result_generator(),
context=context,
model_name="test-model",
tokenizer=tokenizer,
request_metadata=metadata,
)
response = await _run_full_generator(serving, request, context, tokenizer)
assert response.usage.output_tokens_details.reasoning_tokens == 1
assert response.usage.input_tokens_details.cache_write_tokens == 3
@@ -359,6 +359,10 @@ class ParsableContext(ConversationContext):
enable_auto_tools=self.enable_auto_tools,
model_output_token_ids=completion.token_ids,
)
# Each round is a separate generation, so count it on its own ids.
self.num_reasoning_tokens += self.response_parser.count_reasoning_tokens(
completion.token_ids or []
)
if not self.request.include_reasoning:
reasoning = None
self.response_messages.extend(
+5 -11
View File
@@ -832,18 +832,12 @@ class OpenAIServingResponses(GenerateBaseServing):
num_generated_tokens = context.num_output_tokens
num_cached_tokens = context.num_cached_tokens
num_reasoning_tokens = context.num_reasoning_tokens
# For text-based reasoning parsers (e.g., <think>...</think>),
# HarmonyContext already counts reasoning tokens via channels.
# For Simple/Parsable contexts, derive reasoning_tokens from
# accumulated output token IDs using the parser if not already set.
if (
num_reasoning_tokens == 0
and isinstance(context, (SimpleContext, ParsableContext))
and context.response_parser is not None
):
accumulated = getattr(context, "_accumulated_token_ids", []) or []
# HarmonyContext and ParsableContext count reasoning tokens as each
# round is appended. SimpleContext is single-round but accumulates
# streaming deltas, so count its full output once here.
if isinstance(context, SimpleContext) and context.response_parser is not None:
num_reasoning_tokens = context.response_parser.count_reasoning_tokens(
accumulated
context._accumulated_token_ids
)
usage = ResponseUsage(