[Bugfix][Frontend] Count Responses reasoning tokens per tool round (#58927)
Signed-off-by: sfeng33 <4florafeng@gmail.com>
This commit is contained in:
@@ -84,6 +84,18 @@ class _ReasoningOnlyParser(DelegatingParser):
|
||||
return None
|
||||
|
||||
|
||||
class _CountingParser(_NoOpParser):
|
||||
"""Parser that records the ids it counts; token id 7 is reasoning."""
|
||||
|
||||
def __init__(self, tokenizer, *args, **kwargs):
|
||||
super().__init__(tokenizer)
|
||||
self.counted_ids: list[list[int]] = []
|
||||
|
||||
def count_reasoning_tokens(self, token_ids: Sequence[int]) -> int:
|
||||
self.counted_ids.append(list(token_ids))
|
||||
return sum(token_id == 7 for token_id in token_ids)
|
||||
|
||||
|
||||
class _StubToolParser:
|
||||
"""Minimal tool parser stub that always returns a hardcoded tool call."""
|
||||
|
||||
@@ -339,6 +351,25 @@ def test_multi_turn_accumulation():
|
||||
assert texts == ["First turn", "Second turn"]
|
||||
|
||||
|
||||
def test_reasoning_tokens_counted_per_round():
|
||||
"""Each round is counted on its own ids, not the concatenated rounds."""
|
||||
ctx = _make_context(_CountingParser)
|
||||
|
||||
ctx.append_output(_make_request_output(token_ids=[7, 7, 1]))
|
||||
ctx.append_output(_make_request_output(token_ids=[7, 2]))
|
||||
|
||||
assert ctx.response_parser.counted_ids == [[7, 7, 1], [7, 2]]
|
||||
assert ctx.num_reasoning_tokens == 3
|
||||
|
||||
|
||||
def test_reasoning_tokens_zero_without_parser():
|
||||
ctx = _make_context(None)
|
||||
|
||||
ctx.append_output(_make_request_output(token_ids=[7, 7, 1]))
|
||||
|
||||
assert ctx.num_reasoning_tokens == 0
|
||||
|
||||
|
||||
def test_num_init_messages_offset():
|
||||
"""Initial messages are preserved and offset works correctly."""
|
||||
init_messages = [MagicMock(type="message")]
|
||||
|
||||
@@ -960,24 +960,19 @@ class TestValidateGeneratorInput:
|
||||
assert isinstance(result, ErrorResponse)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_reasoning_tokens_counted_for_text_reasoning_model(monkeypatch):
|
||||
"""Ensure reasoning_tokens usage is derived from thinking token spans."""
|
||||
class _Qwen3FakeTokenizer:
|
||||
def __init__(self):
|
||||
self._vocab = {"<think>": 1, "</think>": 2, "reason": 3, "final": 4}
|
||||
|
||||
class FakeTokenizer:
|
||||
def __init__(self):
|
||||
self._vocab = {"<think>": 1, "</think>": 2, "reason": 3, "final": 4}
|
||||
def get_vocab(self):
|
||||
return self._vocab
|
||||
|
||||
def get_vocab(self):
|
||||
return self._vocab
|
||||
def decode(self, token_ids):
|
||||
id_to_token = {v: k for k, v in self._vocab.items()}
|
||||
return "".join(id_to_token.get(token_id, "x") for token_id in token_ids)
|
||||
|
||||
def decode(self, token_ids):
|
||||
id_to_token = {v: k for k, v in self._vocab.items()}
|
||||
return "".join(id_to_token.get(token_id, "x") for token_id in token_ids)
|
||||
|
||||
# Force non-harmony, SimpleContext path
|
||||
monkeypatch.setattr(envs, "VLLM_USE_EXPERIMENTAL_PARSER_CONTEXT", False)
|
||||
|
||||
def _make_qwen3_serving() -> tuple[OpenAIServingResponses, _Qwen3FakeTokenizer]:
|
||||
engine_client = MagicMock()
|
||||
model_config = MagicMock()
|
||||
model_config.hf_config.model_type = "test"
|
||||
@@ -987,21 +982,66 @@ async def test_reasoning_tokens_counted_for_text_reasoning_model(monkeypatch):
|
||||
engine_client.input_processor = MagicMock()
|
||||
engine_client.renderer = MagicMock()
|
||||
|
||||
tokenizer = FakeTokenizer()
|
||||
tokenizer = _Qwen3FakeTokenizer()
|
||||
engine_client.renderer.get_tokenizer.return_value = tokenizer
|
||||
|
||||
models = MagicMock()
|
||||
|
||||
serving = OpenAIServingResponses(
|
||||
engine_client=engine_client,
|
||||
models=models,
|
||||
models=MagicMock(),
|
||||
online_renderer=MagicMock(),
|
||||
request_logger=None,
|
||||
chat_template=None,
|
||||
chat_template_content_format="auto",
|
||||
reasoning_parser="qwen3",
|
||||
)
|
||||
return serving, tokenizer
|
||||
|
||||
|
||||
def _make_text_request_output(text: str, token_ids: list[int]) -> RequestOutput:
|
||||
return RequestOutput(
|
||||
request_id="req",
|
||||
prompt="hi",
|
||||
prompt_token_ids=[7, 8],
|
||||
prompt_logprobs=None,
|
||||
outputs=[
|
||||
CompletionOutput(
|
||||
index=0,
|
||||
text=text,
|
||||
token_ids=token_ids,
|
||||
cumulative_logprob=0.0,
|
||||
logprobs=None,
|
||||
finish_reason="stop",
|
||||
stop_reason=None,
|
||||
)
|
||||
],
|
||||
finished=True,
|
||||
num_cached_tokens=0,
|
||||
num_cache_creation_tokens=3,
|
||||
)
|
||||
|
||||
|
||||
async def _run_full_generator(serving, request, context, tokenizer):
|
||||
async def dummy_result_generator():
|
||||
yield None
|
||||
|
||||
return await serving.responses_full_generator(
|
||||
request=request,
|
||||
sampling_params=SamplingParams(max_tokens=16),
|
||||
result_generator=dummy_result_generator(),
|
||||
context=context,
|
||||
model_name="test-model",
|
||||
tokenizer=tokenizer,
|
||||
request_metadata=RequestResponseMetadata(request_id="req"),
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_reasoning_tokens_counted_for_text_reasoning_model(monkeypatch):
|
||||
"""Ensure reasoning_tokens usage is derived from thinking token spans."""
|
||||
# Force non-harmony, SimpleContext path
|
||||
monkeypatch.setattr(envs, "VLLM_USE_EXPERIMENTAL_PARSER_CONTEXT", False)
|
||||
|
||||
serving, tokenizer = _make_qwen3_serving()
|
||||
request = ResponsesRequest(input="hi", tools=[], stream=False)
|
||||
response_parser = serving._make_response_parser(
|
||||
request,
|
||||
@@ -1011,43 +1051,12 @@ async def test_reasoning_tokens_counted_for_text_reasoning_model(monkeypatch):
|
||||
|
||||
# Build a SimpleContext with thinking tokens in the output.
|
||||
context = SimpleContext(response_parser=response_parser)
|
||||
token_ids = [1, 10, 2, 20] # <think> 10 </think> 20 -> reasoning token count = 1
|
||||
completion = CompletionOutput(
|
||||
index=0,
|
||||
text="<think>reason</think>final",
|
||||
token_ids=token_ids,
|
||||
cumulative_logprob=0.0,
|
||||
logprobs=None,
|
||||
finish_reason="stop",
|
||||
stop_reason=None,
|
||||
# <think> 10 </think> 20 -> reasoning token count = 1
|
||||
context.append_output(
|
||||
_make_text_request_output("<think>reason</think>final", [1, 10, 2, 20])
|
||||
)
|
||||
req_output = RequestOutput(
|
||||
request_id="req",
|
||||
prompt="hi",
|
||||
prompt_token_ids=[7, 8],
|
||||
prompt_logprobs=None,
|
||||
outputs=[completion],
|
||||
finished=True,
|
||||
num_cached_tokens=0,
|
||||
num_cache_creation_tokens=3,
|
||||
)
|
||||
context.append_output(req_output)
|
||||
|
||||
async def dummy_result_generator():
|
||||
yield None
|
||||
|
||||
sampling_params = SamplingParams(max_tokens=16)
|
||||
metadata = RequestResponseMetadata(request_id="req")
|
||||
|
||||
response = await serving.responses_full_generator(
|
||||
request=request,
|
||||
sampling_params=sampling_params,
|
||||
result_generator=dummy_result_generator(),
|
||||
context=context,
|
||||
model_name="test-model",
|
||||
tokenizer=tokenizer,
|
||||
request_metadata=metadata,
|
||||
)
|
||||
response = await _run_full_generator(serving, request, context, tokenizer)
|
||||
|
||||
assert response.usage.output_tokens_details.reasoning_tokens == 1
|
||||
assert response.usage.input_tokens_details.cache_write_tokens == 3
|
||||
|
||||
@@ -359,6 +359,10 @@ class ParsableContext(ConversationContext):
|
||||
enable_auto_tools=self.enable_auto_tools,
|
||||
model_output_token_ids=completion.token_ids,
|
||||
)
|
||||
# Each round is a separate generation, so count it on its own ids.
|
||||
self.num_reasoning_tokens += self.response_parser.count_reasoning_tokens(
|
||||
completion.token_ids or []
|
||||
)
|
||||
if not self.request.include_reasoning:
|
||||
reasoning = None
|
||||
self.response_messages.extend(
|
||||
|
||||
@@ -832,18 +832,12 @@ class OpenAIServingResponses(GenerateBaseServing):
|
||||
num_generated_tokens = context.num_output_tokens
|
||||
num_cached_tokens = context.num_cached_tokens
|
||||
num_reasoning_tokens = context.num_reasoning_tokens
|
||||
# For text-based reasoning parsers (e.g., <think>...</think>),
|
||||
# HarmonyContext already counts reasoning tokens via channels.
|
||||
# For Simple/Parsable contexts, derive reasoning_tokens from
|
||||
# accumulated output token IDs using the parser if not already set.
|
||||
if (
|
||||
num_reasoning_tokens == 0
|
||||
and isinstance(context, (SimpleContext, ParsableContext))
|
||||
and context.response_parser is not None
|
||||
):
|
||||
accumulated = getattr(context, "_accumulated_token_ids", []) or []
|
||||
# HarmonyContext and ParsableContext count reasoning tokens as each
|
||||
# round is appended. SimpleContext is single-round but accumulates
|
||||
# streaming deltas, so count its full output once here.
|
||||
if isinstance(context, SimpleContext) and context.response_parser is not None:
|
||||
num_reasoning_tokens = context.response_parser.count_reasoning_tokens(
|
||||
accumulated
|
||||
context._accumulated_token_ids
|
||||
)
|
||||
|
||||
usage = ResponseUsage(
|
||||
|
||||
Reference in New Issue
Block a user