diff --git a/tests/entrypoints/openai/responses/test_parsable_context_unit.py b/tests/entrypoints/openai/responses/test_parsable_context_unit.py index 85bf6f2020..0398723184 100644 --- a/tests/entrypoints/openai/responses/test_parsable_context_unit.py +++ b/tests/entrypoints/openai/responses/test_parsable_context_unit.py @@ -84,6 +84,18 @@ class _ReasoningOnlyParser(DelegatingParser): return None +class _CountingParser(_NoOpParser): + """Parser that records the ids it counts; token id 7 is reasoning.""" + + def __init__(self, tokenizer, *args, **kwargs): + super().__init__(tokenizer) + self.counted_ids: list[list[int]] = [] + + def count_reasoning_tokens(self, token_ids: Sequence[int]) -> int: + self.counted_ids.append(list(token_ids)) + return sum(token_id == 7 for token_id in token_ids) + + class _StubToolParser: """Minimal tool parser stub that always returns a hardcoded tool call.""" @@ -339,6 +351,25 @@ def test_multi_turn_accumulation(): assert texts == ["First turn", "Second turn"] +def test_reasoning_tokens_counted_per_round(): + """Each round is counted on its own ids, not the concatenated rounds.""" + ctx = _make_context(_CountingParser) + + ctx.append_output(_make_request_output(token_ids=[7, 7, 1])) + ctx.append_output(_make_request_output(token_ids=[7, 2])) + + assert ctx.response_parser.counted_ids == [[7, 7, 1], [7, 2]] + assert ctx.num_reasoning_tokens == 3 + + +def test_reasoning_tokens_zero_without_parser(): + ctx = _make_context(None) + + ctx.append_output(_make_request_output(token_ids=[7, 7, 1])) + + assert ctx.num_reasoning_tokens == 0 + + def test_num_init_messages_offset(): """Initial messages are preserved and offset works correctly.""" init_messages = [MagicMock(type="message")] diff --git a/tests/entrypoints/openai/responses/test_serving_responses.py b/tests/entrypoints/openai/responses/test_serving_responses.py index 429fcf1d0d..89c188ec8a 100644 --- a/tests/entrypoints/openai/responses/test_serving_responses.py +++ b/tests/entrypoints/openai/responses/test_serving_responses.py @@ -960,24 +960,19 @@ class TestValidateGeneratorInput: assert isinstance(result, ErrorResponse) -@pytest.mark.asyncio -async def test_reasoning_tokens_counted_for_text_reasoning_model(monkeypatch): - """Ensure reasoning_tokens usage is derived from thinking token spans.""" +class _Qwen3FakeTokenizer: + def __init__(self): + self._vocab = {"": 1, "": 2, "reason": 3, "final": 4} - class FakeTokenizer: - def __init__(self): - self._vocab = {"": 1, "": 2, "reason": 3, "final": 4} + def get_vocab(self): + return self._vocab - def get_vocab(self): - return self._vocab + def decode(self, token_ids): + id_to_token = {v: k for k, v in self._vocab.items()} + return "".join(id_to_token.get(token_id, "x") for token_id in token_ids) - def decode(self, token_ids): - id_to_token = {v: k for k, v in self._vocab.items()} - return "".join(id_to_token.get(token_id, "x") for token_id in token_ids) - - # Force non-harmony, SimpleContext path - monkeypatch.setattr(envs, "VLLM_USE_EXPERIMENTAL_PARSER_CONTEXT", False) +def _make_qwen3_serving() -> tuple[OpenAIServingResponses, _Qwen3FakeTokenizer]: engine_client = MagicMock() model_config = MagicMock() model_config.hf_config.model_type = "test" @@ -987,21 +982,66 @@ async def test_reasoning_tokens_counted_for_text_reasoning_model(monkeypatch): engine_client.input_processor = MagicMock() engine_client.renderer = MagicMock() - tokenizer = FakeTokenizer() + tokenizer = _Qwen3FakeTokenizer() engine_client.renderer.get_tokenizer.return_value = tokenizer - models = MagicMock() - serving = OpenAIServingResponses( engine_client=engine_client, - models=models, + models=MagicMock(), online_renderer=MagicMock(), request_logger=None, chat_template=None, chat_template_content_format="auto", reasoning_parser="qwen3", ) + return serving, tokenizer + +def _make_text_request_output(text: str, token_ids: list[int]) -> RequestOutput: + return RequestOutput( + request_id="req", + prompt="hi", + prompt_token_ids=[7, 8], + prompt_logprobs=None, + outputs=[ + CompletionOutput( + index=0, + text=text, + token_ids=token_ids, + cumulative_logprob=0.0, + logprobs=None, + finish_reason="stop", + stop_reason=None, + ) + ], + finished=True, + num_cached_tokens=0, + num_cache_creation_tokens=3, + ) + + +async def _run_full_generator(serving, request, context, tokenizer): + async def dummy_result_generator(): + yield None + + return await serving.responses_full_generator( + request=request, + sampling_params=SamplingParams(max_tokens=16), + result_generator=dummy_result_generator(), + context=context, + model_name="test-model", + tokenizer=tokenizer, + request_metadata=RequestResponseMetadata(request_id="req"), + ) + + +@pytest.mark.asyncio +async def test_reasoning_tokens_counted_for_text_reasoning_model(monkeypatch): + """Ensure reasoning_tokens usage is derived from thinking token spans.""" + # Force non-harmony, SimpleContext path + monkeypatch.setattr(envs, "VLLM_USE_EXPERIMENTAL_PARSER_CONTEXT", False) + + serving, tokenizer = _make_qwen3_serving() request = ResponsesRequest(input="hi", tools=[], stream=False) response_parser = serving._make_response_parser( request, @@ -1011,43 +1051,12 @@ async def test_reasoning_tokens_counted_for_text_reasoning_model(monkeypatch): # Build a SimpleContext with thinking tokens in the output. context = SimpleContext(response_parser=response_parser) - token_ids = [1, 10, 2, 20] # 10 20 -> reasoning token count = 1 - completion = CompletionOutput( - index=0, - text="reasonfinal", - token_ids=token_ids, - cumulative_logprob=0.0, - logprobs=None, - finish_reason="stop", - stop_reason=None, + # 10 20 -> reasoning token count = 1 + context.append_output( + _make_text_request_output("reasonfinal", [1, 10, 2, 20]) ) - req_output = RequestOutput( - request_id="req", - prompt="hi", - prompt_token_ids=[7, 8], - prompt_logprobs=None, - outputs=[completion], - finished=True, - num_cached_tokens=0, - num_cache_creation_tokens=3, - ) - context.append_output(req_output) - async def dummy_result_generator(): - yield None - - sampling_params = SamplingParams(max_tokens=16) - metadata = RequestResponseMetadata(request_id="req") - - response = await serving.responses_full_generator( - request=request, - sampling_params=sampling_params, - result_generator=dummy_result_generator(), - context=context, - model_name="test-model", - tokenizer=tokenizer, - request_metadata=metadata, - ) + response = await _run_full_generator(serving, request, context, tokenizer) assert response.usage.output_tokens_details.reasoning_tokens == 1 assert response.usage.input_tokens_details.cache_write_tokens == 3 diff --git a/vllm/entrypoints/openai/responses/context.py b/vllm/entrypoints/openai/responses/context.py index 59b42f96a3..fa233dcce1 100644 --- a/vllm/entrypoints/openai/responses/context.py +++ b/vllm/entrypoints/openai/responses/context.py @@ -359,6 +359,10 @@ class ParsableContext(ConversationContext): enable_auto_tools=self.enable_auto_tools, model_output_token_ids=completion.token_ids, ) + # Each round is a separate generation, so count it on its own ids. + self.num_reasoning_tokens += self.response_parser.count_reasoning_tokens( + completion.token_ids or [] + ) if not self.request.include_reasoning: reasoning = None self.response_messages.extend( diff --git a/vllm/entrypoints/openai/responses/serving.py b/vllm/entrypoints/openai/responses/serving.py index 9fba86b80c..f18c493259 100644 --- a/vllm/entrypoints/openai/responses/serving.py +++ b/vllm/entrypoints/openai/responses/serving.py @@ -832,18 +832,12 @@ class OpenAIServingResponses(GenerateBaseServing): num_generated_tokens = context.num_output_tokens num_cached_tokens = context.num_cached_tokens num_reasoning_tokens = context.num_reasoning_tokens - # For text-based reasoning parsers (e.g., ...), - # HarmonyContext already counts reasoning tokens via channels. - # For Simple/Parsable contexts, derive reasoning_tokens from - # accumulated output token IDs using the parser if not already set. - if ( - num_reasoning_tokens == 0 - and isinstance(context, (SimpleContext, ParsableContext)) - and context.response_parser is not None - ): - accumulated = getattr(context, "_accumulated_token_ids", []) or [] + # HarmonyContext and ParsableContext count reasoning tokens as each + # round is appended. SimpleContext is single-round but accumulates + # streaming deltas, so count its full output once here. + if isinstance(context, SimpleContext) and context.response_parser is not None: num_reasoning_tokens = context.response_parser.count_reasoning_tokens( - accumulated + context._accumulated_token_ids ) usage = ResponseUsage(