[Bugfix][Frontend] Count Responses reasoning tokens per tool round (#58927)
Signed-off-by: sfeng33 <4florafeng@gmail.com>
This commit is contained in:
@@ -84,6 +84,18 @@ class _ReasoningOnlyParser(DelegatingParser):
|
||||
return None
|
||||
|
||||
|
||||
class _CountingParser(_NoOpParser):
|
||||
"""Parser that records the ids it counts; token id 7 is reasoning."""
|
||||
|
||||
def __init__(self, tokenizer, *args, **kwargs):
|
||||
super().__init__(tokenizer)
|
||||
self.counted_ids: list[list[int]] = []
|
||||
|
||||
def count_reasoning_tokens(self, token_ids: Sequence[int]) -> int:
|
||||
self.counted_ids.append(list(token_ids))
|
||||
return sum(token_id == 7 for token_id in token_ids)
|
||||
|
||||
|
||||
class _StubToolParser:
|
||||
"""Minimal tool parser stub that always returns a hardcoded tool call."""
|
||||
|
||||
@@ -339,6 +351,25 @@ def test_multi_turn_accumulation():
|
||||
assert texts == ["First turn", "Second turn"]
|
||||
|
||||
|
||||
def test_reasoning_tokens_counted_per_round():
|
||||
"""Each round is counted on its own ids, not the concatenated rounds."""
|
||||
ctx = _make_context(_CountingParser)
|
||||
|
||||
ctx.append_output(_make_request_output(token_ids=[7, 7, 1]))
|
||||
ctx.append_output(_make_request_output(token_ids=[7, 2]))
|
||||
|
||||
assert ctx.response_parser.counted_ids == [[7, 7, 1], [7, 2]]
|
||||
assert ctx.num_reasoning_tokens == 3
|
||||
|
||||
|
||||
def test_reasoning_tokens_zero_without_parser():
|
||||
ctx = _make_context(None)
|
||||
|
||||
ctx.append_output(_make_request_output(token_ids=[7, 7, 1]))
|
||||
|
||||
assert ctx.num_reasoning_tokens == 0
|
||||
|
||||
|
||||
def test_num_init_messages_offset():
|
||||
"""Initial messages are preserved and offset works correctly."""
|
||||
init_messages = [MagicMock(type="message")]
|
||||
|
||||
@@ -960,24 +960,19 @@ class TestValidateGeneratorInput:
|
||||
assert isinstance(result, ErrorResponse)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_reasoning_tokens_counted_for_text_reasoning_model(monkeypatch):
|
||||
"""Ensure reasoning_tokens usage is derived from thinking token spans."""
|
||||
class _Qwen3FakeTokenizer:
|
||||
def __init__(self):
|
||||
self._vocab = {"<think>": 1, "</think>": 2, "reason": 3, "final": 4}
|
||||
|
||||
class FakeTokenizer:
|
||||
def __init__(self):
|
||||
self._vocab = {"<think>": 1, "</think>": 2, "reason": 3, "final": 4}
|
||||
def get_vocab(self):
|
||||
return self._vocab
|
||||
|
||||
def get_vocab(self):
|
||||
return self._vocab
|
||||
def decode(self, token_ids):
|
||||
id_to_token = {v: k for k, v in self._vocab.items()}
|
||||
return "".join(id_to_token.get(token_id, "x") for token_id in token_ids)
|
||||
|
||||
def decode(self, token_ids):
|
||||
id_to_token = {v: k for k, v in self._vocab.items()}
|
||||
return "".join(id_to_token.get(token_id, "x") for token_id in token_ids)
|
||||
|
||||
# Force non-harmony, SimpleContext path
|
||||
monkeypatch.setattr(envs, "VLLM_USE_EXPERIMENTAL_PARSER_CONTEXT", False)
|
||||
|
||||
def _make_qwen3_serving() -> tuple[OpenAIServingResponses, _Qwen3FakeTokenizer]:
|
||||
engine_client = MagicMock()
|
||||
model_config = MagicMock()
|
||||
model_config.hf_config.model_type = "test"
|
||||
@@ -987,21 +982,66 @@ async def test_reasoning_tokens_counted_for_text_reasoning_model(monkeypatch):
|
||||
engine_client.input_processor = MagicMock()
|
||||
engine_client.renderer = MagicMock()
|
||||
|
||||
tokenizer = FakeTokenizer()
|
||||
tokenizer = _Qwen3FakeTokenizer()
|
||||
engine_client.renderer.get_tokenizer.return_value = tokenizer
|
||||
|
||||
models = MagicMock()
|
||||
|
||||
serving = OpenAIServingResponses(
|
||||
engine_client=engine_client,
|
||||
models=models,
|
||||
models=MagicMock(),
|
||||
online_renderer=MagicMock(),
|
||||
request_logger=None,
|
||||
chat_template=None,
|
||||
chat_template_content_format="auto",
|
||||
reasoning_parser="qwen3",
|
||||
)
|
||||
return serving, tokenizer
|
||||
|
||||
|
||||
def _make_text_request_output(text: str, token_ids: list[int]) -> RequestOutput:
|
||||
return RequestOutput(
|
||||
request_id="req",
|
||||
prompt="hi",
|
||||
prompt_token_ids=[7, 8],
|
||||
prompt_logprobs=None,
|
||||
outputs=[
|
||||
CompletionOutput(
|
||||
index=0,
|
||||
text=text,
|
||||
token_ids=token_ids,
|
||||
cumulative_logprob=0.0,
|
||||
logprobs=None,
|
||||
finish_reason="stop",
|
||||
stop_reason=None,
|
||||
)
|
||||
],
|
||||
finished=True,
|
||||
num_cached_tokens=0,
|
||||
num_cache_creation_tokens=3,
|
||||
)
|
||||
|
||||
|
||||
async def _run_full_generator(serving, request, context, tokenizer):
|
||||
async def dummy_result_generator():
|
||||
yield None
|
||||
|
||||
return await serving.responses_full_generator(
|
||||
request=request,
|
||||
sampling_params=SamplingParams(max_tokens=16),
|
||||
result_generator=dummy_result_generator(),
|
||||
context=context,
|
||||
model_name="test-model",
|
||||
tokenizer=tokenizer,
|
||||
request_metadata=RequestResponseMetadata(request_id="req"),
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_reasoning_tokens_counted_for_text_reasoning_model(monkeypatch):
|
||||
"""Ensure reasoning_tokens usage is derived from thinking token spans."""
|
||||
# Force non-harmony, SimpleContext path
|
||||
monkeypatch.setattr(envs, "VLLM_USE_EXPERIMENTAL_PARSER_CONTEXT", False)
|
||||
|
||||
serving, tokenizer = _make_qwen3_serving()
|
||||
request = ResponsesRequest(input="hi", tools=[], stream=False)
|
||||
response_parser = serving._make_response_parser(
|
||||
request,
|
||||
@@ -1011,43 +1051,12 @@ async def test_reasoning_tokens_counted_for_text_reasoning_model(monkeypatch):
|
||||
|
||||
# Build a SimpleContext with thinking tokens in the output.
|
||||
context = SimpleContext(response_parser=response_parser)
|
||||
token_ids = [1, 10, 2, 20] # <think> 10 </think> 20 -> reasoning token count = 1
|
||||
completion = CompletionOutput(
|
||||
index=0,
|
||||
text="<think>reason</think>final",
|
||||
token_ids=token_ids,
|
||||
cumulative_logprob=0.0,
|
||||
logprobs=None,
|
||||
finish_reason="stop",
|
||||
stop_reason=None,
|
||||
# <think> 10 </think> 20 -> reasoning token count = 1
|
||||
context.append_output(
|
||||
_make_text_request_output("<think>reason</think>final", [1, 10, 2, 20])
|
||||
)
|
||||
req_output = RequestOutput(
|
||||
request_id="req",
|
||||
prompt="hi",
|
||||
prompt_token_ids=[7, 8],
|
||||
prompt_logprobs=None,
|
||||
outputs=[completion],
|
||||
finished=True,
|
||||
num_cached_tokens=0,
|
||||
num_cache_creation_tokens=3,
|
||||
)
|
||||
context.append_output(req_output)
|
||||
|
||||
async def dummy_result_generator():
|
||||
yield None
|
||||
|
||||
sampling_params = SamplingParams(max_tokens=16)
|
||||
metadata = RequestResponseMetadata(request_id="req")
|
||||
|
||||
response = await serving.responses_full_generator(
|
||||
request=request,
|
||||
sampling_params=sampling_params,
|
||||
result_generator=dummy_result_generator(),
|
||||
context=context,
|
||||
model_name="test-model",
|
||||
tokenizer=tokenizer,
|
||||
request_metadata=metadata,
|
||||
)
|
||||
response = await _run_full_generator(serving, request, context, tokenizer)
|
||||
|
||||
assert response.usage.output_tokens_details.reasoning_tokens == 1
|
||||
assert response.usage.input_tokens_details.cache_write_tokens == 3
|
||||
|
||||
Reference in New Issue
Block a user