[Frontend] Support count_reasoning_tokens in the Streaming Parser Engine (#45802)
Signed-off-by: chaunceyjiang <chaunceyjiang@gmail.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
5cecfc0137
commit
ac2ae8798c
@@ -28,9 +28,11 @@ from vllm.entrypoints.openai.chat_completion.protocol import (
|
||||
from vllm.entrypoints.openai.chat_completion.serving import (
|
||||
OpenAIServingChat,
|
||||
_get_mm_token_counts,
|
||||
_make_completion_tokens_details,
|
||||
_make_prompt_tokens_details,
|
||||
)
|
||||
from vllm.entrypoints.openai.engine.protocol import (
|
||||
DeltaMessage,
|
||||
ErrorResponse,
|
||||
RequestResponseMetadata,
|
||||
)
|
||||
@@ -626,6 +628,7 @@ def _build_minimal_metrics_serving_chat(
|
||||
serving.response_role = "assistant"
|
||||
serving.parser_cls = None
|
||||
serving.enable_auto_tools = False
|
||||
serving._include_reasoning_tokens_details = False
|
||||
serving.enable_prompt_tokens_details = False
|
||||
serving.enable_log_outputs = False
|
||||
serving.enable_log_deltas = False
|
||||
@@ -666,6 +669,13 @@ async def _single_request_output(
|
||||
yield request_output
|
||||
|
||||
|
||||
async def _stream_request_outputs(
|
||||
*request_outputs: RequestOutput,
|
||||
) -> AsyncIterator[RequestOutput]:
|
||||
for request_output in request_outputs:
|
||||
yield request_output
|
||||
|
||||
|
||||
async def _collect_metrics_stream_chunks(
|
||||
serving: OpenAIServingChat,
|
||||
request: ChatCompletionRequest,
|
||||
@@ -724,6 +734,7 @@ async def test_chat_per_request_metrics_follow_server_flag():
|
||||
request_metadata=RequestResponseMetadata(request_id="chatcmpl-test-id"),
|
||||
)
|
||||
assert disabled_response.metrics is None
|
||||
assert disabled_response.usage.completion_tokens_details is None
|
||||
|
||||
enabled_serving = _build_minimal_metrics_serving_chat(
|
||||
enable_per_request_metrics=True
|
||||
@@ -781,6 +792,55 @@ async def test_chat_streaming_metrics_ride_on_usage_chunk():
|
||||
assert usage_chunks[-1]["metrics"]["time_to_first_token_ms"] == pytest.approx(500.0)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_streaming_reasoning_usage_counts_across_deltas():
|
||||
serving = _build_minimal_metrics_serving_chat(enable_per_request_metrics=False)
|
||||
serving._include_reasoning_tokens_details = True
|
||||
serving.model_config = None
|
||||
|
||||
parser = MagicMock()
|
||||
parser.parse_delta.side_effect = [
|
||||
DeltaMessage(reasoning="reasoning"),
|
||||
DeltaMessage(content="answer"),
|
||||
]
|
||||
parser.count_reasoning_tokens.side_effect = lambda token_ids: sum(
|
||||
token_id == 20 for token_id in token_ids
|
||||
)
|
||||
serving.parser_cls = MagicMock(return_value=parser)
|
||||
|
||||
first = _make_metrics_request_output(metrics=None, token_ids=(10, 20))
|
||||
first.outputs[0].text = "<think>reasoning"
|
||||
first.outputs[0].finish_reason = None
|
||||
second = _make_metrics_request_output(metrics=None, token_ids=(11, 30))
|
||||
second.outputs[0].text = "</think>answer"
|
||||
|
||||
request = ChatCompletionRequest(
|
||||
model="test-model",
|
||||
messages=[{"role": "user", "content": "Test prompt"}],
|
||||
max_tokens=10,
|
||||
stream=True,
|
||||
stream_options={"include_usage": True},
|
||||
)
|
||||
chunks: list[dict[str, Any]] = []
|
||||
async for line in serving.chat_completion_stream_generator(
|
||||
request,
|
||||
_stream_request_outputs(first, second),
|
||||
"chatcmpl-test-id",
|
||||
"test-model",
|
||||
conversation=[{"role": "user", "content": "Test"}],
|
||||
tokenizer=MagicMock(),
|
||||
request_metadata=RequestResponseMetadata(request_id="chatcmpl-test-id"),
|
||||
):
|
||||
payload = line.removeprefix("data: ").strip()
|
||||
if payload != "[DONE]":
|
||||
chunks.append(json.loads(payload))
|
||||
|
||||
usage_chunks = [chunk for chunk in chunks if chunk.get("usage")]
|
||||
assert usage_chunks[-1]["usage"]["completion_tokens_details"] == {
|
||||
"reasoning_tokens": 1
|
||||
}
|
||||
|
||||
|
||||
@dataclass
|
||||
class MockEngine:
|
||||
model_config: MockModelConfig = field(default_factory=MockModelConfig)
|
||||
@@ -810,6 +870,7 @@ async def _async_serving_chat_init():
|
||||
def test_async_serving_chat_init():
|
||||
serving_completion = asyncio.run(_async_serving_chat_init())
|
||||
assert serving_completion.chat_template == CHAT_TEMPLATE
|
||||
assert serving_completion._include_reasoning_tokens_details is False
|
||||
|
||||
|
||||
def test_mm_prompt_tokens_details():
|
||||
@@ -850,6 +911,10 @@ def test_mm_prompt_tokens_details():
|
||||
assert details.multimodal_tokens == {"image": 600, "video": 1200}
|
||||
|
||||
|
||||
def test_completion_tokens_details():
|
||||
assert _make_completion_tokens_details(7).reasoning_tokens == 7
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_serving_chat_returns_correct_model_name():
|
||||
mock_engine = MagicMock(spec=AsyncLLM)
|
||||
|
||||
@@ -360,6 +360,10 @@ async def test_reasoning_tokens_counted_for_text_reasoning_model(monkeypatch):
|
||||
def get_vocab(self):
|
||||
return self._vocab
|
||||
|
||||
def decode(self, token_ids):
|
||||
id_to_token = {v: k for k, v in self._vocab.items()}
|
||||
return "".join(id_to_token.get(token_id, "x") for token_id in token_ids)
|
||||
|
||||
# Force non-harmony, SimpleContext path
|
||||
monkeypatch.setattr(envs, "VLLM_USE_EXPERIMENTAL_PARSER_CONTEXT", False)
|
||||
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Tests for the streaming parser engine core pipeline."""
|
||||
|
||||
from dataclasses import replace
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import pytest
|
||||
@@ -71,6 +72,16 @@ def _think_config() -> ParserEngineConfig:
|
||||
)
|
||||
|
||||
|
||||
def _token_think_config() -> ParserEngineConfig:
|
||||
return replace(
|
||||
_think_config(),
|
||||
token_id_terminals={
|
||||
"THINK_START": "<think>",
|
||||
"THINK_END": "</think>",
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
class TestNonStreaming:
|
||||
def test_plain_text(self):
|
||||
engine = StreamingParserEngine(_hermes_config(), tokenizer=None)
|
||||
@@ -441,6 +452,46 @@ class TestTokenIdFiltering:
|
||||
assert sum(1 for e in all_events if e.type == EventType.TOOL_CALL_END) == 1
|
||||
|
||||
|
||||
class TestReasoningTokenCounts:
|
||||
def test_counts_reasoning_and_excludes_boundaries_and_content(self):
|
||||
engine = StreamingParserEngine(_token_think_config(), _make_think_tokenizer())
|
||||
|
||||
events = engine.feed(
|
||||
"<think>tok1tok2</think>tok3",
|
||||
[_START_ID, 1, 2, _END_ID, 3],
|
||||
)
|
||||
events.extend(engine.finish())
|
||||
|
||||
reasoning = [e for e in events if e.type == EventType.REASONING_CHUNK]
|
||||
assert "".join(e.value for e in reasoning) == "tok1tok2"
|
||||
assert sum(e.token_count for e in reasoning) == 2
|
||||
assert engine.reasoning_token_count == 2
|
||||
|
||||
def test_counts_tokens_after_final_deferred_start_terminal(self):
|
||||
engine = StreamingParserEngine(_token_think_config(), _make_think_tokenizer())
|
||||
|
||||
assert engine.feed("", [_START_ID, 1, 2]) == []
|
||||
events = engine.feed("<think>tok1tok2", [])
|
||||
|
||||
reasoning = [e for e in events if e.type == EventType.REASONING_CHUNK]
|
||||
assert "".join(e.value for e in reasoning) == "tok1tok2"
|
||||
assert sum(e.token_count for e in reasoning) == 2
|
||||
assert engine.reasoning_token_count == 2
|
||||
|
||||
def test_deferred_reasoning_tokens_stay_before_next_end_terminal(self):
|
||||
engine = StreamingParserEngine(_token_think_config(), _make_think_tokenizer())
|
||||
|
||||
assert engine.feed("", [_START_ID, 1, 2]) == []
|
||||
events = engine.feed("<think>tok1tok2</think>tok3", [_END_ID, 3])
|
||||
|
||||
assert (
|
||||
sum(e.token_count for e in events if e.type == EventType.REASONING_CHUNK)
|
||||
== 2
|
||||
)
|
||||
assert sum(e.token_count for e in events if e.type == EventType.TEXT_CHUNK) == 1
|
||||
assert engine.reasoning_token_count == 2
|
||||
|
||||
|
||||
def _func_prefix_config() -> ParserEngineConfig:
|
||||
"""Config mixing token-ID terminals (TOOL_START/END) with
|
||||
text-only terminals (FUNC_PREFIX) and fallback transitions."""
|
||||
|
||||
@@ -33,6 +33,7 @@ from vllm.parser.engine.parser_engine_config import (
|
||||
ParserState,
|
||||
Transition,
|
||||
)
|
||||
from vllm.parser.parser_manager import ParserManager
|
||||
|
||||
# ── Shared test configs ──────────────────────────────────────────────
|
||||
|
||||
@@ -892,6 +893,129 @@ class _CombinedDelegating(DelegatingParser):
|
||||
tool_parser_cls = _CombinedToolAdapter
|
||||
|
||||
|
||||
def test_parser_manager_uses_shared_engine_directly(monkeypatch):
|
||||
monkeypatch.setattr(
|
||||
ParserManager,
|
||||
"get_reasoning_parser",
|
||||
classmethod(lambda cls, name: _CombinedReasoningAdapter),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
ParserManager,
|
||||
"get_tool_parser",
|
||||
classmethod(lambda cls, name, enabled, model: _CombinedToolAdapter),
|
||||
)
|
||||
|
||||
parser_cls = ParserManager.get_parser(
|
||||
tool_parser_name="combined",
|
||||
reasoning_parser_name="combined",
|
||||
enable_auto_tools=True,
|
||||
)
|
||||
|
||||
assert parser_cls is not None
|
||||
assert parser_cls is _CombinedTestEngine
|
||||
parser = parser_cls(make_mock_tokenizer(_VOCAB))
|
||||
request = _make_delegating_request()
|
||||
reasoning, content, _ = parser.parse(
|
||||
"ab</think>c",
|
||||
request,
|
||||
model_output_token_ids=[ord("a"), ord("b"), 201, ord("c")],
|
||||
)
|
||||
assert reasoning == "ab"
|
||||
assert content == "c"
|
||||
assert parser.count_reasoning_tokens([]) == 2
|
||||
|
||||
|
||||
def test_parser_manager_preserves_reasoning_only_adapter(monkeypatch):
|
||||
monkeypatch.setattr(
|
||||
ParserManager,
|
||||
"get_reasoning_parser",
|
||||
classmethod(lambda cls, name: _CombinedReasoningAdapter),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
ParserManager,
|
||||
"get_tool_parser",
|
||||
classmethod(lambda cls, name, enabled, model: None),
|
||||
)
|
||||
|
||||
parser_cls = ParserManager.get_parser(reasoning_parser_name="combined")
|
||||
|
||||
assert parser_cls is not None
|
||||
parser = parser_cls(make_mock_tokenizer(_VOCAB))
|
||||
assert parser.reasoning_parser is not None
|
||||
assert parser.tool_parser is None
|
||||
reasoning, content, _ = parser.parse(
|
||||
'ab</think><tool_call>{"name":"h","arguments":{}}</tool_call>',
|
||||
_make_delegating_request(),
|
||||
model_output_token_ids=[ord("a"), ord("b"), 201],
|
||||
)
|
||||
assert reasoning == "ab"
|
||||
assert content == '<tool_call>{"name":"h","arguments":{}}</tool_call>'
|
||||
assert parser.count_reasoning_tokens([ord("a"), ord("b"), 201]) == 2
|
||||
|
||||
|
||||
def test_parser_manager_preserves_tool_only_adapter(monkeypatch):
|
||||
monkeypatch.setattr(
|
||||
ParserManager,
|
||||
"get_reasoning_parser",
|
||||
classmethod(lambda cls, name: None),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
ParserManager,
|
||||
"get_tool_parser",
|
||||
classmethod(lambda cls, name, enabled, model: _CombinedToolAdapter),
|
||||
)
|
||||
|
||||
parser_cls = ParserManager.get_parser(
|
||||
tool_parser_name="combined", enable_auto_tools=True
|
||||
)
|
||||
|
||||
assert parser_cls is not None
|
||||
parser = parser_cls(make_mock_tokenizer(_VOCAB))
|
||||
assert parser.reasoning_parser is None
|
||||
assert parser.tool_parser is not None
|
||||
reasoning, content, tool_calls = parser.parse(
|
||||
"<think>ab</think>c",
|
||||
_make_delegating_request(),
|
||||
model_output_token_ids=[200, ord("a"), ord("b"), 201, ord("c")],
|
||||
)
|
||||
assert reasoning is None
|
||||
assert content == "<think>ab</think>c"
|
||||
assert tool_calls == []
|
||||
|
||||
|
||||
def test_parser_manager_rejects_non_engine_adapter_metadata():
|
||||
class TraditionalParser:
|
||||
pass
|
||||
|
||||
class InvalidEngineAdapter:
|
||||
_parser_engine_cls = TraditionalParser
|
||||
|
||||
assert ParserManager._get_parser_engine_cls(TraditionalParser) is None
|
||||
assert ParserManager._get_parser_engine_cls(InvalidEngineAdapter) is None
|
||||
assert (
|
||||
ParserManager._get_parser_engine_cls(_CombinedReasoningAdapter)
|
||||
is _CombinedTestEngine
|
||||
)
|
||||
|
||||
|
||||
def test_reasoning_adapter_counts_after_final_non_streaming_parse():
|
||||
parser = _CombinedReasoningAdapter(make_mock_tokenizer(_VOCAB))
|
||||
request = _make_delegating_request()
|
||||
token_ids = [ord("a"), ord("b"), 201, ord("c")]
|
||||
|
||||
parser.extract_reasoning_streaming(
|
||||
"",
|
||||
"ab</think>c",
|
||||
"ab</think>c",
|
||||
[],
|
||||
token_ids,
|
||||
token_ids,
|
||||
)
|
||||
parser.extract_reasoning("ab</think>c", request)
|
||||
|
||||
assert parser.count_reasoning_tokens(token_ids) == 2
|
||||
|
||||
|
||||
def _make_delegating_request():
|
||||
req = MagicMock(spec=ChatCompletionRequest)
|
||||
req.tools = []
|
||||
|
||||
@@ -163,9 +163,28 @@ class TestHoldbackTextRecovery:
|
||||
assert len(result) == 0
|
||||
|
||||
flushed = scanner.flush_pending()
|
||||
assert len(flushed) == 1
|
||||
assert len(flushed) == 2
|
||||
assert isinstance(flushed[0], PreLexedTerminal)
|
||||
assert flushed[0].terminal == "TOOL_START"
|
||||
assert isinstance(flushed[1], TextChunk)
|
||||
assert flushed[1].text == ""
|
||||
assert flushed[1].token_count == 2
|
||||
|
||||
def test_deferred_terminal_preserves_trailing_token_count(self, tokenizer):
|
||||
tokenizer.decode.side_effect = lambda ids: {
|
||||
CHANNEL_START_ID: CHANNEL_START,
|
||||
201: "alpha",
|
||||
202: "beta",
|
||||
}[ids[0]]
|
||||
scanner = TokenIDScanner({CHANNEL_START_ID: "THINK_START"}, tokenizer)
|
||||
|
||||
assert scanner.scan("", [CHANNEL_START_ID, 201, 202]) == []
|
||||
|
||||
result = scanner.scan(f"{CHANNEL_START}alphabeta", [])
|
||||
assert isinstance(result[0], PreLexedTerminal)
|
||||
assert isinstance(result[1], TextChunk)
|
||||
assert result[1].text == "alphabeta"
|
||||
assert result[1].token_count == 2
|
||||
|
||||
def test_holdback_before_start_tag(self, scanner):
|
||||
result = scanner.scan(
|
||||
|
||||
Reference in New Issue
Block a user