[Frontend] Support count_reasoning_tokens in the Streaming Parser Engine (#45802)
Signed-off-by: chaunceyjiang <chaunceyjiang@gmail.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
5cecfc0137
commit
ac2ae8798c
@@ -28,9 +28,11 @@ from vllm.entrypoints.openai.chat_completion.protocol import (
|
|||||||
from vllm.entrypoints.openai.chat_completion.serving import (
|
from vllm.entrypoints.openai.chat_completion.serving import (
|
||||||
OpenAIServingChat,
|
OpenAIServingChat,
|
||||||
_get_mm_token_counts,
|
_get_mm_token_counts,
|
||||||
|
_make_completion_tokens_details,
|
||||||
_make_prompt_tokens_details,
|
_make_prompt_tokens_details,
|
||||||
)
|
)
|
||||||
from vllm.entrypoints.openai.engine.protocol import (
|
from vllm.entrypoints.openai.engine.protocol import (
|
||||||
|
DeltaMessage,
|
||||||
ErrorResponse,
|
ErrorResponse,
|
||||||
RequestResponseMetadata,
|
RequestResponseMetadata,
|
||||||
)
|
)
|
||||||
@@ -626,6 +628,7 @@ def _build_minimal_metrics_serving_chat(
|
|||||||
serving.response_role = "assistant"
|
serving.response_role = "assistant"
|
||||||
serving.parser_cls = None
|
serving.parser_cls = None
|
||||||
serving.enable_auto_tools = False
|
serving.enable_auto_tools = False
|
||||||
|
serving._include_reasoning_tokens_details = False
|
||||||
serving.enable_prompt_tokens_details = False
|
serving.enable_prompt_tokens_details = False
|
||||||
serving.enable_log_outputs = False
|
serving.enable_log_outputs = False
|
||||||
serving.enable_log_deltas = False
|
serving.enable_log_deltas = False
|
||||||
@@ -666,6 +669,13 @@ async def _single_request_output(
|
|||||||
yield request_output
|
yield request_output
|
||||||
|
|
||||||
|
|
||||||
|
async def _stream_request_outputs(
|
||||||
|
*request_outputs: RequestOutput,
|
||||||
|
) -> AsyncIterator[RequestOutput]:
|
||||||
|
for request_output in request_outputs:
|
||||||
|
yield request_output
|
||||||
|
|
||||||
|
|
||||||
async def _collect_metrics_stream_chunks(
|
async def _collect_metrics_stream_chunks(
|
||||||
serving: OpenAIServingChat,
|
serving: OpenAIServingChat,
|
||||||
request: ChatCompletionRequest,
|
request: ChatCompletionRequest,
|
||||||
@@ -724,6 +734,7 @@ async def test_chat_per_request_metrics_follow_server_flag():
|
|||||||
request_metadata=RequestResponseMetadata(request_id="chatcmpl-test-id"),
|
request_metadata=RequestResponseMetadata(request_id="chatcmpl-test-id"),
|
||||||
)
|
)
|
||||||
assert disabled_response.metrics is None
|
assert disabled_response.metrics is None
|
||||||
|
assert disabled_response.usage.completion_tokens_details is None
|
||||||
|
|
||||||
enabled_serving = _build_minimal_metrics_serving_chat(
|
enabled_serving = _build_minimal_metrics_serving_chat(
|
||||||
enable_per_request_metrics=True
|
enable_per_request_metrics=True
|
||||||
@@ -781,6 +792,55 @@ async def test_chat_streaming_metrics_ride_on_usage_chunk():
|
|||||||
assert usage_chunks[-1]["metrics"]["time_to_first_token_ms"] == pytest.approx(500.0)
|
assert usage_chunks[-1]["metrics"]["time_to_first_token_ms"] == pytest.approx(500.0)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_streaming_reasoning_usage_counts_across_deltas():
|
||||||
|
serving = _build_minimal_metrics_serving_chat(enable_per_request_metrics=False)
|
||||||
|
serving._include_reasoning_tokens_details = True
|
||||||
|
serving.model_config = None
|
||||||
|
|
||||||
|
parser = MagicMock()
|
||||||
|
parser.parse_delta.side_effect = [
|
||||||
|
DeltaMessage(reasoning="reasoning"),
|
||||||
|
DeltaMessage(content="answer"),
|
||||||
|
]
|
||||||
|
parser.count_reasoning_tokens.side_effect = lambda token_ids: sum(
|
||||||
|
token_id == 20 for token_id in token_ids
|
||||||
|
)
|
||||||
|
serving.parser_cls = MagicMock(return_value=parser)
|
||||||
|
|
||||||
|
first = _make_metrics_request_output(metrics=None, token_ids=(10, 20))
|
||||||
|
first.outputs[0].text = "<think>reasoning"
|
||||||
|
first.outputs[0].finish_reason = None
|
||||||
|
second = _make_metrics_request_output(metrics=None, token_ids=(11, 30))
|
||||||
|
second.outputs[0].text = "</think>answer"
|
||||||
|
|
||||||
|
request = ChatCompletionRequest(
|
||||||
|
model="test-model",
|
||||||
|
messages=[{"role": "user", "content": "Test prompt"}],
|
||||||
|
max_tokens=10,
|
||||||
|
stream=True,
|
||||||
|
stream_options={"include_usage": True},
|
||||||
|
)
|
||||||
|
chunks: list[dict[str, Any]] = []
|
||||||
|
async for line in serving.chat_completion_stream_generator(
|
||||||
|
request,
|
||||||
|
_stream_request_outputs(first, second),
|
||||||
|
"chatcmpl-test-id",
|
||||||
|
"test-model",
|
||||||
|
conversation=[{"role": "user", "content": "Test"}],
|
||||||
|
tokenizer=MagicMock(),
|
||||||
|
request_metadata=RequestResponseMetadata(request_id="chatcmpl-test-id"),
|
||||||
|
):
|
||||||
|
payload = line.removeprefix("data: ").strip()
|
||||||
|
if payload != "[DONE]":
|
||||||
|
chunks.append(json.loads(payload))
|
||||||
|
|
||||||
|
usage_chunks = [chunk for chunk in chunks if chunk.get("usage")]
|
||||||
|
assert usage_chunks[-1]["usage"]["completion_tokens_details"] == {
|
||||||
|
"reasoning_tokens": 1
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
@dataclass
|
@dataclass
|
||||||
class MockEngine:
|
class MockEngine:
|
||||||
model_config: MockModelConfig = field(default_factory=MockModelConfig)
|
model_config: MockModelConfig = field(default_factory=MockModelConfig)
|
||||||
@@ -810,6 +870,7 @@ async def _async_serving_chat_init():
|
|||||||
def test_async_serving_chat_init():
|
def test_async_serving_chat_init():
|
||||||
serving_completion = asyncio.run(_async_serving_chat_init())
|
serving_completion = asyncio.run(_async_serving_chat_init())
|
||||||
assert serving_completion.chat_template == CHAT_TEMPLATE
|
assert serving_completion.chat_template == CHAT_TEMPLATE
|
||||||
|
assert serving_completion._include_reasoning_tokens_details is False
|
||||||
|
|
||||||
|
|
||||||
def test_mm_prompt_tokens_details():
|
def test_mm_prompt_tokens_details():
|
||||||
@@ -850,6 +911,10 @@ def test_mm_prompt_tokens_details():
|
|||||||
assert details.multimodal_tokens == {"image": 600, "video": 1200}
|
assert details.multimodal_tokens == {"image": 600, "video": 1200}
|
||||||
|
|
||||||
|
|
||||||
|
def test_completion_tokens_details():
|
||||||
|
assert _make_completion_tokens_details(7).reasoning_tokens == 7
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
@pytest.mark.asyncio
|
||||||
async def test_serving_chat_returns_correct_model_name():
|
async def test_serving_chat_returns_correct_model_name():
|
||||||
mock_engine = MagicMock(spec=AsyncLLM)
|
mock_engine = MagicMock(spec=AsyncLLM)
|
||||||
|
|||||||
@@ -360,6 +360,10 @@ async def test_reasoning_tokens_counted_for_text_reasoning_model(monkeypatch):
|
|||||||
def get_vocab(self):
|
def get_vocab(self):
|
||||||
return self._vocab
|
return self._vocab
|
||||||
|
|
||||||
|
def decode(self, token_ids):
|
||||||
|
id_to_token = {v: k for k, v in self._vocab.items()}
|
||||||
|
return "".join(id_to_token.get(token_id, "x") for token_id in token_ids)
|
||||||
|
|
||||||
# Force non-harmony, SimpleContext path
|
# Force non-harmony, SimpleContext path
|
||||||
monkeypatch.setattr(envs, "VLLM_USE_EXPERIMENTAL_PARSER_CONTEXT", False)
|
monkeypatch.setattr(envs, "VLLM_USE_EXPERIMENTAL_PARSER_CONTEXT", False)
|
||||||
|
|
||||||
|
|||||||
@@ -2,6 +2,7 @@
|
|||||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||||
"""Tests for the streaming parser engine core pipeline."""
|
"""Tests for the streaming parser engine core pipeline."""
|
||||||
|
|
||||||
|
from dataclasses import replace
|
||||||
from unittest.mock import MagicMock
|
from unittest.mock import MagicMock
|
||||||
|
|
||||||
import pytest
|
import pytest
|
||||||
@@ -71,6 +72,16 @@ def _think_config() -> ParserEngineConfig:
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _token_think_config() -> ParserEngineConfig:
|
||||||
|
return replace(
|
||||||
|
_think_config(),
|
||||||
|
token_id_terminals={
|
||||||
|
"THINK_START": "<think>",
|
||||||
|
"THINK_END": "</think>",
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
class TestNonStreaming:
|
class TestNonStreaming:
|
||||||
def test_plain_text(self):
|
def test_plain_text(self):
|
||||||
engine = StreamingParserEngine(_hermes_config(), tokenizer=None)
|
engine = StreamingParserEngine(_hermes_config(), tokenizer=None)
|
||||||
@@ -441,6 +452,46 @@ class TestTokenIdFiltering:
|
|||||||
assert sum(1 for e in all_events if e.type == EventType.TOOL_CALL_END) == 1
|
assert sum(1 for e in all_events if e.type == EventType.TOOL_CALL_END) == 1
|
||||||
|
|
||||||
|
|
||||||
|
class TestReasoningTokenCounts:
|
||||||
|
def test_counts_reasoning_and_excludes_boundaries_and_content(self):
|
||||||
|
engine = StreamingParserEngine(_token_think_config(), _make_think_tokenizer())
|
||||||
|
|
||||||
|
events = engine.feed(
|
||||||
|
"<think>tok1tok2</think>tok3",
|
||||||
|
[_START_ID, 1, 2, _END_ID, 3],
|
||||||
|
)
|
||||||
|
events.extend(engine.finish())
|
||||||
|
|
||||||
|
reasoning = [e for e in events if e.type == EventType.REASONING_CHUNK]
|
||||||
|
assert "".join(e.value for e in reasoning) == "tok1tok2"
|
||||||
|
assert sum(e.token_count for e in reasoning) == 2
|
||||||
|
assert engine.reasoning_token_count == 2
|
||||||
|
|
||||||
|
def test_counts_tokens_after_final_deferred_start_terminal(self):
|
||||||
|
engine = StreamingParserEngine(_token_think_config(), _make_think_tokenizer())
|
||||||
|
|
||||||
|
assert engine.feed("", [_START_ID, 1, 2]) == []
|
||||||
|
events = engine.feed("<think>tok1tok2", [])
|
||||||
|
|
||||||
|
reasoning = [e for e in events if e.type == EventType.REASONING_CHUNK]
|
||||||
|
assert "".join(e.value for e in reasoning) == "tok1tok2"
|
||||||
|
assert sum(e.token_count for e in reasoning) == 2
|
||||||
|
assert engine.reasoning_token_count == 2
|
||||||
|
|
||||||
|
def test_deferred_reasoning_tokens_stay_before_next_end_terminal(self):
|
||||||
|
engine = StreamingParserEngine(_token_think_config(), _make_think_tokenizer())
|
||||||
|
|
||||||
|
assert engine.feed("", [_START_ID, 1, 2]) == []
|
||||||
|
events = engine.feed("<think>tok1tok2</think>tok3", [_END_ID, 3])
|
||||||
|
|
||||||
|
assert (
|
||||||
|
sum(e.token_count for e in events if e.type == EventType.REASONING_CHUNK)
|
||||||
|
== 2
|
||||||
|
)
|
||||||
|
assert sum(e.token_count for e in events if e.type == EventType.TEXT_CHUNK) == 1
|
||||||
|
assert engine.reasoning_token_count == 2
|
||||||
|
|
||||||
|
|
||||||
def _func_prefix_config() -> ParserEngineConfig:
|
def _func_prefix_config() -> ParserEngineConfig:
|
||||||
"""Config mixing token-ID terminals (TOOL_START/END) with
|
"""Config mixing token-ID terminals (TOOL_START/END) with
|
||||||
text-only terminals (FUNC_PREFIX) and fallback transitions."""
|
text-only terminals (FUNC_PREFIX) and fallback transitions."""
|
||||||
|
|||||||
@@ -33,6 +33,7 @@ from vllm.parser.engine.parser_engine_config import (
|
|||||||
ParserState,
|
ParserState,
|
||||||
Transition,
|
Transition,
|
||||||
)
|
)
|
||||||
|
from vllm.parser.parser_manager import ParserManager
|
||||||
|
|
||||||
# ── Shared test configs ──────────────────────────────────────────────
|
# ── Shared test configs ──────────────────────────────────────────────
|
||||||
|
|
||||||
@@ -892,6 +893,129 @@ class _CombinedDelegating(DelegatingParser):
|
|||||||
tool_parser_cls = _CombinedToolAdapter
|
tool_parser_cls = _CombinedToolAdapter
|
||||||
|
|
||||||
|
|
||||||
|
def test_parser_manager_uses_shared_engine_directly(monkeypatch):
|
||||||
|
monkeypatch.setattr(
|
||||||
|
ParserManager,
|
||||||
|
"get_reasoning_parser",
|
||||||
|
classmethod(lambda cls, name: _CombinedReasoningAdapter),
|
||||||
|
)
|
||||||
|
monkeypatch.setattr(
|
||||||
|
ParserManager,
|
||||||
|
"get_tool_parser",
|
||||||
|
classmethod(lambda cls, name, enabled, model: _CombinedToolAdapter),
|
||||||
|
)
|
||||||
|
|
||||||
|
parser_cls = ParserManager.get_parser(
|
||||||
|
tool_parser_name="combined",
|
||||||
|
reasoning_parser_name="combined",
|
||||||
|
enable_auto_tools=True,
|
||||||
|
)
|
||||||
|
|
||||||
|
assert parser_cls is not None
|
||||||
|
assert parser_cls is _CombinedTestEngine
|
||||||
|
parser = parser_cls(make_mock_tokenizer(_VOCAB))
|
||||||
|
request = _make_delegating_request()
|
||||||
|
reasoning, content, _ = parser.parse(
|
||||||
|
"ab</think>c",
|
||||||
|
request,
|
||||||
|
model_output_token_ids=[ord("a"), ord("b"), 201, ord("c")],
|
||||||
|
)
|
||||||
|
assert reasoning == "ab"
|
||||||
|
assert content == "c"
|
||||||
|
assert parser.count_reasoning_tokens([]) == 2
|
||||||
|
|
||||||
|
|
||||||
|
def test_parser_manager_preserves_reasoning_only_adapter(monkeypatch):
|
||||||
|
monkeypatch.setattr(
|
||||||
|
ParserManager,
|
||||||
|
"get_reasoning_parser",
|
||||||
|
classmethod(lambda cls, name: _CombinedReasoningAdapter),
|
||||||
|
)
|
||||||
|
monkeypatch.setattr(
|
||||||
|
ParserManager,
|
||||||
|
"get_tool_parser",
|
||||||
|
classmethod(lambda cls, name, enabled, model: None),
|
||||||
|
)
|
||||||
|
|
||||||
|
parser_cls = ParserManager.get_parser(reasoning_parser_name="combined")
|
||||||
|
|
||||||
|
assert parser_cls is not None
|
||||||
|
parser = parser_cls(make_mock_tokenizer(_VOCAB))
|
||||||
|
assert parser.reasoning_parser is not None
|
||||||
|
assert parser.tool_parser is None
|
||||||
|
reasoning, content, _ = parser.parse(
|
||||||
|
'ab</think><tool_call>{"name":"h","arguments":{}}</tool_call>',
|
||||||
|
_make_delegating_request(),
|
||||||
|
model_output_token_ids=[ord("a"), ord("b"), 201],
|
||||||
|
)
|
||||||
|
assert reasoning == "ab"
|
||||||
|
assert content == '<tool_call>{"name":"h","arguments":{}}</tool_call>'
|
||||||
|
assert parser.count_reasoning_tokens([ord("a"), ord("b"), 201]) == 2
|
||||||
|
|
||||||
|
|
||||||
|
def test_parser_manager_preserves_tool_only_adapter(monkeypatch):
|
||||||
|
monkeypatch.setattr(
|
||||||
|
ParserManager,
|
||||||
|
"get_reasoning_parser",
|
||||||
|
classmethod(lambda cls, name: None),
|
||||||
|
)
|
||||||
|
monkeypatch.setattr(
|
||||||
|
ParserManager,
|
||||||
|
"get_tool_parser",
|
||||||
|
classmethod(lambda cls, name, enabled, model: _CombinedToolAdapter),
|
||||||
|
)
|
||||||
|
|
||||||
|
parser_cls = ParserManager.get_parser(
|
||||||
|
tool_parser_name="combined", enable_auto_tools=True
|
||||||
|
)
|
||||||
|
|
||||||
|
assert parser_cls is not None
|
||||||
|
parser = parser_cls(make_mock_tokenizer(_VOCAB))
|
||||||
|
assert parser.reasoning_parser is None
|
||||||
|
assert parser.tool_parser is not None
|
||||||
|
reasoning, content, tool_calls = parser.parse(
|
||||||
|
"<think>ab</think>c",
|
||||||
|
_make_delegating_request(),
|
||||||
|
model_output_token_ids=[200, ord("a"), ord("b"), 201, ord("c")],
|
||||||
|
)
|
||||||
|
assert reasoning is None
|
||||||
|
assert content == "<think>ab</think>c"
|
||||||
|
assert tool_calls == []
|
||||||
|
|
||||||
|
|
||||||
|
def test_parser_manager_rejects_non_engine_adapter_metadata():
|
||||||
|
class TraditionalParser:
|
||||||
|
pass
|
||||||
|
|
||||||
|
class InvalidEngineAdapter:
|
||||||
|
_parser_engine_cls = TraditionalParser
|
||||||
|
|
||||||
|
assert ParserManager._get_parser_engine_cls(TraditionalParser) is None
|
||||||
|
assert ParserManager._get_parser_engine_cls(InvalidEngineAdapter) is None
|
||||||
|
assert (
|
||||||
|
ParserManager._get_parser_engine_cls(_CombinedReasoningAdapter)
|
||||||
|
is _CombinedTestEngine
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_reasoning_adapter_counts_after_final_non_streaming_parse():
|
||||||
|
parser = _CombinedReasoningAdapter(make_mock_tokenizer(_VOCAB))
|
||||||
|
request = _make_delegating_request()
|
||||||
|
token_ids = [ord("a"), ord("b"), 201, ord("c")]
|
||||||
|
|
||||||
|
parser.extract_reasoning_streaming(
|
||||||
|
"",
|
||||||
|
"ab</think>c",
|
||||||
|
"ab</think>c",
|
||||||
|
[],
|
||||||
|
token_ids,
|
||||||
|
token_ids,
|
||||||
|
)
|
||||||
|
parser.extract_reasoning("ab</think>c", request)
|
||||||
|
|
||||||
|
assert parser.count_reasoning_tokens(token_ids) == 2
|
||||||
|
|
||||||
|
|
||||||
def _make_delegating_request():
|
def _make_delegating_request():
|
||||||
req = MagicMock(spec=ChatCompletionRequest)
|
req = MagicMock(spec=ChatCompletionRequest)
|
||||||
req.tools = []
|
req.tools = []
|
||||||
|
|||||||
@@ -163,9 +163,28 @@ class TestHoldbackTextRecovery:
|
|||||||
assert len(result) == 0
|
assert len(result) == 0
|
||||||
|
|
||||||
flushed = scanner.flush_pending()
|
flushed = scanner.flush_pending()
|
||||||
assert len(flushed) == 1
|
assert len(flushed) == 2
|
||||||
assert isinstance(flushed[0], PreLexedTerminal)
|
assert isinstance(flushed[0], PreLexedTerminal)
|
||||||
assert flushed[0].terminal == "TOOL_START"
|
assert flushed[0].terminal == "TOOL_START"
|
||||||
|
assert isinstance(flushed[1], TextChunk)
|
||||||
|
assert flushed[1].text == ""
|
||||||
|
assert flushed[1].token_count == 2
|
||||||
|
|
||||||
|
def test_deferred_terminal_preserves_trailing_token_count(self, tokenizer):
|
||||||
|
tokenizer.decode.side_effect = lambda ids: {
|
||||||
|
CHANNEL_START_ID: CHANNEL_START,
|
||||||
|
201: "alpha",
|
||||||
|
202: "beta",
|
||||||
|
}[ids[0]]
|
||||||
|
scanner = TokenIDScanner({CHANNEL_START_ID: "THINK_START"}, tokenizer)
|
||||||
|
|
||||||
|
assert scanner.scan("", [CHANNEL_START_ID, 201, 202]) == []
|
||||||
|
|
||||||
|
result = scanner.scan(f"{CHANNEL_START}alphabeta", [])
|
||||||
|
assert isinstance(result[0], PreLexedTerminal)
|
||||||
|
assert isinstance(result[1], TextChunk)
|
||||||
|
assert result[1].text == "alphabeta"
|
||||||
|
assert result[1].token_count == 2
|
||||||
|
|
||||||
def test_holdback_before_start_tag(self, scanner):
|
def test_holdback_before_start_tag(self, scanner):
|
||||||
result = scanner.scan(
|
result = scanner.scan(
|
||||||
|
|||||||
@@ -36,6 +36,7 @@ from vllm.entrypoints.openai.chat_completion.protocol import (
|
|||||||
ChatMessage,
|
ChatMessage,
|
||||||
)
|
)
|
||||||
from vllm.entrypoints.openai.engine.protocol import (
|
from vllm.entrypoints.openai.engine.protocol import (
|
||||||
|
CompletionTokenUsageInfo,
|
||||||
DeltaMessage,
|
DeltaMessage,
|
||||||
ErrorResponse,
|
ErrorResponse,
|
||||||
FunctionCall,
|
FunctionCall,
|
||||||
@@ -105,6 +106,12 @@ def _make_prompt_tokens_details(
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _make_completion_tokens_details(
|
||||||
|
reasoning_tokens: int,
|
||||||
|
) -> CompletionTokenUsageInfo:
|
||||||
|
return CompletionTokenUsageInfo(reasoning_tokens=reasoning_tokens)
|
||||||
|
|
||||||
|
|
||||||
class OpenAIServingChat(GenerateBaseServing):
|
class OpenAIServingChat(GenerateBaseServing):
|
||||||
def __init__(
|
def __init__(
|
||||||
self,
|
self,
|
||||||
@@ -146,6 +153,7 @@ class OpenAIServingChat(GenerateBaseServing):
|
|||||||
self.enable_log_deltas = enable_log_deltas
|
self.enable_log_deltas = enable_log_deltas
|
||||||
|
|
||||||
self.enable_auto_tools: bool = enable_auto_tools
|
self.enable_auto_tools: bool = enable_auto_tools
|
||||||
|
self._include_reasoning_tokens_details = bool(reasoning_parser)
|
||||||
self.parser_cls = ParserManager.get_parser(
|
self.parser_cls = ParserManager.get_parser(
|
||||||
tool_parser_name=tool_parser,
|
tool_parser_name=tool_parser,
|
||||||
reasoning_parser_name=reasoning_parser,
|
reasoning_parser_name=reasoning_parser,
|
||||||
@@ -439,6 +447,9 @@ class OpenAIServingChat(GenerateBaseServing):
|
|||||||
# Send response for each token for each request.n (index)
|
# Send response for each token for each request.n (index)
|
||||||
num_choices = 1 if request.n is None else request.n
|
num_choices = 1 if request.n is None else request.n
|
||||||
previous_num_tokens = [0] * num_choices
|
previous_num_tokens = [0] * num_choices
|
||||||
|
# TODO: Remove once all reasoning parsers use the Parser Engine.
|
||||||
|
generated_token_ids: list[list[int]] = [[] for _ in range(num_choices)]
|
||||||
|
previous_reasoning_tokens = [0] * num_choices
|
||||||
finish_reason_sent = [False] * num_choices
|
finish_reason_sent = [False] * num_choices
|
||||||
num_prompt_tokens = 0
|
num_prompt_tokens = 0
|
||||||
num_cached_tokens = None
|
num_cached_tokens = None
|
||||||
@@ -537,6 +548,11 @@ class OpenAIServingChat(GenerateBaseServing):
|
|||||||
prompt_tokens=num_prompt_tokens,
|
prompt_tokens=num_prompt_tokens,
|
||||||
completion_tokens=0,
|
completion_tokens=0,
|
||||||
total_tokens=num_prompt_tokens,
|
total_tokens=num_prompt_tokens,
|
||||||
|
completion_tokens_details=(
|
||||||
|
_make_completion_tokens_details(0)
|
||||||
|
if self._include_reasoning_tokens_details
|
||||||
|
else None
|
||||||
|
),
|
||||||
)
|
)
|
||||||
|
|
||||||
data = chunk.model_dump_json(exclude_unset=True)
|
data = chunk.model_dump_json(exclude_unset=True)
|
||||||
@@ -573,6 +589,11 @@ class OpenAIServingChat(GenerateBaseServing):
|
|||||||
prompt_tokens=num_prompt_tokens,
|
prompt_tokens=num_prompt_tokens,
|
||||||
completion_tokens=0,
|
completion_tokens=0,
|
||||||
total_tokens=num_prompt_tokens,
|
total_tokens=num_prompt_tokens,
|
||||||
|
completion_tokens_details=(
|
||||||
|
_make_completion_tokens_details(0)
|
||||||
|
if self._include_reasoning_tokens_details
|
||||||
|
else None
|
||||||
|
),
|
||||||
)
|
)
|
||||||
|
|
||||||
data = chunk.model_dump_json(exclude_unset=True)
|
data = chunk.model_dump_json(exclude_unset=True)
|
||||||
@@ -631,6 +652,11 @@ class OpenAIServingChat(GenerateBaseServing):
|
|||||||
|
|
||||||
# set the previous values for the next iteration
|
# set the previous values for the next iteration
|
||||||
previous_num_tokens[i] += len(output.token_ids)
|
previous_num_tokens[i] += len(output.token_ids)
|
||||||
|
if parser is not None:
|
||||||
|
generated_token_ids[i].extend(output.token_ids)
|
||||||
|
previous_reasoning_tokens[i] = parser.count_reasoning_tokens(
|
||||||
|
tuple(generated_token_ids[i])
|
||||||
|
)
|
||||||
|
|
||||||
# if the message delta is None (e.g. because it was a
|
# if the message delta is None (e.g. because it was a
|
||||||
# "control token" for tool calls or the parser otherwise
|
# "control token" for tool calls or the parser otherwise
|
||||||
@@ -756,6 +782,13 @@ class OpenAIServingChat(GenerateBaseServing):
|
|||||||
prompt_tokens=num_prompt_tokens,
|
prompt_tokens=num_prompt_tokens,
|
||||||
completion_tokens=completion_tokens,
|
completion_tokens=completion_tokens,
|
||||||
total_tokens=num_prompt_tokens + completion_tokens,
|
total_tokens=num_prompt_tokens + completion_tokens,
|
||||||
|
completion_tokens_details=(
|
||||||
|
_make_completion_tokens_details(
|
||||||
|
previous_reasoning_tokens[i]
|
||||||
|
)
|
||||||
|
if self._include_reasoning_tokens_details
|
||||||
|
else None
|
||||||
|
),
|
||||||
)
|
)
|
||||||
|
|
||||||
data = chunk.model_dump_json(exclude_unset=True)
|
data = chunk.model_dump_json(exclude_unset=True)
|
||||||
@@ -769,6 +802,11 @@ class OpenAIServingChat(GenerateBaseServing):
|
|||||||
prompt_tokens=num_prompt_tokens,
|
prompt_tokens=num_prompt_tokens,
|
||||||
completion_tokens=completion_tokens,
|
completion_tokens=completion_tokens,
|
||||||
total_tokens=num_prompt_tokens + completion_tokens,
|
total_tokens=num_prompt_tokens + completion_tokens,
|
||||||
|
completion_tokens_details=_make_completion_tokens_details(
|
||||||
|
sum(previous_reasoning_tokens)
|
||||||
|
)
|
||||||
|
if self._include_reasoning_tokens_details
|
||||||
|
else None,
|
||||||
)
|
)
|
||||||
final_usage.prompt_tokens_details = _make_prompt_tokens_details(
|
final_usage.prompt_tokens_details = _make_prompt_tokens_details(
|
||||||
self.enable_prompt_tokens_details,
|
self.enable_prompt_tokens_details,
|
||||||
@@ -813,6 +851,11 @@ class OpenAIServingChat(GenerateBaseServing):
|
|||||||
prompt_tokens=num_prompt_tokens,
|
prompt_tokens=num_prompt_tokens,
|
||||||
completion_tokens=num_completion_tokens,
|
completion_tokens=num_completion_tokens,
|
||||||
total_tokens=num_prompt_tokens + num_completion_tokens,
|
total_tokens=num_prompt_tokens + num_completion_tokens,
|
||||||
|
completion_tokens_details=_make_completion_tokens_details(
|
||||||
|
sum(previous_reasoning_tokens)
|
||||||
|
)
|
||||||
|
if self._include_reasoning_tokens_details
|
||||||
|
else None,
|
||||||
)
|
)
|
||||||
|
|
||||||
# Log complete streaming response if output logging is enabled
|
# Log complete streaming response if output logging is enabled
|
||||||
@@ -871,6 +914,7 @@ class OpenAIServingChat(GenerateBaseServing):
|
|||||||
)
|
)
|
||||||
|
|
||||||
choices: list[ChatCompletionResponseChoice] = []
|
choices: list[ChatCompletionResponseChoice] = []
|
||||||
|
total_reasoning_tokens = 0
|
||||||
|
|
||||||
role = self.get_chat_request_role(request)
|
role = self.get_chat_request_role(request)
|
||||||
tool_parser_cls = (
|
tool_parser_cls = (
|
||||||
@@ -908,6 +952,7 @@ class OpenAIServingChat(GenerateBaseServing):
|
|||||||
suppress_metadata = not request.include_reasoning and parser is not None
|
suppress_metadata = not request.include_reasoning and parser is not None
|
||||||
if not request.include_reasoning:
|
if not request.include_reasoning:
|
||||||
reasoning = None
|
reasoning = None
|
||||||
|
total_reasoning_tokens += parser.count_reasoning_tokens(token_ids)
|
||||||
if suppress_metadata:
|
if suppress_metadata:
|
||||||
logprobs = None
|
logprobs = None
|
||||||
else:
|
else:
|
||||||
@@ -1058,6 +1103,11 @@ class OpenAIServingChat(GenerateBaseServing):
|
|||||||
prompt_tokens=num_prompt_tokens,
|
prompt_tokens=num_prompt_tokens,
|
||||||
completion_tokens=num_generated_tokens,
|
completion_tokens=num_generated_tokens,
|
||||||
total_tokens=num_prompt_tokens + num_generated_tokens,
|
total_tokens=num_prompt_tokens + num_generated_tokens,
|
||||||
|
completion_tokens_details=_make_completion_tokens_details(
|
||||||
|
total_reasoning_tokens
|
||||||
|
)
|
||||||
|
if self._include_reasoning_tokens_details
|
||||||
|
else None,
|
||||||
)
|
)
|
||||||
usage.prompt_tokens_details = _make_prompt_tokens_details(
|
usage.prompt_tokens_details = _make_prompt_tokens_details(
|
||||||
self.enable_prompt_tokens_details,
|
self.enable_prompt_tokens_details,
|
||||||
|
|||||||
@@ -117,11 +117,16 @@ class PromptTokenUsageInfo(OpenAIBaseModel):
|
|||||||
request has no multimodal input."""
|
request has no multimodal input."""
|
||||||
|
|
||||||
|
|
||||||
|
class CompletionTokenUsageInfo(OpenAIBaseModel):
|
||||||
|
reasoning_tokens: int = 0
|
||||||
|
|
||||||
|
|
||||||
class UsageInfo(OpenAIBaseModel):
|
class UsageInfo(OpenAIBaseModel):
|
||||||
prompt_tokens: int = 0
|
prompt_tokens: int = 0
|
||||||
total_tokens: int = 0
|
total_tokens: int = 0
|
||||||
completion_tokens: int | None = 0
|
completion_tokens: int | None = 0
|
||||||
prompt_tokens_details: PromptTokenUsageInfo | None = None
|
prompt_tokens_details: PromptTokenUsageInfo | None = None
|
||||||
|
completion_tokens_details: CompletionTokenUsageInfo | None = None
|
||||||
|
|
||||||
|
|
||||||
class PerRequestTimingMetrics(OpenAIBaseModel):
|
class PerRequestTimingMetrics(OpenAIBaseModel):
|
||||||
|
|||||||
@@ -487,6 +487,7 @@ class OpenAIServingResponses(GenerateBaseServing):
|
|||||||
response_parser=response_parser,
|
response_parser=response_parser,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
reasoning_parser_kwargs = None
|
||||||
if (
|
if (
|
||||||
context.response_parser is not None
|
context.response_parser is not None
|
||||||
and context.response_parser.reasoning_parser is not None
|
and context.response_parser.reasoning_parser is not None
|
||||||
@@ -518,9 +519,7 @@ class OpenAIServingResponses(GenerateBaseServing):
|
|||||||
priority=self._get_priority(request, raw_request),
|
priority=self._get_priority(request, raw_request),
|
||||||
trace_headers=trace_headers,
|
trace_headers=trace_headers,
|
||||||
session_id=session_id,
|
session_id=session_id,
|
||||||
reasoning_parser_kwargs=reasoning_parser_kwargs
|
reasoning_parser_kwargs=reasoning_parser_kwargs,
|
||||||
if self.parser and self.parser.reasoning_parser_cls is not None
|
|
||||||
else None,
|
|
||||||
)
|
)
|
||||||
generators.append(generator)
|
generators.append(generator)
|
||||||
|
|
||||||
@@ -873,6 +872,8 @@ class OpenAIServingResponses(GenerateBaseServing):
|
|||||||
if final_output.finish_reason == "length":
|
if final_output.finish_reason == "length":
|
||||||
status = "incomplete"
|
status = "incomplete"
|
||||||
|
|
||||||
|
# TODO: Build final response items from the accumulated streaming
|
||||||
|
# parser results instead of reparsing the complete output.
|
||||||
output = self._make_response_output_items(
|
output = self._make_response_output_items(
|
||||||
request,
|
request,
|
||||||
final_output,
|
final_output,
|
||||||
@@ -901,13 +902,10 @@ class OpenAIServingResponses(GenerateBaseServing):
|
|||||||
num_reasoning_tokens == 0
|
num_reasoning_tokens == 0
|
||||||
and isinstance(context, (SimpleContext, ParsableContext))
|
and isinstance(context, (SimpleContext, ParsableContext))
|
||||||
and context.response_parser is not None
|
and context.response_parser is not None
|
||||||
and context.response_parser.reasoning_parser is not None
|
|
||||||
):
|
):
|
||||||
accumulated = getattr(context, "_accumulated_token_ids", []) or []
|
accumulated = getattr(context, "_accumulated_token_ids", []) or []
|
||||||
num_reasoning_tokens = (
|
num_reasoning_tokens = context.response_parser.count_reasoning_tokens(
|
||||||
context.response_parser.reasoning_parser.count_reasoning_tokens(
|
accumulated
|
||||||
accumulated
|
|
||||||
)
|
|
||||||
)
|
)
|
||||||
|
|
||||||
usage = ResponseUsage(
|
usage = ResponseUsage(
|
||||||
|
|||||||
@@ -367,6 +367,10 @@ class Parser:
|
|||||||
tool call extraction via internal stream state.
|
tool call extraction via internal stream state.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
|
def count_reasoning_tokens(self, token_ids: Sequence[int]) -> int:
|
||||||
|
"""Return the number of reasoning tokens in generated token IDs."""
|
||||||
|
return 0
|
||||||
|
|
||||||
|
|
||||||
class DelegatingParser(Parser):
|
class DelegatingParser(Parser):
|
||||||
"""
|
"""
|
||||||
@@ -939,6 +943,12 @@ class DelegatingParser(Parser):
|
|||||||
|
|
||||||
return delta_message
|
return delta_message
|
||||||
|
|
||||||
|
def count_reasoning_tokens(self, token_ids: Sequence[int]) -> int:
|
||||||
|
"""Count reasoning tokens through the configured reasoning parser."""
|
||||||
|
if self._reasoning_parser is None:
|
||||||
|
return 0
|
||||||
|
return self._reasoning_parser.count_reasoning_tokens(token_ids)
|
||||||
|
|
||||||
def _flush_engine_parsers(
|
def _flush_engine_parsers(
|
||||||
self, delta_message: DeltaMessage | None
|
self, delta_message: DeltaMessage | None
|
||||||
) -> DeltaMessage | None:
|
) -> DeltaMessage | None:
|
||||||
|
|||||||
@@ -47,6 +47,11 @@ class ParserEngineReasoningAdapter(ReasoningParser):
|
|||||||
def __init__(self, tokenizer: TokenizerLike, *args, **kwargs) -> None:
|
def __init__(self, tokenizer: TokenizerLike, *args, **kwargs) -> None:
|
||||||
super().__init__(tokenizer, *args, **kwargs)
|
super().__init__(tokenizer, *args, **kwargs)
|
||||||
self._parser_engine = self._parser_engine_cls(tokenizer, **kwargs) # type: ignore[call-arg]
|
self._parser_engine = self._parser_engine_cls(tokenizer, **kwargs) # type: ignore[call-arg]
|
||||||
|
self._parser_engine_kwargs = kwargs
|
||||||
|
self._counting_parser_engine: ParserEngine | None = None
|
||||||
|
# TODO: Remove once Responses finalization reuses accumulated streaming
|
||||||
|
# parser results instead of reparsing the complete output.
|
||||||
|
self._streaming_count_valid = False
|
||||||
|
|
||||||
@contextmanager
|
@contextmanager
|
||||||
def _skip_tool_parsing(self) -> Iterator[None]:
|
def _skip_tool_parsing(self) -> Iterator[None]:
|
||||||
@@ -71,6 +76,7 @@ class ParserEngineReasoningAdapter(ReasoningParser):
|
|||||||
model_output: str,
|
model_output: str,
|
||||||
request: ChatCompletionRequest | ResponsesRequest,
|
request: ChatCompletionRequest | ResponsesRequest,
|
||||||
) -> tuple[str | None, str | None]:
|
) -> tuple[str | None, str | None]:
|
||||||
|
self._streaming_count_valid = False
|
||||||
with self._skip_tool_parsing():
|
with self._skip_tool_parsing():
|
||||||
return self._parser_engine.extract_reasoning(model_output, request)
|
return self._parser_engine.extract_reasoning(model_output, request)
|
||||||
|
|
||||||
@@ -83,6 +89,7 @@ class ParserEngineReasoningAdapter(ReasoningParser):
|
|||||||
current_token_ids: Sequence[int],
|
current_token_ids: Sequence[int],
|
||||||
delta_token_ids: Sequence[int],
|
delta_token_ids: Sequence[int],
|
||||||
) -> DeltaMessage | None:
|
) -> DeltaMessage | None:
|
||||||
|
self._streaming_count_valid = True
|
||||||
with self._skip_tool_parsing():
|
with self._skip_tool_parsing():
|
||||||
return self._parser_engine.extract_reasoning_streaming(
|
return self._parser_engine.extract_reasoning_streaming(
|
||||||
previous_text,
|
previous_text,
|
||||||
@@ -122,7 +129,18 @@ class ParserEngineReasoningAdapter(ReasoningParser):
|
|||||||
return self._parser_engine.get_streaming_fallback_content(text, request)
|
return self._parser_engine.get_streaming_fallback_content(text, request)
|
||||||
|
|
||||||
def count_reasoning_tokens(self, token_ids: Sequence[int]) -> int:
|
def count_reasoning_tokens(self, token_ids: Sequence[int]) -> int:
|
||||||
return self._parser_engine.count_reasoning_tokens(token_ids)
|
if self._streaming_count_valid:
|
||||||
|
return self._parser_engine.count_reasoning_tokens(token_ids)
|
||||||
|
if not token_ids:
|
||||||
|
return 0
|
||||||
|
if self._counting_parser_engine is None:
|
||||||
|
self._counting_parser_engine = self._parser_engine_cls(
|
||||||
|
self.model_tokenizer, **self._parser_engine_kwargs
|
||||||
|
) # type: ignore[call-arg]
|
||||||
|
self._counting_parser_engine._single_pass_parse(
|
||||||
|
self.model_tokenizer.decode(token_ids), token_ids
|
||||||
|
)
|
||||||
|
return self._counting_parser_engine.count_reasoning_tokens(token_ids)
|
||||||
|
|
||||||
|
|
||||||
class ParserEngineToolAdapter(ToolParser):
|
class ParserEngineToolAdapter(ToolParser):
|
||||||
|
|||||||
@@ -24,3 +24,4 @@ class SemanticEvent:
|
|||||||
type: EventType
|
type: EventType
|
||||||
value: str = ""
|
value: str = ""
|
||||||
tool_index: int = -1
|
tool_index: int = -1
|
||||||
|
token_count: int = 0
|
||||||
|
|||||||
@@ -24,6 +24,7 @@ class TerminalDef:
|
|||||||
class LexToken:
|
class LexToken:
|
||||||
terminal: str
|
terminal: str
|
||||||
value: str
|
value: str
|
||||||
|
token_count: int = 0
|
||||||
|
|
||||||
|
|
||||||
class LexerShape:
|
class LexerShape:
|
||||||
@@ -104,6 +105,7 @@ class IncrementalLexer:
|
|||||||
self.terminals = shape.terminals
|
self.terminals = shape.terminals
|
||||||
self.content_terminal = content_terminal
|
self.content_terminal = content_terminal
|
||||||
self.buffer = ""
|
self.buffer = ""
|
||||||
|
self._token_counts: list[int] = []
|
||||||
|
|
||||||
self._literal_strings = shape.literal_strings
|
self._literal_strings = shape.literal_strings
|
||||||
self._max_literal_len = shape.max_literal_len
|
self._max_literal_len = shape.max_literal_len
|
||||||
@@ -114,15 +116,23 @@ class IncrementalLexer:
|
|||||||
|
|
||||||
def reset(self) -> None:
|
def reset(self) -> None:
|
||||||
self.buffer = ""
|
self.buffer = ""
|
||||||
|
self._token_counts.clear()
|
||||||
|
|
||||||
def feed(self, text: str) -> list[LexToken]:
|
def feed(
|
||||||
|
self,
|
||||||
|
text: str,
|
||||||
|
token_texts: tuple[str, ...] = (),
|
||||||
|
token_count: int = 0,
|
||||||
|
) -> list[LexToken]:
|
||||||
|
char_token_counts = self._char_token_counts(text, token_texts, token_count)
|
||||||
if not self.buffer and self._has_only_literals and self._literal_first_chars:
|
if not self.buffer and self._has_only_literals and self._literal_first_chars:
|
||||||
for ch in text:
|
for ch in text:
|
||||||
if ch in self._literal_first_chars:
|
if ch in self._literal_first_chars:
|
||||||
break
|
break
|
||||||
else:
|
else:
|
||||||
return [LexToken(self.content_terminal, text)]
|
return [LexToken(self.content_terminal, text, sum(char_token_counts))]
|
||||||
self.buffer += text
|
self.buffer += text
|
||||||
|
self._token_counts.extend(char_token_counts)
|
||||||
return self._drain()
|
return self._drain()
|
||||||
|
|
||||||
def flush(self) -> list[LexToken]:
|
def flush(self) -> list[LexToken]:
|
||||||
@@ -130,10 +140,46 @@ class IncrementalLexer:
|
|||||||
if self.buffer:
|
if self.buffer:
|
||||||
tokens.extend(self._drain(final=True))
|
tokens.extend(self._drain(final=True))
|
||||||
if self.buffer:
|
if self.buffer:
|
||||||
tokens.append(LexToken(self.content_terminal, self.buffer))
|
tokens.append(
|
||||||
|
LexToken(self.content_terminal, self.buffer, sum(self._token_counts))
|
||||||
|
)
|
||||||
self.buffer = ""
|
self.buffer = ""
|
||||||
|
self._token_counts.clear()
|
||||||
return tokens
|
return tokens
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _char_token_counts(
|
||||||
|
text: str,
|
||||||
|
token_texts: tuple[str, ...],
|
||||||
|
token_count: int,
|
||||||
|
) -> list[int]:
|
||||||
|
counts = [0] * len(text)
|
||||||
|
if not text:
|
||||||
|
return counts
|
||||||
|
if token_texts:
|
||||||
|
pos = 0
|
||||||
|
assigned = 0
|
||||||
|
for token_text in token_texts:
|
||||||
|
if not token_text:
|
||||||
|
continue
|
||||||
|
found = text.find(token_text, pos)
|
||||||
|
if found < 0:
|
||||||
|
continue
|
||||||
|
counts[found] += 1
|
||||||
|
assigned += 1
|
||||||
|
pos = found + len(token_text)
|
||||||
|
missing = token_count - assigned
|
||||||
|
if missing > 0:
|
||||||
|
counts[0] += missing
|
||||||
|
elif token_count:
|
||||||
|
counts[0] = token_count
|
||||||
|
return counts
|
||||||
|
|
||||||
|
def _pop_token_count(self, length: int) -> int:
|
||||||
|
token_count = sum(self._token_counts[:length])
|
||||||
|
del self._token_counts[:length]
|
||||||
|
return token_count
|
||||||
|
|
||||||
def _drain(self, *, final: bool = False) -> list[LexToken]:
|
def _drain(self, *, final: bool = False) -> list[LexToken]:
|
||||||
tokens: list[LexToken] = []
|
tokens: list[LexToken] = []
|
||||||
first_chars = self._literal_first_chars
|
first_chars = self._literal_first_chars
|
||||||
@@ -150,8 +196,11 @@ class IncrementalLexer:
|
|||||||
has_potential = True
|
has_potential = True
|
||||||
break
|
break
|
||||||
if not has_potential:
|
if not has_potential:
|
||||||
tokens.append(LexToken(content_terminal, self.buffer))
|
tokens.append(
|
||||||
|
LexToken(content_terminal, self.buffer, sum(self._token_counts))
|
||||||
|
)
|
||||||
self.buffer = ""
|
self.buffer = ""
|
||||||
|
self._token_counts.clear()
|
||||||
break
|
break
|
||||||
|
|
||||||
best_match: tuple[str, str, int] | None = None
|
best_match: tuple[str, str, int] | None = None
|
||||||
@@ -175,7 +224,13 @@ class IncrementalLexer:
|
|||||||
longer_match = True
|
longer_match = True
|
||||||
break
|
break
|
||||||
if not longer_match:
|
if not longer_match:
|
||||||
tokens.append(LexToken(best_match[0], best_match[1]))
|
tokens.append(
|
||||||
|
LexToken(
|
||||||
|
best_match[0],
|
||||||
|
best_match[1],
|
||||||
|
self._pop_token_count(best_match[2]),
|
||||||
|
)
|
||||||
|
)
|
||||||
self.buffer = self.buffer[best_match[2] :]
|
self.buffer = self.buffer[best_match[2] :]
|
||||||
continue
|
continue
|
||||||
break
|
break
|
||||||
@@ -183,15 +238,33 @@ class IncrementalLexer:
|
|||||||
break
|
break
|
||||||
|
|
||||||
if best_match is not None:
|
if best_match is not None:
|
||||||
tokens.append(LexToken(best_match[0], best_match[1]))
|
tokens.append(
|
||||||
|
LexToken(
|
||||||
|
best_match[0],
|
||||||
|
best_match[1],
|
||||||
|
self._pop_token_count(best_match[2]),
|
||||||
|
)
|
||||||
|
)
|
||||||
self.buffer = self.buffer[best_match[2] :]
|
self.buffer = self.buffer[best_match[2] :]
|
||||||
else:
|
else:
|
||||||
content_end = self._find_content_boundary()
|
content_end = self._find_content_boundary()
|
||||||
if content_end > 0:
|
if content_end > 0:
|
||||||
tokens.append(LexToken(content_terminal, self.buffer[:content_end]))
|
tokens.append(
|
||||||
|
LexToken(
|
||||||
|
content_terminal,
|
||||||
|
self.buffer[:content_end],
|
||||||
|
self._pop_token_count(content_end),
|
||||||
|
)
|
||||||
|
)
|
||||||
self.buffer = self.buffer[content_end:]
|
self.buffer = self.buffer[content_end:]
|
||||||
else:
|
else:
|
||||||
tokens.append(LexToken(content_terminal, self.buffer[0]))
|
tokens.append(
|
||||||
|
LexToken(
|
||||||
|
content_terminal,
|
||||||
|
self.buffer[0],
|
||||||
|
self._pop_token_count(1),
|
||||||
|
)
|
||||||
|
)
|
||||||
self.buffer = self.buffer[1:]
|
self.buffer = self.buffer[1:]
|
||||||
|
|
||||||
return tokens
|
return tokens
|
||||||
|
|||||||
@@ -625,23 +625,8 @@ class ParserEngine(Parser):
|
|||||||
return None
|
return None
|
||||||
|
|
||||||
def count_reasoning_tokens(self, token_ids: Sequence[int]) -> int:
|
def count_reasoning_tokens(self, token_ids: Sequence[int]) -> int:
|
||||||
start_id = self._reasoning_start_token_id
|
"""Return reasoning tokens observed by the parser engine so far."""
|
||||||
end_id = self._reasoning_end_token_id
|
return self._engine.reasoning_token_count
|
||||||
if start_id is None or end_id is None:
|
|
||||||
return 0
|
|
||||||
count = 0
|
|
||||||
depth = 0
|
|
||||||
for token_id in token_ids:
|
|
||||||
if token_id == start_id:
|
|
||||||
depth += 1
|
|
||||||
continue
|
|
||||||
if token_id == end_id:
|
|
||||||
if depth > 0:
|
|
||||||
depth -= 1
|
|
||||||
continue
|
|
||||||
if depth > 0:
|
|
||||||
count += 1
|
|
||||||
return count
|
|
||||||
|
|
||||||
# ── Single-pass parse helper ────────────────────────────────────────
|
# ── Single-pass parse helper ────────────────────────────────────────
|
||||||
|
|
||||||
|
|||||||
@@ -167,6 +167,17 @@ class StreamingParserEngine:
|
|||||||
self.skip_tool_parsing = False
|
self.skip_tool_parsing = False
|
||||||
self.reset(initial_state=initial_state)
|
self.reset(initial_state=initial_state)
|
||||||
|
|
||||||
|
@property
|
||||||
|
def reasoning_token_count(self) -> int:
|
||||||
|
return self._reasoning_token_count
|
||||||
|
|
||||||
|
def _record_reasoning_tokens(self, events: Sequence[SemanticEvent]) -> None:
|
||||||
|
self._reasoning_token_count += sum(
|
||||||
|
event.token_count
|
||||||
|
for event in events
|
||||||
|
if event.type == EventType.REASONING_CHUNK
|
||||||
|
)
|
||||||
|
|
||||||
def _reset_args_state(self) -> None:
|
def _reset_args_state(self) -> None:
|
||||||
self._args_buffer: str = ""
|
self._args_buffer: str = ""
|
||||||
self._args_safe_end: int = 0
|
self._args_safe_end: int = 0
|
||||||
@@ -186,6 +197,7 @@ class StreamingParserEngine:
|
|||||||
)
|
)
|
||||||
self.tool_index = -1
|
self.tool_index = -1
|
||||||
self._ever_had_token_ids = False
|
self._ever_had_token_ids = False
|
||||||
|
self._reasoning_token_count = 0
|
||||||
# DO NOT reset skip_tool_parsing here — callers set it before
|
# DO NOT reset skip_tool_parsing here — callers set it before
|
||||||
# calling methods that trigger reset() (e.g. extract_reasoning),
|
# calling methods that trigger reset() (e.g. extract_reasoning),
|
||||||
# and clearing it silently breaks non-streaming tool-call-as-
|
# and clearing it silently breaks non-streaming tool-call-as-
|
||||||
@@ -193,6 +205,7 @@ class StreamingParserEngine:
|
|||||||
self._scanner.reset()
|
self._scanner.reset()
|
||||||
self._lexer.reset()
|
self._lexer.reset()
|
||||||
self._message_header_buffer = ""
|
self._message_header_buffer = ""
|
||||||
|
self._message_header_token_count = 0
|
||||||
self._in_skipped_tool_span = False
|
self._in_skipped_tool_span = False
|
||||||
self._reset_args_state()
|
self._reset_args_state()
|
||||||
|
|
||||||
@@ -218,18 +231,30 @@ class StreamingParserEngine:
|
|||||||
has_special = True
|
has_special = True
|
||||||
break
|
break
|
||||||
if not has_special:
|
if not has_special:
|
||||||
return self._emit_for_state(delta_text)
|
events = self._emit_for_state(
|
||||||
|
delta_text, token_count=len(delta_token_ids)
|
||||||
|
)
|
||||||
|
self._record_reasoning_tokens(events)
|
||||||
|
return events
|
||||||
|
|
||||||
scanner_items = self._scanner.scan(delta_text, delta_token_ids)
|
scanner_items = self._scanner.scan(delta_text, delta_token_ids)
|
||||||
|
|
||||||
if len(scanner_items) == 1 and isinstance(scanner_items[0], TextChunk):
|
if len(scanner_items) == 1 and isinstance(scanner_items[0], TextChunk):
|
||||||
lex_tokens = self._lexer.feed(scanner_items[0].text)
|
item = scanner_items[0]
|
||||||
|
lex_tokens = self._lexer.feed(item.text, item.token_texts, item.token_count)
|
||||||
if len(lex_tokens) == 1 and lex_tokens[0].terminal == CONTENT_TERMINAL:
|
if len(lex_tokens) == 1 and lex_tokens[0].terminal == CONTENT_TERMINAL:
|
||||||
text = lex_tokens[0].value
|
events = self._emit_for_state(
|
||||||
return self._emit_for_state(text)
|
lex_tokens[0].value,
|
||||||
return self._process_lex_tokens(lex_tokens)
|
token_count=lex_tokens[0].token_count,
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
events = self._process_lex_tokens(lex_tokens)
|
||||||
|
self._record_reasoning_tokens(events)
|
||||||
|
return events
|
||||||
|
|
||||||
return self._process_scanner_items(scanner_items)
|
events = self._process_scanner_items(scanner_items)
|
||||||
|
self._record_reasoning_tokens(events)
|
||||||
|
return events
|
||||||
|
|
||||||
def _process_scanner_items(
|
def _process_scanner_items(
|
||||||
self, items: Sequence[LexerInput]
|
self, items: Sequence[LexerInput]
|
||||||
@@ -240,7 +265,18 @@ class StreamingParserEngine:
|
|||||||
events.extend(self._process_lex_tokens(self._lexer.flush()))
|
events.extend(self._process_lex_tokens(self._lexer.flush()))
|
||||||
events.extend(self._on_terminal(item.terminal, item.text))
|
events.extend(self._on_terminal(item.terminal, item.text))
|
||||||
elif isinstance(item, TextChunk):
|
elif isinstance(item, TextChunk):
|
||||||
events.extend(self._process_lex_tokens(self._lexer.feed(item.text)))
|
if not item.text and item.token_count:
|
||||||
|
events.extend(
|
||||||
|
self._emit_for_state("", token_count=item.token_count)
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
events.extend(
|
||||||
|
self._process_lex_tokens(
|
||||||
|
self._lexer.feed(
|
||||||
|
item.text, item.token_texts, item.token_count
|
||||||
|
)
|
||||||
|
)
|
||||||
|
)
|
||||||
return events
|
return events
|
||||||
|
|
||||||
def finish(self) -> list[SemanticEvent]:
|
def finish(self) -> list[SemanticEvent]:
|
||||||
@@ -285,11 +321,14 @@ class StreamingParserEngine:
|
|||||||
EventType.TEXT_CHUNK,
|
EventType.TEXT_CHUNK,
|
||||||
value=self._message_header_buffer,
|
value=self._message_header_buffer,
|
||||||
tool_index=self.tool_index,
|
tool_index=self.tool_index,
|
||||||
|
token_count=self._message_header_token_count,
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
self._message_header_buffer = ""
|
self._message_header_buffer = ""
|
||||||
|
self._message_header_token_count = 0
|
||||||
self.state = ParserState.CONTENT
|
self.state = ParserState.CONTENT
|
||||||
|
|
||||||
|
self._record_reasoning_tokens(events)
|
||||||
return events
|
return events
|
||||||
|
|
||||||
def parse_complete(self, text: str) -> list[SemanticEvent]:
|
def parse_complete(self, text: str) -> list[SemanticEvent]:
|
||||||
@@ -303,9 +342,11 @@ class StreamingParserEngine:
|
|||||||
strict = self._token_id_terminal_names if self._ever_had_token_ids else None
|
strict = self._token_id_terminal_names if self._ever_had_token_ids else None
|
||||||
for tok in tokens:
|
for tok in tokens:
|
||||||
if tok.terminal == CONTENT_TERMINAL or (strict and tok.terminal in strict):
|
if tok.terminal == CONTENT_TERMINAL or (strict and tok.terminal in strict):
|
||||||
events.extend(self._on_content(tok.value))
|
events.extend(self._on_content(tok.value, tok.token_count))
|
||||||
else:
|
else:
|
||||||
events.extend(self._on_terminal(tok.terminal, tok.value))
|
events.extend(
|
||||||
|
self._on_terminal(tok.terminal, tok.value, tok.token_count)
|
||||||
|
)
|
||||||
return events
|
return events
|
||||||
|
|
||||||
_TOOL_STATES = frozenset(
|
_TOOL_STATES = frozenset(
|
||||||
@@ -317,7 +358,9 @@ class StreamingParserEngine:
|
|||||||
}
|
}
|
||||||
)
|
)
|
||||||
|
|
||||||
def _on_terminal(self, terminal: str, value: str) -> list[SemanticEvent]:
|
def _on_terminal(
|
||||||
|
self, terminal: str, value: str, token_count: int = 0
|
||||||
|
) -> list[SemanticEvent]:
|
||||||
key = (self.state, terminal)
|
key = (self.state, terminal)
|
||||||
transition = self.config.transitions.get(key)
|
transition = self.config.transitions.get(key)
|
||||||
|
|
||||||
@@ -327,7 +370,7 @@ class StreamingParserEngine:
|
|||||||
# The projected skip state may not define the wrapper closer.
|
# The projected skip state may not define the wrapper closer.
|
||||||
if self.skip_tool_parsing and terminal in self._tool_exit_terminals:
|
if self.skip_tool_parsing and terminal in self._tool_exit_terminals:
|
||||||
self._in_skipped_tool_span = False
|
self._in_skipped_tool_span = False
|
||||||
return self._emit_for_state(value)
|
return self._emit_for_state(value, token_count)
|
||||||
|
|
||||||
if self.skip_tool_parsing and terminal in self._tool_terminals:
|
if self.skip_tool_parsing and terminal in self._tool_terminals:
|
||||||
# Inkling reuses one terminal for tool, text, and reasoning exits.
|
# Inkling reuses one terminal for tool, text, and reasoning exits.
|
||||||
@@ -345,6 +388,7 @@ class StreamingParserEngine:
|
|||||||
leaving_message_header = self.state == ParserState.MESSAGE_HEADER
|
leaving_message_header = self.state == ParserState.MESSAGE_HEADER
|
||||||
if leaving_message_header:
|
if leaving_message_header:
|
||||||
self._message_header_buffer = ""
|
self._message_header_buffer = ""
|
||||||
|
self._message_header_token_count = 0
|
||||||
# A tool terminal that implicitly ends reasoning must report
|
# A tool terminal that implicitly ends reasoning must report
|
||||||
# that even from the header state, or the reasoning pass never
|
# that even from the header state, or the reasoning pass never
|
||||||
# hands the block to the tool pass.
|
# hands the block to the tool pass.
|
||||||
@@ -381,13 +425,14 @@ class StreamingParserEngine:
|
|||||||
return []
|
return []
|
||||||
|
|
||||||
if transition.skip_in_token_id_mode and self._ever_had_token_ids:
|
if transition.skip_in_token_id_mode and self._ever_had_token_ids:
|
||||||
return self._emit_for_state(value)
|
return self._emit_for_state(value, token_count)
|
||||||
|
|
||||||
return self._apply_transition(transition, value)
|
return self._apply_transition(transition, value, token_count)
|
||||||
|
|
||||||
def _emit_for_state(self, text: str) -> list[SemanticEvent]:
|
def _emit_for_state(self, text: str, token_count: int = 0) -> list[SemanticEvent]:
|
||||||
if self.state == ParserState.MESSAGE_HEADER:
|
if self.state == ParserState.MESSAGE_HEADER:
|
||||||
self._message_header_buffer += text
|
self._message_header_buffer += text
|
||||||
|
self._message_header_token_count += token_count
|
||||||
return []
|
return []
|
||||||
if self.state == ParserState.TOOL_ARGS:
|
if self.state == ParserState.TOOL_ARGS:
|
||||||
if self.config.tool_args_json:
|
if self.config.tool_args_json:
|
||||||
@@ -397,26 +442,36 @@ class StreamingParserEngine:
|
|||||||
EventType.ARG_VALUE_CHUNK,
|
EventType.ARG_VALUE_CHUNK,
|
||||||
value=text,
|
value=text,
|
||||||
tool_index=self.tool_index,
|
tool_index=self.tool_index,
|
||||||
|
token_count=token_count,
|
||||||
)
|
)
|
||||||
]
|
]
|
||||||
content_type = self.config.content_events.get(self.state)
|
content_type = self.config.content_events.get(self.state)
|
||||||
if content_type is not None:
|
if content_type is not None:
|
||||||
return [SemanticEvent(content_type, value=text, tool_index=self.tool_index)]
|
return [
|
||||||
|
SemanticEvent(
|
||||||
|
content_type,
|
||||||
|
value=text,
|
||||||
|
tool_index=self.tool_index,
|
||||||
|
token_count=token_count,
|
||||||
|
)
|
||||||
|
]
|
||||||
return []
|
return []
|
||||||
|
|
||||||
def _on_content(self, text: str) -> list[SemanticEvent]:
|
def _on_content(self, text: str, token_count: int = 0) -> list[SemanticEvent]:
|
||||||
if not text:
|
if not text:
|
||||||
return []
|
return []
|
||||||
return self._emit_for_state(text)
|
return self._emit_for_state(text, token_count)
|
||||||
|
|
||||||
def _apply_transition(
|
def _apply_transition(
|
||||||
self,
|
self,
|
||||||
transition: Transition,
|
transition: Transition,
|
||||||
value: str,
|
value: str,
|
||||||
|
token_count: int = 0,
|
||||||
) -> list[SemanticEvent]:
|
) -> list[SemanticEvent]:
|
||||||
events: list[SemanticEvent] = []
|
events: list[SemanticEvent] = []
|
||||||
previous_state = self.state
|
previous_state = self.state
|
||||||
message_header = ""
|
message_header = ""
|
||||||
|
message_header_token_count = 0
|
||||||
|
|
||||||
if (
|
if (
|
||||||
self.state == ParserState.TOOL_ARGS
|
self.state == ParserState.TOOL_ARGS
|
||||||
@@ -434,7 +489,9 @@ class StreamingParserEngine:
|
|||||||
|
|
||||||
if previous_state == ParserState.MESSAGE_HEADER:
|
if previous_state == ParserState.MESSAGE_HEADER:
|
||||||
message_header = self._message_header_buffer
|
message_header = self._message_header_buffer
|
||||||
|
message_header_token_count = self._message_header_token_count
|
||||||
self._message_header_buffer = ""
|
self._message_header_buffer = ""
|
||||||
|
self._message_header_token_count = 0
|
||||||
|
|
||||||
self.state = transition.next_state
|
self.state = transition.next_state
|
||||||
|
|
||||||
@@ -454,6 +511,12 @@ class StreamingParserEngine:
|
|||||||
event_type,
|
event_type,
|
||||||
value=event_value,
|
value=event_value,
|
||||||
tool_index=self.tool_index,
|
tool_index=self.tool_index,
|
||||||
|
token_count=(
|
||||||
|
message_header_token_count
|
||||||
|
if previous_state == ParserState.MESSAGE_HEADER
|
||||||
|
and event_type == EventType.TEXT_CHUNK
|
||||||
|
else token_count
|
||||||
|
),
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|||||||
@@ -14,6 +14,8 @@ DROP_TERMINAL = "__DROP__"
|
|||||||
@dataclass(slots=True)
|
@dataclass(slots=True)
|
||||||
class TextChunk:
|
class TextChunk:
|
||||||
text: str
|
text: str
|
||||||
|
token_texts: tuple[str, ...] = ()
|
||||||
|
token_count: int = 0
|
||||||
|
|
||||||
|
|
||||||
@dataclass(slots=True)
|
@dataclass(slots=True)
|
||||||
@@ -49,18 +51,30 @@ class TokenIDScanner:
|
|||||||
self.tokenizer = tokenizer
|
self.tokenizer = tokenizer
|
||||||
self._token_text_cache: dict[int, str] = {}
|
self._token_text_cache: dict[int, str] = {}
|
||||||
self._deferred_terminals: list[PreLexedTerminal] = []
|
self._deferred_terminals: list[PreLexedTerminal] = []
|
||||||
|
self._deferred_prefix_token_counts: list[int] = []
|
||||||
|
self._deferred_trailing_token_count = 0
|
||||||
self._deferred_post_text: str = ""
|
self._deferred_post_text: str = ""
|
||||||
|
|
||||||
def reset(self) -> None:
|
def reset(self) -> None:
|
||||||
"""Clear mutable state for reuse. Preserves the token text cache."""
|
"""Clear mutable state for reuse. Preserves the token text cache."""
|
||||||
self._deferred_terminals.clear()
|
self._deferred_terminals.clear()
|
||||||
|
self._deferred_prefix_token_counts.clear()
|
||||||
|
self._deferred_trailing_token_count = 0
|
||||||
self._deferred_post_text = ""
|
self._deferred_post_text = ""
|
||||||
|
|
||||||
def _decode_token(self, token_id: int) -> str:
|
def _decode_token(self, token_id: int) -> str:
|
||||||
if token_id not in self._token_text_cache:
|
if token_id not in self._token_text_cache:
|
||||||
self._token_text_cache[token_id] = self.tokenizer.decode([token_id])
|
token_text = self.tokenizer.decode([token_id])
|
||||||
|
self._token_text_cache[token_id] = (
|
||||||
|
token_text if isinstance(token_text, str) else ""
|
||||||
|
)
|
||||||
return self._token_text_cache[token_id]
|
return self._token_text_cache[token_id]
|
||||||
|
|
||||||
|
def _decode_tokens(self, token_ids: Sequence[int]) -> tuple[str, ...]:
|
||||||
|
if self.tokenizer is None:
|
||||||
|
return ()
|
||||||
|
return tuple(self._decode_token(tid) for tid in token_ids)
|
||||||
|
|
||||||
_EMPTY: tuple[LexerInput, ...] = ()
|
_EMPTY: tuple[LexerInput, ...] = ()
|
||||||
|
|
||||||
def scan(
|
def scan(
|
||||||
@@ -70,13 +84,23 @@ class TokenIDScanner:
|
|||||||
) -> Sequence[LexerInput]:
|
) -> Sequence[LexerInput]:
|
||||||
prefix_items: list[LexerInput] = []
|
prefix_items: list[LexerInput] = []
|
||||||
effective_text = delta_text
|
effective_text = delta_text
|
||||||
|
deferred_trailing_count = 0
|
||||||
|
|
||||||
if self._deferred_terminals:
|
if self._deferred_terminals:
|
||||||
prefix_items, effective_text = self._resolve_deferred(delta_text)
|
prefix_items, effective_text, deferred_trailing_count = (
|
||||||
|
self._resolve_deferred(delta_text, len(delta_token_ids))
|
||||||
|
)
|
||||||
|
|
||||||
if not self.token_id_to_terminal:
|
if not self.token_id_to_terminal:
|
||||||
if effective_text:
|
if effective_text:
|
||||||
prefix_items.append(TextChunk(effective_text))
|
token_texts = self._decode_tokens(delta_token_ids)
|
||||||
|
prefix_items.append(
|
||||||
|
TextChunk(
|
||||||
|
effective_text,
|
||||||
|
token_texts,
|
||||||
|
deferred_trailing_count + len(token_texts),
|
||||||
|
)
|
||||||
|
)
|
||||||
return prefix_items
|
return prefix_items
|
||||||
|
|
||||||
has_special = False
|
has_special = False
|
||||||
@@ -89,14 +113,23 @@ class TokenIDScanner:
|
|||||||
if not has_special:
|
if not has_special:
|
||||||
if effective_text:
|
if effective_text:
|
||||||
if not prefix_items:
|
if not prefix_items:
|
||||||
return [TextChunk(effective_text)]
|
token_texts = self._decode_tokens(delta_token_ids)
|
||||||
prefix_items.append(TextChunk(effective_text))
|
return [TextChunk(effective_text, token_texts, len(token_texts))]
|
||||||
|
token_texts = self._decode_tokens(delta_token_ids)
|
||||||
|
prefix_items.append(
|
||||||
|
TextChunk(
|
||||||
|
effective_text,
|
||||||
|
token_texts,
|
||||||
|
deferred_trailing_count + len(token_texts),
|
||||||
|
)
|
||||||
|
)
|
||||||
return prefix_items or self._EMPTY
|
return prefix_items or self._EMPTY
|
||||||
|
|
||||||
token_texts = [self._decode_token(tid) for tid in delta_token_ids]
|
decoded_token_texts = [self._decode_token(tid) for tid in delta_token_ids]
|
||||||
|
|
||||||
results: list[LexerInput] = []
|
results: list[LexerInput] = []
|
||||||
text_accum: list[str] = []
|
text_accum: list[str] = []
|
||||||
|
token_text_accum: list[str] = []
|
||||||
|
|
||||||
for idx, tid in enumerate(delta_token_ids):
|
for idx, tid in enumerate(delta_token_ids):
|
||||||
terminal = self.token_id_to_terminal.get(tid)
|
terminal = self.token_id_to_terminal.get(tid)
|
||||||
@@ -104,16 +137,31 @@ class TokenIDScanner:
|
|||||||
if text_accum:
|
if text_accum:
|
||||||
joined = "".join(text_accum)
|
joined = "".join(text_accum)
|
||||||
if joined:
|
if joined:
|
||||||
results.append(TextChunk(joined))
|
results.append(
|
||||||
|
TextChunk(
|
||||||
|
joined,
|
||||||
|
tuple(token_text_accum),
|
||||||
|
len(token_text_accum),
|
||||||
|
)
|
||||||
|
)
|
||||||
text_accum.clear()
|
text_accum.clear()
|
||||||
results.append(PreLexedTerminal(terminal, tid, token_texts[idx]))
|
token_text_accum.clear()
|
||||||
|
results.append(
|
||||||
|
PreLexedTerminal(terminal, tid, decoded_token_texts[idx])
|
||||||
|
)
|
||||||
else:
|
else:
|
||||||
text_accum.append(token_texts[idx])
|
text_accum.append(decoded_token_texts[idx])
|
||||||
|
token_text_accum.append(decoded_token_texts[idx])
|
||||||
|
|
||||||
if text_accum:
|
if text_accum:
|
||||||
joined = "".join(text_accum)
|
joined = "".join(text_accum)
|
||||||
if joined:
|
if joined:
|
||||||
results.append(TextChunk(joined))
|
results.append(
|
||||||
|
TextChunk(joined, tuple(token_text_accum), len(token_text_accum))
|
||||||
|
)
|
||||||
|
|
||||||
|
if deferred_trailing_count:
|
||||||
|
results.insert(0, TextChunk("", token_count=deferred_trailing_count))
|
||||||
|
|
||||||
if effective_text:
|
if effective_text:
|
||||||
results = self._recover_holdback_text(effective_text, results)
|
results = self._recover_holdback_text(effective_text, results)
|
||||||
@@ -124,9 +172,15 @@ class TokenIDScanner:
|
|||||||
# transition before the preceding text has arrived. The
|
# transition before the preceding text has arrived. The
|
||||||
# deferred terminals will be resolved against the actual
|
# deferred terminals will be resolved against the actual
|
||||||
# delta_text in a subsequent scan() or flushed by finish().
|
# delta_text in a subsequent scan() or flushed by finish().
|
||||||
|
prefix_token_count = 0
|
||||||
for r in results:
|
for r in results:
|
||||||
|
if isinstance(r, TextChunk):
|
||||||
|
prefix_token_count += r.token_count
|
||||||
if isinstance(r, PreLexedTerminal):
|
if isinstance(r, PreLexedTerminal):
|
||||||
self._deferred_terminals.append(r)
|
self._deferred_terminals.append(r)
|
||||||
|
self._deferred_prefix_token_counts.append(prefix_token_count)
|
||||||
|
prefix_token_count = 0
|
||||||
|
self._deferred_trailing_token_count += prefix_token_count
|
||||||
results = []
|
results = []
|
||||||
|
|
||||||
return prefix_items + results
|
return prefix_items + results
|
||||||
@@ -136,16 +190,30 @@ class TokenIDScanner:
|
|||||||
return []
|
return []
|
||||||
results: list[LexerInput] = []
|
results: list[LexerInput] = []
|
||||||
if self._deferred_post_text:
|
if self._deferred_post_text:
|
||||||
results.append(TextChunk(self._deferred_post_text))
|
prefix_count = (
|
||||||
|
self._deferred_prefix_token_counts[0]
|
||||||
|
if self._deferred_prefix_token_counts
|
||||||
|
else 0
|
||||||
|
)
|
||||||
|
results.append(
|
||||||
|
TextChunk(self._deferred_post_text, token_count=prefix_count)
|
||||||
|
)
|
||||||
self._deferred_post_text = ""
|
self._deferred_post_text = ""
|
||||||
results.extend(self._deferred_terminals)
|
results.extend(self._deferred_terminals)
|
||||||
|
if self._deferred_trailing_token_count:
|
||||||
|
results.append(
|
||||||
|
TextChunk("", token_count=self._deferred_trailing_token_count)
|
||||||
|
)
|
||||||
self._deferred_terminals.clear()
|
self._deferred_terminals.clear()
|
||||||
|
self._deferred_prefix_token_counts.clear()
|
||||||
|
self._deferred_trailing_token_count = 0
|
||||||
return results
|
return results
|
||||||
|
|
||||||
def _resolve_deferred(
|
def _resolve_deferred(
|
||||||
self,
|
self,
|
||||||
delta_text: str,
|
delta_text: str,
|
||||||
) -> tuple[list[LexerInput], str]:
|
current_token_count: int = 0,
|
||||||
|
) -> tuple[list[LexerInput], str, int]:
|
||||||
"""Resolve deferred terminals against new delta_text.
|
"""Resolve deferred terminals against new delta_text.
|
||||||
|
|
||||||
When a previous ``scan()`` deferred a terminal (its text hadn't
|
When a previous ``scan()`` deferred a terminal (its text hadn't
|
||||||
@@ -160,7 +228,11 @@ class TokenIDScanner:
|
|||||||
should be scanned with the current delta's token IDs.
|
should be scanned with the current delta's token IDs.
|
||||||
"""
|
"""
|
||||||
deferred = self._deferred_terminals
|
deferred = self._deferred_terminals
|
||||||
|
prefix_token_counts = self._deferred_prefix_token_counts
|
||||||
|
trailing_token_count = self._deferred_trailing_token_count
|
||||||
self._deferred_terminals = []
|
self._deferred_terminals = []
|
||||||
|
self._deferred_prefix_token_counts = []
|
||||||
|
self._deferred_trailing_token_count = 0
|
||||||
|
|
||||||
results: list[LexerInput] = []
|
results: list[LexerInput] = []
|
||||||
remaining = delta_text
|
remaining = delta_text
|
||||||
@@ -171,10 +243,13 @@ class TokenIDScanner:
|
|||||||
|
|
||||||
# Duplicate-text deferred terminals resolve left-to-right via
|
# Duplicate-text deferred terminals resolve left-to-right via
|
||||||
# find(); correct when each terminal text appears once in sequence.
|
# find(); correct when each terminal text appears once in sequence.
|
||||||
for terminal in deferred:
|
for idx, terminal in enumerate(deferred):
|
||||||
|
prefix_token_count = prefix_token_counts[idx]
|
||||||
pos = remaining.find(terminal.text)
|
pos = remaining.find(terminal.text)
|
||||||
if pos > 0:
|
if pos > 0:
|
||||||
results.append(TextChunk(remaining[:pos]))
|
results.append(
|
||||||
|
TextChunk(remaining[:pos], token_count=prefix_token_count)
|
||||||
|
)
|
||||||
results.append(terminal)
|
results.append(terminal)
|
||||||
remaining = remaining[pos + len(terminal.text) :]
|
remaining = remaining[pos + len(terminal.text) :]
|
||||||
elif pos == 0:
|
elif pos == 0:
|
||||||
@@ -185,10 +260,16 @@ class TokenIDScanner:
|
|||||||
# only the terminal provides a reliable split point.
|
# only the terminal provides a reliable split point.
|
||||||
if remaining:
|
if remaining:
|
||||||
self._deferred_post_text += remaining
|
self._deferred_post_text += remaining
|
||||||
|
prefix_token_count += current_token_count
|
||||||
remaining = ""
|
remaining = ""
|
||||||
self._deferred_terminals.append(terminal)
|
self._deferred_terminals.append(terminal)
|
||||||
|
self._deferred_prefix_token_counts.append(prefix_token_count)
|
||||||
|
|
||||||
return results, remaining
|
if self._deferred_terminals:
|
||||||
|
self._deferred_trailing_token_count = trailing_token_count
|
||||||
|
trailing_token_count = 0
|
||||||
|
|
||||||
|
return results, remaining, trailing_token_count
|
||||||
|
|
||||||
def _recover_holdback_text(
|
def _recover_holdback_text(
|
||||||
self,
|
self,
|
||||||
@@ -257,6 +338,16 @@ class TokenIDScanner:
|
|||||||
if not anchors:
|
if not anchors:
|
||||||
return [TextChunk(delta_text)]
|
return [TextChunk(delta_text)]
|
||||||
|
|
||||||
|
token_groups: list[list[str]] = [[] for _ in range(len(anchors) + 1)]
|
||||||
|
count_groups = [0] * (len(anchors) + 1)
|
||||||
|
group_idx = 0
|
||||||
|
for item in results:
|
||||||
|
if isinstance(item, PreLexedTerminal):
|
||||||
|
group_idx += 1
|
||||||
|
else:
|
||||||
|
token_groups[group_idx].extend(item.token_texts)
|
||||||
|
count_groups[group_idx] += item.token_count
|
||||||
|
|
||||||
# Resolve positions right-to-left: each anchor gets the
|
# Resolve positions right-to-left: each anchor gets the
|
||||||
# rightmost occurrence that is still before the next anchor.
|
# rightmost occurrence that is still before the next anchor.
|
||||||
positions: list[int] = [-1] * len(anchors)
|
positions: list[int] = [-1] * len(anchors)
|
||||||
@@ -274,7 +365,13 @@ class TokenIDScanner:
|
|||||||
pos = positions[i]
|
pos = positions[i]
|
||||||
if pos >= consumed:
|
if pos >= consumed:
|
||||||
if pos > consumed:
|
if pos > consumed:
|
||||||
new_results.append(TextChunk(delta_text[consumed:pos]))
|
new_results.append(
|
||||||
|
TextChunk(
|
||||||
|
delta_text[consumed:pos],
|
||||||
|
tuple(token_groups[i]),
|
||||||
|
count_groups[i],
|
||||||
|
)
|
||||||
|
)
|
||||||
new_results.append(anchor)
|
new_results.append(anchor)
|
||||||
consumed = pos + len(anchor.text)
|
consumed = pos + len(anchor.text)
|
||||||
else:
|
else:
|
||||||
@@ -291,6 +388,13 @@ class TokenIDScanner:
|
|||||||
self._deferred_post_text += delta_text[consumed:]
|
self._deferred_post_text += delta_text[consumed:]
|
||||||
consumed = len(delta_text)
|
consumed = len(delta_text)
|
||||||
self._deferred_terminals.append(anchor)
|
self._deferred_terminals.append(anchor)
|
||||||
|
self._deferred_prefix_token_counts.append(count_groups[i])
|
||||||
if consumed < len(delta_text):
|
if consumed < len(delta_text):
|
||||||
new_results.append(TextChunk(delta_text[consumed:]))
|
new_results.append(
|
||||||
|
TextChunk(
|
||||||
|
delta_text[consumed:],
|
||||||
|
tuple(token_groups[-1]),
|
||||||
|
count_groups[-1],
|
||||||
|
)
|
||||||
|
)
|
||||||
return new_results
|
return new_results
|
||||||
|
|||||||
@@ -9,6 +9,7 @@ from vllm.logger import init_logger
|
|||||||
|
|
||||||
if TYPE_CHECKING:
|
if TYPE_CHECKING:
|
||||||
from vllm.parser.abstract_parser import Parser
|
from vllm.parser.abstract_parser import Parser
|
||||||
|
from vllm.parser.engine.parser_engine import ParserEngine
|
||||||
from vllm.reasoning import ReasoningParser
|
from vllm.reasoning import ReasoningParser
|
||||||
from vllm.tool_parsers import ToolParser
|
from vllm.tool_parsers import ToolParser
|
||||||
|
|
||||||
@@ -17,10 +18,30 @@ logger = init_logger(__name__)
|
|||||||
|
|
||||||
class ParserManager:
|
class ParserManager:
|
||||||
"""
|
"""
|
||||||
Provides a unified Parser by composing individual reasoning and tool
|
Provides a unified Parser from the reasoning and tool parser registries.
|
||||||
parsers from their respective registries.
|
|
||||||
|
Parser engine adapters backed by the same engine are collapsed back into
|
||||||
|
that engine. Other parser pairs are composed through ``DelegatingParser``.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _get_parser_engine_cls(
|
||||||
|
parser_cls: type[object] | None,
|
||||||
|
) -> type[ParserEngine] | None:
|
||||||
|
if parser_cls is None:
|
||||||
|
return None
|
||||||
|
parser_engine_cls = getattr(parser_cls, "_parser_engine_cls", None)
|
||||||
|
if parser_engine_cls is None:
|
||||||
|
return None
|
||||||
|
|
||||||
|
from vllm.parser.engine.parser_engine import ParserEngine
|
||||||
|
|
||||||
|
if not isinstance(parser_engine_cls, type) or not issubclass(
|
||||||
|
parser_engine_cls, ParserEngine
|
||||||
|
):
|
||||||
|
return None
|
||||||
|
return parser_engine_cls
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
def get_tool_parser(
|
def get_tool_parser(
|
||||||
cls,
|
cls,
|
||||||
@@ -84,8 +105,8 @@ class ParserManager:
|
|||||||
"""
|
"""
|
||||||
Get a Parser that handles both reasoning and tool parsing.
|
Get a Parser that handles both reasoning and tool parsing.
|
||||||
|
|
||||||
Composes individual reasoning and tool parsers into a single
|
Reuses a shared parser engine when possible, otherwise composes the
|
||||||
DelegatingParser subclass.
|
individual parsers into a ``DelegatingParser`` subclass.
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
tool_parser_name: The name of the tool parser.
|
tool_parser_name: The name of the tool parser.
|
||||||
@@ -116,6 +137,11 @@ class ParserManager:
|
|||||||
HarmonyParser.tool_parser_cls = tool_parser_cls
|
HarmonyParser.tool_parser_cls = tool_parser_cls
|
||||||
return HarmonyParser
|
return HarmonyParser
|
||||||
|
|
||||||
|
reasoning_engine_cls = cls._get_parser_engine_cls(reasoning_parser_cls)
|
||||||
|
tool_engine_cls = cls._get_parser_engine_cls(tool_parser_cls)
|
||||||
|
if reasoning_engine_cls is not None and reasoning_engine_cls is tool_engine_cls:
|
||||||
|
return reasoning_engine_cls
|
||||||
|
|
||||||
if reasoning_parser_name == "kimi_k3" or tool_parser_name == "kimi_k3":
|
if reasoning_parser_name == "kimi_k3" or tool_parser_name == "kimi_k3":
|
||||||
from vllm.parser.kimi_k3 import KimiK3Parser
|
from vllm.parser.kimi_k3 import KimiK3Parser
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user