[Frontend] Parse tool calls and reasoning from checkpoint response templates (#58604)

Co-authored-by: Cursor <cursoragent@cursor.com>
Co-authored-by: Bugen Zhao <i@bugenzhao.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
Signed-off-by: Yoni Gozlan <yonigozlan@users.noreply.github.com>
Signed-off-by: Bugen Zhao <i@bugenzhao.com>
This commit is contained in:
Yoni Gozlan
2026-10-02 02:33:00 +00:00
committed by GitHub
co-authored by Cursor Bugen Zhao Claude Opus 5.5
parent 5a661487e2
commit 9bd993564f
18 changed files with 2298 additions and 21 deletions
+23
View File
@@ -171,6 +171,29 @@ from HuggingFace; and you can find an example of this in a `tokenizer_config.jso
If your favorite tool-calling model is not supported, please feel free to contribute a parser & tool use chat template!
### Checkpoint Response Templates (`hf`)
Checkpoints can define a `response_template` in `tokenizer_config.json` to describe their reasoning, content, and tool-call wire format. Select the
`hf` parser to parse output from this metadata instead of a model-specific parser. Like `--tokenizer-mode hf`, it relies on the checkpoint's
Hugging Face metadata rather than model-specific code:
```bash
vllm serve <model> \
--enable-auto-tool-choice \
--tool-call-parser hf \
--reasoning-parser hf
```
Either parser can be selected on its own. Startup fails if the tokenizer has no `response_template`, or if it lacks the `thinking` or `tool_calls`
field the selected parser needs. The `hf` parser cannot be combined with other reasoning or tool call parsers.
When streaming, each tool call is emitted whole once its region parses. A call cut off before its closer, for example by a closer that is also a
stop token, is parsed from the text generated so far; a call that does not parse is dropped.
Response templates currently provide parsing, not format-specific constrained decoding. The parser therefore rejects requests for strict tools,
required or named tool choice, or `parallel_tool_calls=false`. A custom server or request chat template is allowed, but the parser still expects the
checkpoint's output format.
!!! note
With `tool_choice="auto"`, structural-tag constraints require both `VLLM_ENFORCE_STRICT_TOOL_CALLING=true` (the default) and at least one tool with `strict: true`, or a server-side floor set via `--tool-strict-level`. When these conditions are met and the selected parser supports structural tags, vLLM constrains the tool-call envelope and pins the argument schema of each tool that sets `strict: true` (or of every tool under `--tool-strict-level parameter`). Otherwise, vLLM extracts tool calls from raw text, so arguments may occasionally be malformed or violate the function's parameter schema.
@@ -246,11 +246,15 @@ async def test_batched_chat_completions_logprob_token_ids(
}
def _make_request_output(prompt_idx: int, text: str) -> RequestOutput:
def _make_request_output(
prompt_idx: int,
text: str,
prompt_token_ids: list[int] | None = None,
) -> RequestOutput:
return RequestOutput(
request_id=f"req-{prompt_idx}",
prompt=None,
prompt_token_ids=[1, 2, 3],
prompt_token_ids=prompt_token_ids or [1, 2, 3],
prompt_logprobs=None,
outputs=[
CompletionOutput(
@@ -266,8 +270,12 @@ def _make_request_output(prompt_idx: int, text: str) -> RequestOutput:
)
async def _generator(prompt_idx: int, text: str) -> AsyncGenerator[RequestOutput, None]:
yield _make_request_output(prompt_idx, text)
async def _generator(
prompt_idx: int,
text: str,
prompt_token_ids: list[int] | None = None,
) -> AsyncGenerator[RequestOutput, None]:
yield _make_request_output(prompt_idx, text, prompt_token_ids)
@pytest.mark.asyncio
@@ -402,3 +410,47 @@ def test_batch_rejects_kv_transfer_prompt_token_ids():
],
kv_transfer_params={"prompt_token_ids": [10, 20, 30]},
)
@pytest.mark.asyncio
@pytest.mark.skip_global_cleanup
async def test_batched_parser_receives_each_prompt_prefix() -> None:
prefixes = []
class RecordingParser:
def __init__(self, *args, **kwargs):
self.prompt_token_ids = None
def set_prompt_token_ids(self, prompt_token_ids):
self.prompt_token_ids = prompt_token_ids
def parse(self, model_output, **kwargs):
prefixes.append(self.prompt_token_ids)
return None, model_output, None
serving = OpenAIServingChatBatch.__new__(OpenAIServingChatBatch)
serving.response_role = "assistant"
serving.system_fingerprint = None
serving.chat_template = None
serving.chat_template_content_format = "auto"
serving.default_chat_template_kwargs = {}
request = BatchChatCompletionRequest(
model="test-model",
messages=[
[{"role": "user", "content": "first"}],
[{"role": "user", "content": "second"}],
],
)
await serving.chat_completion_full_generator_batch(
request=request,
generators=[_generator(0, "one", [1]), _generator(1, "two", [2])],
request_id="req-prefix",
model_name="test-model",
all_conversations=request.messages,
tokenizer=object(),
request_metadata=RequestResponseMetadata(request_id="req-prefix"),
parser_cls=RecordingParser,
)
assert prefixes == [[1], [2]]
@@ -790,6 +790,64 @@ async def test_chat_per_request_metrics_suppressed_for_n_greater_than_one():
assert response.metrics is None
@pytest.mark.asyncio
async def test_chat_full_generator_gives_each_choice_parser_the_prompt():
prompts: list[list[int] | None] = []
class RecordingParser:
tool_parser_cls = None
def __init__(self, *args, **kwargs):
self.prompt_token_ids = None
def set_prompt_token_ids(self, prompt_token_ids):
self.prompt_token_ids = list(prompt_token_ids)
def parse(self, model_output, request, **kwargs):
prompts.append(self.prompt_token_ids)
return None, model_output, None
def count_reasoning_tokens(self, token_ids):
return 0
serving = _build_minimal_metrics_serving_chat(enable_per_request_metrics=False)
serving.parser_cls = RecordingParser
serving.model_config = None
serving.chat_template = None
serving.chat_template_content_format = "auto"
serving.default_chat_template_kwargs = {}
request_output = _make_metrics_request_output()
request_output.prompt_token_ids = [7, 8]
request_output.outputs.append(
CompletionOutput(
index=1,
text="Hi",
token_ids=[102],
cumulative_logprob=None,
logprobs=None,
finish_reason="stop",
)
)
await serving.chat_completion_full_generator(
ChatCompletionRequest(
model="test-model",
messages=[{"role": "user", "content": "Test prompt"}],
max_tokens=10,
n=2,
),
_single_request_output(request_output),
"chatcmpl-test-id",
"test-model",
conversation=[{"role": "user", "content": "Test"}],
tokenizer=MagicMock(),
request_metadata=RequestResponseMetadata(request_id="chatcmpl-test-id"),
parser=RecordingParser(),
)
assert prompts == [[7, 8], [7, 8]]
@pytest.mark.asyncio
async def test_chat_streaming_metrics_ride_on_usage_chunk():
serving = _build_minimal_metrics_serving_chat(enable_per_request_metrics=True)
@@ -1608,6 +1666,21 @@ async def test_serving_chat_truncation_side_controls_prompt_truncation():
assert left_token_ids == full_token_ids[-4:]
def test_serving_chat_starts_without_tokenizer():
"""Regression test: building the serving objects must not require a
tokenizer, which is absent under `skip_tokenizer_init`."""
mock_engine = _build_mock_engine()
mock_engine.model_config = MockModelConfig(skip_tokenizer_init=True)
mock_engine.renderer = HfRenderer(
MockVllmConfig(mock_engine.model_config, parallel_config=MockParallelConfig()),
None,
)
serving_chat = _build_serving_chat(mock_engine)
assert serving_chat.parser_cls is None
@pytest.mark.asyncio
async def test_serving_chat_mistral_token_ids_prompt_is_validated():
"""Regression test: when the Mistral tokenizer path returns token IDs
@@ -296,6 +296,45 @@ def parsed_derenderer(tokenizer):
return dr
def test_non_streaming_derender_initializes_parser_prefix(
derenderer, tokenizer, monkeypatch
):
from unittest.mock import MagicMock
from vllm.entrypoints.generate.base.protocol import FunctionCall
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
parser = MagicMock()
parser.parse.return_value = (
None,
None,
[FunctionCall(name="tool", arguments="{}")],
)
monkeypatch.setattr(derenderer, "parser", MagicMock(return_value=parser))
generated_ids = tokenizer.encode("answer", add_special_tokens=False)
response = GenerateResponse(
request_id="test",
prompt_token_ids=[11, 12],
choices=[
GenerateResponseChoice(
index=0,
token_ids=generated_ids,
finish_reason="length",
)
],
)
request = ChatCompletionRequest(
model=MODEL_NAME,
messages=[{"role": "user", "content": "question"}],
)
choices = derenderer._derender_chat(response, request)
parser.set_prompt_token_ids.assert_called_once_with([11, 12])
assert choices[0].message.tool_calls[0].function.name == "tool"
assert choices[0].finish_reason == "length"
class TestDetokenizeDelta:
"""_detokenize_delta: chunked decode must equal one shot decode."""
File diff suppressed because it is too large Load Diff
+11
View File
@@ -22,6 +22,7 @@ from vllm.entrypoints.chat_utils import (
validate_chat_template,
)
from vllm.entrypoints.openai.models.protocol import LoRAModulePath
from vllm.logger import init_logger
from vllm.tool_parsers import ToolParserManager
from vllm.tool_parsers.tool_strict_level import ToolStrictLevelName
from vllm.utils.argparse_utils import FlexibleArgumentParser
@@ -31,6 +32,8 @@ from .utils.constants import (
H11_MAX_INCOMPLETE_EVENT_SIZE_DEFAULT,
)
logger = init_logger(__name__)
class LoRAParserAction(argparse.Action):
def __call__(
@@ -466,6 +469,14 @@ def validate_parsed_serve_args(args: argparse.Namespace):
# Ensure that the chat template is valid; raises if it likely isn't
validate_chat_template(args.chat_template)
if args.chat_template is not None and "hf" in (
args.tool_call_parser,
getattr(args, "reasoning_parser", None),
):
logger.warning(
"--chat-template is set; the hf parser still expects "
"the checkpoint's output format."
)
# Enable auto tool needs a tool call parser to be valid
if args.enable_auto_tool_choice and not args.tool_call_parser:
@@ -129,17 +129,6 @@ class OpenAIServingChatBatch(OpenAIServingChat):
return render_result
single_requests, all_conversations, engine_prompts = render_result
parser: Parser | None = None
if self.parser_cls is not None:
chat_template_kwargs = self._effective_chat_template_kwargs(
single_requests[0]
)
parser = self.parser_cls(
tokenizer,
None, # tools
chat_template_kwargs=chat_template_kwargs,
)
request_id = (
f"chatcmpl-{self._base_request_id(raw_request, request.request_id)}"
)
@@ -202,7 +191,7 @@ class OpenAIServingChatBatch(OpenAIServingChat):
all_conversations,
tokenizer,
request_metadata,
parser,
self.parser_cls,
)
async def chat_completion_full_generator_batch(
@@ -214,7 +203,7 @@ class OpenAIServingChatBatch(OpenAIServingChat):
all_conversations: list[list[ConversationMessage]],
tokenizer: TokenizerLike,
request_metadata: RequestResponseMetadata,
parser: Parser | None = None,
parser_cls: type[Parser] | None = None,
) -> ErrorResponse | ChatCompletionResponse:
"""Handle batched (non-streaming) chat completions.
@@ -273,10 +262,21 @@ class OpenAIServingChatBatch(OpenAIServingChat):
else:
logprobs = None
if parser is not None:
if parser_cls is not None:
single_request = request.to_chat_completion_request(
request.messages[prompt_idx]
)
parser = parser_cls(
tokenizer,
None,
chat_template_kwargs=self._effective_chat_template_kwargs(
single_request
),
)
parser.set_prompt_token_ids(final_res.prompt_token_ids)
reasoning, content, _ = parser.parse(
output.text,
request=request, # type: ignore[arg-type]
request=single_request,
model_output_token_ids=output.token_ids,
)
if not request.include_reasoning:
@@ -165,6 +165,7 @@ class OpenAIServingChat(GenerateBaseServing):
tool_strict_level=tool_strict_level,
model_name=self.model_config.model,
is_harmony=self.model_config.hf_config.model_type == "gpt_oss",
tokenizer=self.renderer.tokenizer,
)
self.exclude_tools_when_tool_choice_none = exclude_tools_when_tool_choice_none
@@ -974,6 +975,8 @@ class OpenAIServingChat(GenerateBaseServing):
parser = self._make_parser(
request, tokenizer, self._effective_chat_template_kwargs(request)
)
if parser is not None and final_res.prompt_token_ids is not None:
parser.set_prompt_token_ids(final_res.prompt_token_ids)
token_ids = output.token_ids
out_logprobs = output.logprobs
+16 -1
View File
@@ -141,6 +141,7 @@ class OpenAIServingResponses(GenerateBaseServing):
tool_strict_level=tool_strict_level,
model_name=self.model_config.model,
is_harmony=self.model_config.hf_config.model_type == "gpt_oss",
tokenizer=self.renderer.tokenizer,
)
self.enable_prompt_tokens_details = enable_prompt_tokens_details
self.enable_force_include_usage = enable_force_include_usage
@@ -446,6 +447,12 @@ class OpenAIServingResponses(GenerateBaseServing):
response_parser = self._make_response_parser(
request, tokenizer, chat_template_kwargs
)
if response_parser is not None:
prompt_token_ids = self._extract_prompt_components(
engine_input
).token_ids
if prompt_token_ids is not None:
response_parser.set_prompt_token_ids(prompt_token_ids)
context: ConversationContext
function_tool_names = extract_function_tool_names(request.tools)
@@ -703,6 +710,12 @@ class OpenAIServingResponses(GenerateBaseServing):
context.request,
context.response_messages,
)
if context.response_parser is not None:
prompt_token_ids = self._extract_prompt_components(
engine_input
).token_ids
if prompt_token_ids is not None:
context.response_parser.set_prompt_token_ids(prompt_token_ids)
sampling_params.max_tokens = get_max_tokens(
max_model_len,
@@ -1187,7 +1200,9 @@ class OpenAIServingResponses(GenerateBaseServing):
) -> AsyncGenerator[StreamingResponsesResponse, None]:
processor = SimpleStreamingEventProcessor(tools=request.tools)
hide_stream_metadata = not request.include_reasoning and self.parser is not None
hide_stream_metadata = (
not request.include_reasoning and context.response_parser is not None
)
def _get_logprobs(output: CompletionOutput) -> list[Logprob]:
if not request.is_include_output_logprobs():
+5
View File
@@ -115,6 +115,7 @@ class Parser:
tool_parser_cls: type[ToolParser] | None = None
# Server-side floor for tool-call structural tags (--tool-strict-level).
tool_strict_level: ToolStrictLevel = ToolStrictLevel.AUTO
always_adjust_request: bool = False
def __init__(
self,
@@ -206,6 +207,10 @@ class Parser:
"""
return request
def set_prompt_token_ids(self, prompt_token_ids: Sequence[int]) -> None:
"""Provide the exact rendered prompt to parsers that need prefix state."""
return
@abstractmethod
def is_reasoning_end(self, input_ids: list[int]) -> bool:
"""Check if the reasoning content ends in the input_ids.
+40
View File
@@ -11,10 +11,13 @@ from vllm.tool_parsers.tool_strict_level import ToolStrictLevel
if TYPE_CHECKING:
from vllm.parser.abstract_parser import Parser
from vllm.reasoning import ReasoningParser
from vllm.tokenizers import TokenizerLike
from vllm.tool_parsers import ToolParser
logger = init_logger(__name__)
HF_PARSER = "hf"
class ParserManager:
"""Provides a unified Parser by composing reasoning and tool parser adapters."""
@@ -79,6 +82,7 @@ class ParserManager:
model_name: str | None = None,
is_harmony: bool = False,
tool_strict_level: str = "auto",
tokenizer: TokenizerLike | None = None,
) -> type[Parser] | None:
"""Get a Parser that handles both reasoning and tool parsing.
@@ -93,6 +97,8 @@ class ParserManager:
If True, HarmonyParser is always returned.
tool_strict_level: Server-side floor for tool-call structural
tags (``--tool-strict-level``).
tokenizer: Tokenizer whose `response_template` metadata is
validated when the `hf` parser is selected.
Returns:
A Parser class, or None if neither parser is specified.
@@ -119,6 +125,40 @@ class ParserManager:
HarmonyParser.tool_strict_level = strict_level
return HarmonyParser
if HF_PARSER in (reasoning_parser_name, tool_parser_name):
if {reasoning_parser_name, tool_parser_name} - {
HF_PARSER,
None,
"",
}:
raise TypeError(
"The hf parser cannot be combined with other "
"reasoning or tool call parsers"
)
from vllm.parser.response_template import (
ResponseTemplateParser,
validate_tokenizer_response_template,
)
if tokenizer is not None:
validate_tokenizer_response_template(
tokenizer,
reasoning=reasoning_parser_cls is not None,
tools=tool_parser_cls is not None,
)
r_cls = reasoning_parser_cls
t_cls = tool_parser_cls
auto_tools = enable_auto_tools
class _ResponseTemplateParser(ResponseTemplateParser):
reasoning_parser_cls = r_cls
tool_parser_cls = t_cls
tool_strict_level = strict_level
_enable_auto_tools = auto_tools
return _ResponseTemplateParser
if reasoning_parser_name == "kimi_k3" or tool_parser_name == "kimi_k3":
from vllm.parser.kimi_k3 import KimiK3Parser
+642
View File
@@ -0,0 +1,642 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Unified vLLM parser backed by checkpoint `response_template` metadata."""
from __future__ import annotations
import copy
import json
from collections.abc import Callable, Iterable, Sequence
from typing import TYPE_CHECKING, Any, cast
from openai.types.responses import ToolChoiceFunction
from vllm.entrypoints.chat_utils import make_tool_call_id
from vllm.entrypoints.generate.base.protocol import (
DeltaFunctionCall,
DeltaMessage,
DeltaToolCall,
FunctionCall,
)
from vllm.entrypoints.openai.chat_completion.protocol import (
ChatCompletionNamedToolChoiceParam,
)
from vllm.exceptions import VLLMValidationError
from vllm.logger import init_logger
from vllm.parser.abstract_parser import Parser
from vllm.parser.chat_parsing import ResponseParser
from vllm.parser.chat_parsing.response_templates import (
ResponseTemplate,
load_response_template,
)
from vllm.reasoning.abs_reasoning_parsers import ReasoningParser
from vllm.tool_parsers.abstract_tool_parser import ToolParser
from vllm.tool_parsers.utils import iter_response_function_tool_dicts
if TYPE_CHECKING:
from vllm.entrypoints.openai.chat_completion.protocol import (
ChatCompletionRequest,
)
from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
from vllm.tokenizers import TokenizerLike
from vllm.tool_parsers.abstract_tool_parser import Tool
logger = init_logger(__name__)
THINKING_FIELD = "thinking"
CONTENT_FIELD = "content"
TOOL_FIELD = "tool_calls"
_SUPPORTED_FIELDS = frozenset({THINKING_FIELD, CONTENT_FIELD, TOOL_FIELD})
def _tool_dict(tool: Any) -> dict[str, Any]:
if hasattr(tool, "model_dump"):
tool = tool.model_dump(exclude_none=True)
return tool if isinstance(tool, dict) else {}
def _as_tool_dicts(tools: Sequence[Any] | None) -> list[dict[str, Any]]:
"""Responses function tools with flattened namespaces, plus chat tools."""
tools = list(tools or ())
result = iter_response_function_tool_dicts(tools)
result.extend(
tool_dict
for tool_dict in map(_tool_dict, tools)
if isinstance(tool_dict.get("function"), dict)
)
return result
def _tool_is_strict(tool: Any) -> bool:
tool_dict = _tool_dict(tool)
function = tool_dict.get("function")
return bool(
tool_dict.get("strict")
or (isinstance(function, dict) and function.get("strict"))
)
def _decode(tokenizer: TokenizerLike, token_ids: Sequence[int]) -> str:
try:
return cast(Any, tokenizer.decode)(
token_ids,
skip_special_tokens=False,
clean_up_tokenization_spaces=False,
spaces_between_special_tokens=False,
)
except TypeError:
return tokenizer.decode(token_ids, skip_special_tokens=False)
def resolve_response_template(
tokenizer: Any | None,
template: dict[str, Any] | None = None,
) -> dict[str, Any] | None:
"""Return `template` if given, otherwise the tokenizer's metadata."""
if template is not None or tokenizer is None:
return template
template = getattr(tokenizer, "response_template", None)
if not isinstance(template, dict):
init_kwargs = getattr(tokenizer, "init_kwargs", None) or {}
template = init_kwargs.get("response_template")
return template if isinstance(template, dict) else None
def validate_response_template_for_serving(
template: dict[str, Any],
) -> ResponseTemplate:
"""Validate the template and reject semantic fields serving cannot route."""
loaded = load_response_template(template)
unsupported = (set(loaded.fields) - _SUPPORTED_FIELDS) | (
set(loaded.defaults) - {"role"}
)
if unsupported:
raise ValueError(
"response_template contains unsupported semantic fields: "
f"{sorted(unsupported)}. Supported fields are: "
f"{sorted(_SUPPORTED_FIELDS)}"
)
for name in (THINKING_FIELD, CONTENT_FIELD):
field = loaded.fields.get(name)
if field is None:
continue
if (
field.content != "text"
or field.transform is not None
or field.join is not None
):
raise ValueError(
f"response_template field {name!r} uses semantics that cannot "
"be streamed by the OpenAI serving adapter"
)
return loaded
def validate_tokenizer_response_template(
tokenizer: Any,
*,
reasoning: bool,
tools: bool,
) -> None:
"""Fail at startup if the tokenizer cannot serve the requested parsing."""
template = resolve_response_template(tokenizer)
if template is None:
raise TypeError(
"The hf parser requires `response_template` metadata "
"in the tokenizer configuration"
)
try:
fields = validate_response_template_for_serving(template).fields
except (TypeError, ValueError) as exc:
raise TypeError(f"Invalid response_template metadata: {exc}") from exc
if reasoning and THINKING_FIELD not in fields:
raise TypeError(
"--reasoning-parser hf requires a response_template with a thinking field"
)
if tools and TOOL_FIELD not in fields:
raise TypeError(
"--tool-call-parser hf requires a response_template with a tool_calls field"
)
def _streaming_template(template: dict[str, Any]) -> ResponseTemplate:
result = copy.deepcopy(template)
fields = result["fields"]
if CONTENT_FIELD not in fields and all(
"open" in field or "open_pattern" in field for field in fields.values()
):
fields[CONTENT_FIELD] = {}
for field in fields.values():
field["optional"] = True
return load_response_template(result)
def _is_named_tool_choice(request: ChatCompletionRequest | ResponsesRequest) -> bool:
return isinstance(
request.tool_choice, (ToolChoiceFunction, ChatCompletionNamedToolChoiceParam)
)
def _tool_parsing_enabled(
request: ChatCompletionRequest | ResponsesRequest,
*,
enable_auto_tools: bool,
) -> bool:
if request.tool_choice == "none":
return False
if request.tool_choice == "required" or _is_named_tool_choice(request):
return True
return enable_auto_tools and request.tool_choice in (None, "auto")
def _unsupported_tool_guarantee(
request: ChatCompletionRequest | ResponsesRequest,
) -> tuple[str, str] | None:
"""The first requested tool guarantee this parser cannot enforce, with the
request parameter that asked for it."""
if any(_tool_is_strict(tool) for tool in request.tools or ()):
return "strict tools", "tools"
if request.tool_choice == "required":
return "required tool choice", "tool_choice"
if _is_named_tool_choice(request):
return "named tool choice", "tool_choice"
if getattr(request, "parallel_tool_calls", True) is False:
return "parallel_tool_calls=False", "parallel_tool_calls"
return None
def _reasoning_has_ended(
parser: ResponseParser,
template: ResponseTemplate,
*,
thinking_disabled: bool = False,
) -> bool:
thinking = template.fields.get(THINKING_FIELD)
if thinking is None:
return True
if thinking.repeats:
return False
active_field: str | None = None
thinking_closed = False
for event in parser.initial_events:
if event["type"] == "region_open":
active_field = event["field"]
elif event["type"] in {"region_close", "region_malformed"}:
thinking_closed |= event["field"] == THINKING_FIELD
active_field = None
if thinking_closed:
return True
if active_field in (CONTENT_FIELD, TOOL_FIELD):
return True
# Resolve delimiters held at the prompt boundary. A NUL cannot occur in
# the model wire format and any implicit chunk containing it is a probe,
# not evidence that content generation has started.
for event in parser.feed("\0"):
if (
event["type"] == "region_open"
and event["field"] in (CONTENT_FIELD, TOOL_FIELD)
and event["end"] > event["start"]
):
return True
if event["field"] == THINKING_FIELD:
if event["type"] == "region_open":
return False
if event["type"] in {"region_close", "region_malformed"}:
return True
if active_field == THINKING_FIELD:
return False
return thinking_disabled
def _literal_ids(
tokenizer: TokenizerLike, vocab: dict[str, int], literals: Sequence[str]
) -> list[list[int]] | None:
"""Token ids of each literal, or `None` when one encodes to nothing."""
ids = [
[vocab[literal]]
if literal in vocab
else tokenizer.encode(literal, add_special_tokens=False)
for literal in literals
]
return ids if all(ids) else None
def _reasoning_end_ids(
tokenizer: TokenizerLike, vocab: dict[str, int], template: ResponseTemplate
) -> list[list[int]] | None:
"""Token ids of each thinking closer, or `None` when they are not all
literal."""
thinking = template.fields.get(THINKING_FIELD)
if thinking is None or thinking.repeats or not thinking.close_literals:
return None
return _literal_ids(tokenizer, vocab, thinking.close_literals)
def _ends_within(token_ids: Sequence[int], end_ids: list[int], num_new: int) -> bool:
"""Whether `end_ids` completes within the last `num_new` of `token_ids`."""
window = list(token_ids[max(0, len(token_ids) - num_new - len(end_ids) + 1) :])
return any(
window[index : index + len(end_ids)] == end_ids
for index in range(len(window) - len(end_ids) + 1)
)
def _serving_template(
tokenizer: Any,
template: dict[str, Any] | None,
) -> ResponseTemplate:
"""The validated streaming form of `template` or the tokenizer's metadata."""
template = resolve_response_template(tokenizer, template)
if template is None:
raise ValueError(
"The hf parser requires `response_template` "
"metadata in the tokenizer configuration"
)
validate_response_template_for_serving(template)
return _streaming_template(template)
def _to_tool_calls(value: Any) -> list[tuple[str, str]]:
"""`(name, arguments)` of each call in a tool region value, or nothing if
any call is incomplete."""
values = value if isinstance(value, list) else [value]
calls: list[tuple[str, str]] = []
for item in values:
function = item.get("function") if isinstance(item, dict) else None
if not isinstance(function, dict):
return []
name = function.get("name")
arguments = function.get("arguments")
if not isinstance(name, str) or not name or arguments is None:
return []
if not isinstance(arguments, str):
arguments = json.dumps(arguments, ensure_ascii=False)
calls.append((name, arguments))
return calls
class ResponseTemplateReasoningParser(ReasoningParser):
"""Reasoning end detection for `--reasoning-parser hf`.
`ResponseTemplateParser` parses the output; this parser tells it and the
structured-output engine where reasoning ends.
"""
def __init__(
self,
tokenizer: TokenizerLike,
*args,
response_template: dict[str, Any] | None = None,
**kwargs,
) -> None:
super().__init__(tokenizer, *args, **kwargs)
self.response_template = _serving_template(tokenizer, response_template)
chat_template_kwargs = kwargs.get("chat_template_kwargs") or {}
self._thinking_disabled = not chat_template_kwargs.get("enable_thinking", True)
self._reasoning_end_ids = _reasoning_end_ids(
tokenizer, self.vocab, self.response_template
)
@property
def reasoning_start_str(self) -> str | None:
thinking = self.response_template.fields.get(THINKING_FIELD)
literals = thinking.open_literals if thinking else None
return literals[0] if literals else None
@property
def reasoning_end_str(self) -> str | None:
thinking = self.response_template.fields.get(THINKING_FIELD)
literals = thinking.close_literals if thinking else None
return literals[0] if literals else None
def is_reasoning_end(self, input_ids: Sequence[int]) -> bool:
try:
parser = ResponseParser(
self.response_template,
prefix=_decode(self.model_tokenizer, input_ids),
)
except Exception:
return False
return _reasoning_has_ended(
parser,
self.response_template,
thinking_disabled=self._thinking_disabled,
)
def is_reasoning_end_streaming(
self, input_ids: Sequence[int], delta_ids: Iterable[int]
) -> bool:
if self._reasoning_end_ids is None:
return self.is_reasoning_end(input_ids)
num_new = len(list(delta_ids))
return any(
_ends_within(input_ids, end_ids, num_new)
for end_ids in self._reasoning_end_ids
)
def extract_content_ids(self, input_ids: list[int]) -> list[int]:
raise NotImplementedError("ResponseTemplateParser extracts content")
def extract_reasoning(
self,
model_output: str,
request: ChatCompletionRequest | ResponsesRequest,
) -> tuple[str | None, str | None]:
raise NotImplementedError("ResponseTemplateParser extracts reasoning")
def extract_reasoning_streaming(
self,
previous_text: str,
current_text: str,
delta_text: str,
previous_token_ids: Sequence[int],
current_token_ids: Sequence[int],
delta_token_ids: Sequence[int],
) -> DeltaMessage | None:
raise NotImplementedError("ResponseTemplateParser extracts reasoning")
class ResponseTemplateToolParser(ToolParser):
"""Registers `--tool-call-parser hf`; `ResponseTemplateParser` extracts the
tool calls."""
class _DeltaBuilder:
"""Accumulates one `DeltaMessage`."""
def __init__(self) -> None:
self.reasoning: list[str] = []
self.content: list[str] = []
self.tool_calls: list[DeltaToolCall] = []
def build(self) -> DeltaMessage | None:
fields: dict[str, Any] = {}
if reasoning := "".join(self.reasoning):
fields["reasoning"] = reasoning
if content := "".join(self.content):
fields["content"] = content
if self.tool_calls:
fields["tool_calls"] = self.tool_calls
return DeltaMessage(**fields) if fields else None
class _ResponseStream:
"""One generation fed through `ResponseParser`, routed to message deltas."""
def __init__(
self,
template: ResponseTemplate,
*,
prefix: str,
tools: list[dict[str, Any]],
parse_reasoning: bool,
include_reasoning: bool,
parse_tools: bool,
new_tool_call_id: Callable[[str], str],
) -> None:
self.parse_reasoning = parse_reasoning
self.include_reasoning = include_reasoning
self.parse_tools = parse_tools
self.new_tool_call_id = new_tool_call_id
self.parser = ResponseParser(template, prefix=prefix, tools=tools)
self._prefix_end = len(self.parser.input_text)
self._next_tool_index = 0
self._finalized = False
def feed(self, text: str, *, finished: bool) -> DeltaMessage | None:
if self._finalized:
return None
delta = _DeltaBuilder()
self._route(self.parser.feed(text), delta)
if finished:
self._finalized = True
if len(self.parser.input_text) > self._prefix_end:
_, events = self.parser.finalize()
self._route(events, delta)
return delta.build()
def _route(self, events: Sequence[dict[str, Any]], delta: _DeltaBuilder) -> None:
for event in events:
field = event.get("field")
if field == TOOL_FIELD:
self._route_tool_event(event, delta)
elif event["type"] == "region_chunk":
if field == THINKING_FIELD and self.parse_reasoning:
if self.include_reasoning:
delta.reasoning.append(event["text"])
elif field in (THINKING_FIELD, CONTENT_FIELD):
delta.content.append(event["text"])
def _route_tool_event(self, event: dict[str, Any], delta: _DeltaBuilder) -> None:
"""Emit the calls of a tool region once it ends and parses; a region cut
off at the end of the stream is parsed as generated. With tool parsing
disabled, the region passes through as content."""
if not self.parse_tools:
if event["type"] == "region_chunk":
delta.content.append(event["text"])
else:
start = max(event["start"], self._prefix_end)
delta.content.append(self.parser.input_text[start : event["end"]])
return
if event["type"] == "region_malformed":
logger.warning("response_template: dropping a malformed tool call")
elif event["type"] == "region_close":
calls = _to_tool_calls(event.get("value"))
if not calls:
logger.warning("response_template: dropping an incomplete tool call")
for name, arguments in calls:
delta.tool_calls.append(
DeltaToolCall(
index=self._next_tool_index,
id=self.new_tool_call_id(name),
type="function",
function=DeltaFunctionCall(name=name, arguments=arguments),
)
)
self._next_tool_index += 1
class ResponseTemplateParser(Parser):
"""Parse reasoning, content, and tool calls with one response template."""
reasoning_parser_cls: type[ReasoningParser] | None = ResponseTemplateReasoningParser
tool_parser_cls: type[ToolParser] | None = ResponseTemplateToolParser
always_adjust_request = True
_enable_auto_tools = False
def __init__(
self,
tokenizer: TokenizerLike,
tools: list[Tool] | None = None,
*,
response_template: dict[str, Any] | None = None,
enable_auto_tools: bool | None = None,
**kwargs,
) -> None:
self.response_template = _serving_template(tokenizer, response_template)
super().__init__(
tokenizer, tools, response_template=response_template, **kwargs
)
if enable_auto_tools is not None:
self._enable_auto_tools = enable_auto_tools
self._tools = _as_tool_dicts(tools)
self._prompt_token_ids: list[int] | None = None
self._prefix = ""
self._stream: _ResponseStream | None = None
def set_prompt_token_ids(self, prompt_token_ids: Sequence[int]) -> None:
prompt_token_ids = list(prompt_token_ids)
if prompt_token_ids == self._prompt_token_ids:
return
self._prompt_token_ids = prompt_token_ids
self._prefix = _decode(self.model_tokenizer, prompt_token_ids)
self._stream = None
def adjust_request(
self,
request: ChatCompletionRequest | ResponsesRequest,
) -> ChatCompletionRequest | ResponsesRequest:
if getattr(request, "chat_template", None) is not None:
logger.warning_once(
"A request chat template is set; response_template parsing still "
"expects the checkpoint's output format."
)
request.skip_special_tokens = False
if hasattr(request, "spaces_between_special_tokens"):
request.spaces_between_special_tokens = False
if (
self._tool_parser is not None
and request.tools
and request.tool_choice != "none"
and (unsupported := _unsupported_tool_guarantee(request)) is not None
):
guarantee, parameter = unsupported
raise VLLMValidationError(
f"response_template parsing cannot guarantee {guarantee} "
"without a format-compatible structural grammar",
parameter=parameter,
)
return request
def is_reasoning_end(self, input_ids: list[int]) -> bool:
return (
self._reasoning_parser is None
or self._reasoning_parser.is_reasoning_end(input_ids)
)
def parse(
self,
model_output: str,
request: ChatCompletionRequest | ResponsesRequest,
enable_auto_tools: bool = False,
model_output_token_ids: Sequence[int] = (),
) -> tuple[str | None, str | None, list[FunctionCall] | None]:
self._initialize_history_tool_call_cnt(request)
stream = self._new_stream(
request,
enable_auto_tools=enable_auto_tools,
include_reasoning=True,
)
delta = stream.feed(model_output, finished=True)
if delta is None:
return None, None, None
tool_calls = [
FunctionCall(
id=call.id,
name=call.function.name or "",
arguments=call.function.arguments or "{}",
)
for call in delta.tool_calls
if call.function is not None
]
return delta.reasoning, delta.content, tool_calls or None
def parse_delta(
self,
delta_text: str,
delta_token_ids: list[int],
request: ChatCompletionRequest | ResponsesRequest,
prompt_token_ids: list[int] | None = None,
*,
finished: bool,
) -> DeltaMessage | None:
self._initialize_history_tool_call_cnt(request)
if self._prompt_token_ids is None and prompt_token_ids is not None:
self.set_prompt_token_ids(prompt_token_ids)
if self._stream is None:
self._stream = self._new_stream(
request,
enable_auto_tools=self._enable_auto_tools,
include_reasoning=request.include_reasoning,
)
return self._stream.feed(delta_text, finished=finished)
def _new_stream(
self,
request: ChatCompletionRequest | ResponsesRequest,
*,
enable_auto_tools: bool,
include_reasoning: bool,
) -> _ResponseStream:
return _ResponseStream(
self.response_template,
prefix=self._prefix,
tools=self._tools,
parse_reasoning=self._reasoning_parser is not None,
include_reasoning=include_reasoning,
parse_tools=self._tool_parser is not None
and _tool_parsing_enabled(request, enable_auto_tools=enable_auto_tools),
new_tool_call_id=self._new_tool_call_id,
)
def _new_tool_call_id(self, name: str) -> str:
state = self._stream_state
tool_call_id = make_tool_call_id(
id_type=state.tool_call_id_type,
func_name=name,
idx=state.history_tool_call_cnt,
)
state.history_tool_call_cnt += 1
return tool_call_id
+4
View File
@@ -80,6 +80,10 @@ _REASONING_PARSERS_TO_REGISTER = {
"granite_thinking_engine_reasoning_parser",
"GraniteThinkingParserReasoningAdapter",
),
"hf": (
"response_template_reasoning_parser",
"ResponseTemplateReasoningParser",
),
"holo2": (
"deepseek_v3_reasoning_parser",
"DeepSeekV3ReasoningWithThinkingParser",
@@ -0,0 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from vllm.parser.response_template import ResponseTemplateReasoningParser
__all__ = ["ResponseTemplateReasoningParser"]
+3
View File
@@ -79,6 +79,7 @@ class OnlineDerenderer:
tool_strict_level=tool_strict_level,
model_name=model_config.model,
is_harmony=self.use_harmony,
tokenizer=renderer.tokenizer,
)
self.chat_template = chat_template
@@ -160,6 +161,8 @@ class OnlineDerenderer:
chat_template_kwargs=chat_template_kwargs,
model_config=self.model_config,
)
if generate_response.prompt_token_ids is not None:
parser.set_prompt_token_ids(generate_response.prompt_token_ids)
reasoning, content, tool_calls = parser.parse(
decoded_text,
chat_request,
+3 -1
View File
@@ -183,6 +183,7 @@ class OnlineRenderer:
tool_strict_level=tool_strict_level,
model_name=model_config.model,
is_harmony=self.use_harmony,
tokenizer=renderer.tokenizer,
)
self.chat_template = chat_template
@@ -825,7 +826,8 @@ class OnlineRenderer:
and tokenizer.supports_grammar
)
should_adjust_request = (
parser.reasoning_parser_cls is not None
parser.always_adjust_request
or parser.reasoning_parser_cls is not None
or tool_choice != "none"
or is_mistral_grammar_eligible
)
+4
View File
@@ -86,6 +86,10 @@ _TOOL_PARSERS_TO_REGISTER = {
"hermes_tool_parser",
"Hermes2ProToolParser",
),
"hf": (
"response_template_tool_parser",
"ResponseTemplateToolParser",
),
"poolside_v1": (
"poolside_v1_tool_parser",
"PoolsideV1ToolParser",
@@ -0,0 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from vllm.parser.response_template import ResponseTemplateToolParser
__all__ = ["ResponseTemplateToolParser"]