diff --git a/docs/features/tool_calling.md b/docs/features/tool_calling.md index bbbc4d6bcd..8dccd3a8b8 100644 --- a/docs/features/tool_calling.md +++ b/docs/features/tool_calling.md @@ -171,6 +171,29 @@ from HuggingFace; and you can find an example of this in a `tokenizer_config.jso If your favorite tool-calling model is not supported, please feel free to contribute a parser & tool use chat template! +### Checkpoint Response Templates (`hf`) + +Checkpoints can define a `response_template` in `tokenizer_config.json` to describe their reasoning, content, and tool-call wire format. Select the +`hf` parser to parse output from this metadata instead of a model-specific parser. Like `--tokenizer-mode hf`, it relies on the checkpoint's +Hugging Face metadata rather than model-specific code: + +```bash +vllm serve \ + --enable-auto-tool-choice \ + --tool-call-parser hf \ + --reasoning-parser hf +``` + +Either parser can be selected on its own. Startup fails if the tokenizer has no `response_template`, or if it lacks the `thinking` or `tool_calls` +field the selected parser needs. The `hf` parser cannot be combined with other reasoning or tool call parsers. + +When streaming, each tool call is emitted whole once its region parses. A call cut off before its closer, for example by a closer that is also a +stop token, is parsed from the text generated so far; a call that does not parse is dropped. + +Response templates currently provide parsing, not format-specific constrained decoding. The parser therefore rejects requests for strict tools, +required or named tool choice, or `parallel_tool_calls=false`. A custom server or request chat template is allowed, but the parser still expects the +checkpoint's output format. + !!! note With `tool_choice="auto"`, structural-tag constraints require both `VLLM_ENFORCE_STRICT_TOOL_CALLING=true` (the default) and at least one tool with `strict: true`, or a server-side floor set via `--tool-strict-level`. When these conditions are met and the selected parser supports structural tags, vLLM constrains the tool-call envelope and pins the argument schema of each tool that sets `strict: true` (or of every tool under `--tool-strict-level parameter`). Otherwise, vLLM extracts tool calls from raw text, so arguments may occasionally be malformed or violate the function's parameter schema. diff --git a/tests/entrypoints/openai/chat_completion/test_batched_chat_completions.py b/tests/entrypoints/openai/chat_completion/test_batched_chat_completions.py index a6d78c826c..946615f971 100644 --- a/tests/entrypoints/openai/chat_completion/test_batched_chat_completions.py +++ b/tests/entrypoints/openai/chat_completion/test_batched_chat_completions.py @@ -246,11 +246,15 @@ async def test_batched_chat_completions_logprob_token_ids( } -def _make_request_output(prompt_idx: int, text: str) -> RequestOutput: +def _make_request_output( + prompt_idx: int, + text: str, + prompt_token_ids: list[int] | None = None, +) -> RequestOutput: return RequestOutput( request_id=f"req-{prompt_idx}", prompt=None, - prompt_token_ids=[1, 2, 3], + prompt_token_ids=prompt_token_ids or [1, 2, 3], prompt_logprobs=None, outputs=[ CompletionOutput( @@ -266,8 +270,12 @@ def _make_request_output(prompt_idx: int, text: str) -> RequestOutput: ) -async def _generator(prompt_idx: int, text: str) -> AsyncGenerator[RequestOutput, None]: - yield _make_request_output(prompt_idx, text) +async def _generator( + prompt_idx: int, + text: str, + prompt_token_ids: list[int] | None = None, +) -> AsyncGenerator[RequestOutput, None]: + yield _make_request_output(prompt_idx, text, prompt_token_ids) @pytest.mark.asyncio @@ -402,3 +410,47 @@ def test_batch_rejects_kv_transfer_prompt_token_ids(): ], kv_transfer_params={"prompt_token_ids": [10, 20, 30]}, ) + + +@pytest.mark.asyncio +@pytest.mark.skip_global_cleanup +async def test_batched_parser_receives_each_prompt_prefix() -> None: + prefixes = [] + + class RecordingParser: + def __init__(self, *args, **kwargs): + self.prompt_token_ids = None + + def set_prompt_token_ids(self, prompt_token_ids): + self.prompt_token_ids = prompt_token_ids + + def parse(self, model_output, **kwargs): + prefixes.append(self.prompt_token_ids) + return None, model_output, None + + serving = OpenAIServingChatBatch.__new__(OpenAIServingChatBatch) + serving.response_role = "assistant" + serving.system_fingerprint = None + serving.chat_template = None + serving.chat_template_content_format = "auto" + serving.default_chat_template_kwargs = {} + + request = BatchChatCompletionRequest( + model="test-model", + messages=[ + [{"role": "user", "content": "first"}], + [{"role": "user", "content": "second"}], + ], + ) + await serving.chat_completion_full_generator_batch( + request=request, + generators=[_generator(0, "one", [1]), _generator(1, "two", [2])], + request_id="req-prefix", + model_name="test-model", + all_conversations=request.messages, + tokenizer=object(), + request_metadata=RequestResponseMetadata(request_id="req-prefix"), + parser_cls=RecordingParser, + ) + + assert prefixes == [[1], [2]] diff --git a/tests/entrypoints/openai/chat_completion/test_serving_chat.py b/tests/entrypoints/openai/chat_completion/test_serving_chat.py index 9c80f504f7..bfaa2d4ca0 100644 --- a/tests/entrypoints/openai/chat_completion/test_serving_chat.py +++ b/tests/entrypoints/openai/chat_completion/test_serving_chat.py @@ -790,6 +790,64 @@ async def test_chat_per_request_metrics_suppressed_for_n_greater_than_one(): assert response.metrics is None +@pytest.mark.asyncio +async def test_chat_full_generator_gives_each_choice_parser_the_prompt(): + prompts: list[list[int] | None] = [] + + class RecordingParser: + tool_parser_cls = None + + def __init__(self, *args, **kwargs): + self.prompt_token_ids = None + + def set_prompt_token_ids(self, prompt_token_ids): + self.prompt_token_ids = list(prompt_token_ids) + + def parse(self, model_output, request, **kwargs): + prompts.append(self.prompt_token_ids) + return None, model_output, None + + def count_reasoning_tokens(self, token_ids): + return 0 + + serving = _build_minimal_metrics_serving_chat(enable_per_request_metrics=False) + serving.parser_cls = RecordingParser + serving.model_config = None + serving.chat_template = None + serving.chat_template_content_format = "auto" + serving.default_chat_template_kwargs = {} + request_output = _make_metrics_request_output() + request_output.prompt_token_ids = [7, 8] + request_output.outputs.append( + CompletionOutput( + index=1, + text="Hi", + token_ids=[102], + cumulative_logprob=None, + logprobs=None, + finish_reason="stop", + ) + ) + + await serving.chat_completion_full_generator( + ChatCompletionRequest( + model="test-model", + messages=[{"role": "user", "content": "Test prompt"}], + max_tokens=10, + n=2, + ), + _single_request_output(request_output), + "chatcmpl-test-id", + "test-model", + conversation=[{"role": "user", "content": "Test"}], + tokenizer=MagicMock(), + request_metadata=RequestResponseMetadata(request_id="chatcmpl-test-id"), + parser=RecordingParser(), + ) + + assert prompts == [[7, 8], [7, 8]] + + @pytest.mark.asyncio async def test_chat_streaming_metrics_ride_on_usage_chunk(): serving = _build_minimal_metrics_serving_chat(enable_per_request_metrics=True) @@ -1608,6 +1666,21 @@ async def test_serving_chat_truncation_side_controls_prompt_truncation(): assert left_token_ids == full_token_ids[-4:] +def test_serving_chat_starts_without_tokenizer(): + """Regression test: building the serving objects must not require a + tokenizer, which is absent under `skip_tokenizer_init`.""" + mock_engine = _build_mock_engine() + mock_engine.model_config = MockModelConfig(skip_tokenizer_init=True) + mock_engine.renderer = HfRenderer( + MockVllmConfig(mock_engine.model_config, parallel_config=MockParallelConfig()), + None, + ) + + serving_chat = _build_serving_chat(mock_engine) + + assert serving_chat.parser_cls is None + + @pytest.mark.asyncio async def test_serving_chat_mistral_token_ids_prompt_is_validated(): """Regression test: when the Mistral tokenizer path returns token IDs diff --git a/tests/entrypoints/scale_out/derender/test_derender_stream.py b/tests/entrypoints/scale_out/derender/test_derender_stream.py index 63ee9bcad8..b1db1193a2 100644 --- a/tests/entrypoints/scale_out/derender/test_derender_stream.py +++ b/tests/entrypoints/scale_out/derender/test_derender_stream.py @@ -296,6 +296,45 @@ def parsed_derenderer(tokenizer): return dr +def test_non_streaming_derender_initializes_parser_prefix( + derenderer, tokenizer, monkeypatch +): + from unittest.mock import MagicMock + + from vllm.entrypoints.generate.base.protocol import FunctionCall + from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest + + parser = MagicMock() + parser.parse.return_value = ( + None, + None, + [FunctionCall(name="tool", arguments="{}")], + ) + monkeypatch.setattr(derenderer, "parser", MagicMock(return_value=parser)) + generated_ids = tokenizer.encode("answer", add_special_tokens=False) + response = GenerateResponse( + request_id="test", + prompt_token_ids=[11, 12], + choices=[ + GenerateResponseChoice( + index=0, + token_ids=generated_ids, + finish_reason="length", + ) + ], + ) + request = ChatCompletionRequest( + model=MODEL_NAME, + messages=[{"role": "user", "content": "question"}], + ) + + choices = derenderer._derender_chat(response, request) + + parser.set_prompt_token_ids.assert_called_once_with([11, 12]) + assert choices[0].message.tool_calls[0].function.name == "tool" + assert choices[0].finish_reason == "length" + + class TestDetokenizeDelta: """_detokenize_delta: chunked decode must equal one shot decode.""" diff --git a/tests/parser/test_response_template.py b/tests/parser/test_response_template.py new file mode 100644 index 0000000000..84b62c5512 --- /dev/null +++ b/tests/parser/test_response_template.py @@ -0,0 +1,1349 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +import json +from copy import deepcopy +from types import SimpleNamespace +from typing import Any + +import pytest +from openai.types.responses import NamespaceTool + +from vllm.entrypoints.openai.chat_completion.protocol import ( + ChatCompletionNamedToolChoiceParam, + ChatCompletionToolsParam, +) +from vllm.exceptions import VLLMValidationError +from vllm.parser import response_template as response_template_module +from vllm.parser.gemma4 import Gemma4Parser +from vllm.parser.parser_manager import ParserManager +from vllm.parser.response_template import ( + ResponseTemplateParser, + ResponseTemplateReasoningParser, + ResponseTemplateToolParser, + resolve_response_template, +) +from vllm.reasoning import ReasoningParserManager +from vllm.tool_parsers import ToolParserManager + +# Public metadata shared by the Gemma 4 instruction checkpoints. +GEMMA4_RESPONSE_TEMPLATE: dict[str, Any] = { + "defaults": {"role": "assistant"}, + "fields": { + "content": { + "close": ["", "<|tool_response>", ""], + "content": "text", + }, + "thinking": { + "close": "", + "content": "text", + "open": "<|channel>thought\n", + }, + "tool_calls": { + "close": "", + "content": "json", + "content_args": { + "string_delims": [['<|"|>', '<|"|>']], + "unquoted_keys": True, + }, + "open_pattern": r"<\|tool_call>call:(?P\w+)", + "repeats": True, + "transform": { + "function": { + "arguments": "{content}", + "name": "{name}", + }, + "type": "function", + }, + }, + }, + "start_anchor": ["<|turn>model\n", ""], +} + +PREFIX = "<|turn>model\n" +THINKING = "<|channel>thought\nplan" +TOOL_GENERATION = ( + '<|tool_call>call:set_alarm{hour:7,label:<|"|>morning<|"|>}' +) + +TOOLS = [ + ChatCompletionToolsParam( + type="function", + function={ + "name": "set_alarm", + "parameters": { + "type": "object", + "properties": { + "hour": { + "oneOf": [ + {"type": "integer"}, + {"type": "null"}, + ] + }, + "label": {"type": "string"}, + }, + }, + }, + ), + ChatCompletionToolsParam( + type="function", + function={ + "name": "get_weather", + "parameters": { + "type": "object", + "properties": { + "city": {"type": "string"}, + "unit": {"type": "string"}, + }, + }, + }, + ), + ChatCompletionToolsParam( + type="function", + function={ + "name": "set_status", + "parameters": { + "type": "object", + "properties": { + "is_active": {"type": "boolean"}, + "count": {"type": "integer"}, + "score": {"type": "number"}, + }, + }, + }, + ), + ChatCompletionToolsParam( + type="function", + function={ + "name": "complex_function", + "parameters": { + "type": "object", + "properties": { + "nested": { + "type": "object", + "properties": {"inner": {"type": "string"}}, + }, + "list": { + "type": "array", + "items": {"type": "string"}, + }, + }, + }, + }, + ), + ChatCompletionToolsParam( + type="function", + function={ + "name": "configure", + "parameters": { + "type": "object", + "properties": { + "enabled": {"type": "boolean"}, + "ratio": {"type": "number"}, + "label": {"type": "string"}, + "value": {"type": ["string", "null"]}, + }, + }, + }, + ), + ChatCompletionToolsParam( + type="function", + function={ + "name": "get_status", + "parameters": {"type": "object", "properties": {}}, + }, + ), + ChatCompletionToolsParam( + type="function", + function={ + "name": "get-weather", + "parameters": { + "type": "object", + "properties": {"city": {"type": "string"}}, + }, + }, + ), + ChatCompletionToolsParam( + type="function", + function={ + "name": "search", + "parameters": { + "type": "object", + "properties": { + "query": {"type": "string"}, + "filters": { + "type": "object", + "properties": { + "language": {"type": "string"}, + "min_stars": {"type": "integer"}, + }, + }, + }, + }, + }, + ), +] + +VOCAB = { + "<|tool_call>": 48, + "": 49, + "<|channel>": 50, + "": 51, + '<|"|>': 52, + "<|turn>": 53, + "<|tool_response>": 54, + "": 55, +} +PROMPT_TOKEN_IDS = [1] +OPEN_REASONING_PROMPT_TOKEN_IDS = [53, 100, 50, 101] +PROMPT_TOKEN_TEXT = { + 100: "model\n", + 101: "thought\n", + 102: "plan", + 103: "answer", +} + + +class FakeTokenizer: + def __init__(self, response_template=None, *, prefix=PREFIX): + self.response_template = response_template + self.init_kwargs = {} + self.name_or_path = "" + self.prefix = prefix + self.all_special_tokens = list(VOCAB) + self.all_special_ids = list(VOCAB.values()) + + def decode(self, token_ids, skip_special_tokens=False, **_kwargs): + if token_ids == [2]: + return PREFIX + "answer" + if token_ids == OPEN_REASONING_PROMPT_TOKEN_IDS: + return PREFIX + "<|channel>thought\n" + if token_ids != PROMPT_TOKEN_IDS: + inverse_vocab = {token_id: token for token, token_id in VOCAB.items()} + return "".join( + PROMPT_TOKEN_TEXT.get(token_id, inverse_vocab.get(token_id, "")) + for token_id in token_ids + ) + return self.prefix if token_ids else "" + + def get_vocab(self): + return VOCAB + + def encode(self, text, add_special_tokens=False): + del add_special_tokens + if text == "plan": + return [102] + return list(text.encode()) + + +def request(tool_choice="auto", *, include_reasoning=True, **overrides): + values = dict( + tool_choice=tool_choice, + include_reasoning=include_reasoning, + tools=TOOLS, + skip_special_tokens=True, + spaces_between_special_tokens=True, + include_stop_str_in_output=False, + parallel_tool_calls=True, + ) + values.update(overrides) + return SimpleNamespace(**values) + + +def normalize_calls(calls): + return [(call.name, json.loads(call.arguments)) for call in calls or []] + + +def feed_chunks( + parser, + chunks, + req, + *, + prompt_token_ids=PROMPT_TOKEN_IDS, +): + deltas = [] + for index, chunk in enumerate(chunks): + delta = parser.parse_delta( + chunk, + [], + req, + prompt_token_ids=prompt_token_ids if index == 0 else None, + finished=index == len(chunks) - 1, + ) + if delta is not None: + deltas.append(delta) + return deltas + + +def collect_stream( + parser, + text, + req, + chunk_size, + *, + prompt_token_ids=PROMPT_TOKEN_IDS, +): + reasoning = [] + content = [] + calls: dict[int, dict[str, str]] = {} + for start in range(0, len(text), chunk_size): + end = min(start + chunk_size, len(text)) + delta = parser.parse_delta( + text[start:end], + [], + req, + prompt_token_ids=prompt_token_ids if start == 0 else None, + finished=end == len(text), + ) + if delta is None: + continue + if delta.reasoning: + reasoning.append(delta.reasoning) + if delta.content: + content.append(delta.content) + for call in delta.tool_calls: + function = call.function + assert function is not None + slot = calls.setdefault( + call.index, + {"name": "", "arguments": ""}, + ) + slot["name"] += function.name or "" + slot["arguments"] += function.arguments or "" + normalized_calls = [ + (call["name"], json.loads(call["arguments"])) + for _, call in sorted(calls.items()) + ] + return "".join(reasoning), "".join(content), normalized_calls + + +def test_resolve_response_template_from_tokenizer_configuration(): + tokenizer = FakeTokenizer(GEMMA4_RESPONSE_TEMPLATE) + assert resolve_response_template(tokenizer) == GEMMA4_RESPONSE_TEMPLATE + + tokenizer = FakeTokenizer() + del tokenizer.response_template + tokenizer.init_kwargs["response_template"] = GEMMA4_RESPONSE_TEMPLATE + assert resolve_response_template(tokenizer) == GEMMA4_RESPONSE_TEMPLATE + + explicit = deepcopy(GEMMA4_RESPONSE_TEMPLATE) + explicit["defaults"]["role"] = "custom" + assert resolve_response_template(tokenizer, explicit) is explicit + + +def test_response_template_parsers_are_registered(): + assert ToolParserManager.get_tool_parser("hf") is ResponseTemplateToolParser + assert ( + ReasoningParserManager.get_reasoning_parser("hf") + is ResponseTemplateReasoningParser + ) + + +@pytest.mark.parametrize( + ("reasoning", "tools"), + [(True, True), (True, False), (False, True)], + ids=["both", "reasoning", "tools"], +) +def test_parser_manager_selects_response_template_by_name(reasoning, tools): + parser_cls = ParserManager.get_parser( + tool_parser_name="hf" if tools else None, + reasoning_parser_name="hf" if reasoning else None, + enable_auto_tools=tools, + tokenizer=FakeTokenizer(GEMMA4_RESPONSE_TEMPLATE), + ) + + assert parser_cls is not None + assert issubclass(parser_cls, ResponseTemplateParser) + assert (parser_cls.reasoning_parser_cls is not None) is reasoning + assert (parser_cls.tool_parser_cls is not None) is tools + + +def test_parser_manager_validates_checkpoint_metadata_at_startup(): + with pytest.raises(TypeError, match="requires `response_template` metadata"): + ParserManager.get_parser( + reasoning_parser_name="hf", + tokenizer=FakeTokenizer(), + ) + with pytest.raises(TypeError, match="Invalid response_template"): + ParserManager.get_parser( + reasoning_parser_name="hf", + tokenizer=FakeTokenizer({"fields": {}}), + ) + + template = deepcopy(GEMMA4_RESPONSE_TEMPLATE) + del template["fields"]["tool_calls"] + with pytest.raises(TypeError, match="tool_calls field"): + ParserManager.get_parser( + tool_parser_name="hf", + enable_auto_tools=True, + tokenizer=FakeTokenizer(template), + ) + assert ( + ParserManager.get_parser( + reasoning_parser_name="hf", + tokenizer=FakeTokenizer(template), + ) + is not None + ) + + +def test_response_template_parser_rejects_other_parsers(): + with pytest.raises(TypeError, match="cannot be combined"): + ParserManager.get_parser( + tool_parser_name="hermes", + reasoning_parser_name="hf", + enable_auto_tools=True, + tokenizer=FakeTokenizer(GEMMA4_RESPONSE_TEMPLATE), + ) + + +def test_request_chat_template_is_honored_with_warning(monkeypatch): + warnings: list[str] = [] + monkeypatch.setattr( + response_template_module.logger, "warning_once", warnings.append + ) + parser = ResponseTemplateParser(FakeTokenizer(GEMMA4_RESPONSE_TEMPLATE), TOOLS) + + parser.adjust_request(request(chat_template="{{ messages }}")) + + assert len(warnings) == 1 + assert parser.response_template.fields + + +def test_reasoning_gate_waits_for_content_region(): + parser = ResponseTemplateParser( + FakeTokenizer( + GEMMA4_RESPONSE_TEMPLATE, + prefix=PREFIX + "<|channel>thought\n", + ), + TOOLS, + ) + + assert parser.is_reasoning_end([1]) is False + assert parser.is_reasoning_end([2]) is True + + +def test_reasoning_gate_ends_at_content_opener_held_by_prompt(): + template = deepcopy(GEMMA4_RESPONSE_TEMPLATE) + template["fields"]["content"]["open_pattern"] = r"<\|channel>final\n" + parser = ResponseTemplateParser( + FakeTokenizer(template, prefix=PREFIX + "<|channel>final\n"), + TOOLS, + ) + + assert parser.is_reasoning_end([1]) is True + + +def streamed_reasoning_end(reasoner, output, prompt=PROMPT_TOKEN_IDS): + """Index of the output token where the per-step gate reports the end.""" + for index, token in enumerate(output): + if reasoner.is_reasoning_end_streaming(prompt + output[: index + 1], [token]): + return index + return None + + +def test_streaming_reasoning_gate_matches_thinking_closer_without_decoding(): + tokenizer = FakeTokenizer(GEMMA4_RESPONSE_TEMPLATE) + reasoner = ResponseTemplateReasoningParser(tokenizer) + tokenizer.decode = lambda *_args, **_kwargs: pytest.fail("decoded") # type: ignore[method-assign] + output = [50, 101] + [102] * 100 + [51, 103] + + assert streamed_reasoning_end(reasoner, output) == len(output) - 2 + assert reasoner.is_reasoning_end_streaming(PROMPT_TOKEN_IDS + output, output[-3:]) + + +def test_reasoning_gate_honors_disabled_and_repeated_thinking(): + disabled = ResponseTemplateParser( + FakeTokenizer(GEMMA4_RESPONSE_TEMPLATE), + TOOLS, + chat_template_kwargs={"enable_thinking": False}, + ) + assert disabled.is_reasoning_end([1]) is True + + repeated_template = deepcopy(GEMMA4_RESPONSE_TEMPLATE) + repeated_template["fields"]["thinking"]["repeats"] = True + repeated = ResponseTemplateParser( + FakeTokenizer(repeated_template), + TOOLS, + response_template=repeated_template, + ) + assert repeated.is_reasoning_end(OPEN_REASONING_PROMPT_TOKEN_IDS) is False + assert repeated.is_reasoning_end([2]) is False + + +@pytest.mark.parametrize( + "text", + [ + "Hello", + THINKING + "The answer is 42.", + THINKING + TOOL_GENERATION, + "Let me check." + TOOL_GENERATION, + ( + '<|tool_call>call:get_weather{city:<|"|>Paris<|"|>,' + 'unit:<|"|>celsius<|"|>}' + TOOL_GENERATION + ), + ( + "<|tool_call>call:complex_function{" + 'nested:{inner:<|"|>value<|"|>},' + 'list:[<|"|>a<|"|>,<|"|>b<|"|>]}' + "" + ), + ], + ids=[ + "content", + "reasoning", + "tool", + "content_then_tool", + "multiple_tools", + "nested_arguments", + ], +) +def test_gemma4_response_template_matches_registered_parser(text): + tokenizer = FakeTokenizer(GEMMA4_RESPONSE_TEMPLATE) + req = request() + registered = Gemma4Parser(tokenizer, TOOLS) + metadata = ResponseTemplateParser( + tokenizer, + TOOLS, + response_template=GEMMA4_RESPONSE_TEMPLATE, + ) + metadata.set_prompt_token_ids([1]) + + expected_reasoning, expected_content, expected_calls = registered.parse( + text, + req, + enable_auto_tools=True, + ) + reasoning, content, calls = metadata.parse( + text, + req, + enable_auto_tools=True, + ) + + assert reasoning == expected_reasoning + assert content == expected_content + assert normalize_calls(calls) == normalize_calls(expected_calls) + + +def test_metadata_parser_keeps_already_decoded_nullable_values(): + text = ( + "<|tool_call>call:configure{" + "enabled:false,ratio:5.0,label:null,value:null}" + "" + ) + parser = ResponseTemplateParser( + FakeTokenizer(GEMMA4_RESPONSE_TEMPLATE), + TOOLS, + response_template=GEMMA4_RESPONSE_TEMPLATE, + ) + parser.set_prompt_token_ids([1]) + + _, content, calls = parser.parse( + text, + request(), + enable_auto_tools=True, + ) + + assert content is None + assert normalize_calls(calls) == [ + ( + "configure", + { + "enabled": False, + "ratio": 5.0, + "label": None, + "value": None, + }, + ) + ] + + +@pytest.mark.parametrize( + "text", + [ + THINKING + TOOL_GENERATION, + THINKING + "The answer is 42.", + "Let me check. " + TOOL_GENERATION, + ( + '<|tool_call>call:get_weather{city:<|"|>Paris<|"|>}' + + TOOL_GENERATION + ), + ], + ids=[ + "reasoning_then_tool", + "reasoning_then_content", + "content_then_tool", + "multiple_tools", + ], +) +@pytest.mark.parametrize("chunk_size", [1, 7, 10_000]) +def test_gemma4_response_template_streaming_matches_registered_parser( + text, + chunk_size, +): + req = request() + registered = Gemma4Parser( + FakeTokenizer(GEMMA4_RESPONSE_TEMPLATE), + TOOLS, + ) + metadata = ResponseTemplateParser( + FakeTokenizer(GEMMA4_RESPONSE_TEMPLATE), + TOOLS, + enable_auto_tools=True, + ) + + assert collect_stream(metadata, text, req, chunk_size) == collect_stream( + registered, + text, + req, + chunk_size, + ) + + +@pytest.mark.parametrize( + ("prompt_token_ids", "text"), + [ + ( + OPEN_REASONING_PROMPT_TOKEN_IDS, + "continued reasoningFinal answer", + ), + ( + PROMPT_TOKEN_IDS, + "Direct answer without reasoning.", + ), + ], + ids=["prefilled_reasoning", "direct_answer"], +) +def test_gemma4_response_template_prompt_state_matches_registered_parser( + prompt_token_ids, + text, +): + req = request() + registered = Gemma4Parser( + FakeTokenizer(GEMMA4_RESPONSE_TEMPLATE), + TOOLS, + chat_template_kwargs={"enable_thinking": True}, + ) + metadata = ResponseTemplateParser( + FakeTokenizer(GEMMA4_RESPONSE_TEMPLATE), + TOOLS, + enable_auto_tools=True, + ) + + assert collect_stream( + metadata, + text, + req, + 1, + prompt_token_ids=prompt_token_ids, + ) == collect_stream( + registered, + text, + req, + 1, + prompt_token_ids=prompt_token_ids, + ) + + +def test_gemma4_response_template_reasoning_suppression_matches_registered_parser(): + req = request(include_reasoning=False) + text = THINKING + "Final answer" + + assert collect_stream( + ResponseTemplateParser( + FakeTokenizer(GEMMA4_RESPONSE_TEMPLATE), + TOOLS, + enable_auto_tools=True, + ), + text, + req, + 1, + ) == collect_stream( + Gemma4Parser(FakeTokenizer(GEMMA4_RESPONSE_TEMPLATE), TOOLS), + text, + req, + 1, + ) + + +def test_streamed_tool_call_is_emitted_when_its_region_parses(): + chunks = [ + "<|tool_call>call:set_alarm{", + "hour:7,", + 'label:<|"|>morning<|"|>}', + "", + ] + deltas = feed_chunks( + ResponseTemplateParser( + FakeTokenizer(GEMMA4_RESPONSE_TEMPLATE), + TOOLS, + enable_auto_tools=True, + ), + chunks, + request(), + ) + + assert all(not delta.tool_calls for delta in deltas[:-1]) + calls = deltas[-1].tool_calls + assert len(calls) == 1 + assert calls[0].id is not None + assert calls[0].index == 0 + assert calls[0].function is not None + assert calls[0].function.name == "set_alarm" + assert json.loads(calls[0].function.arguments) == { + "hour": 7, + "label": "morning", + } + + +def test_response_template_emits_whole_call_from_one_chunk(): + parser = ResponseTemplateParser( + FakeTokenizer(GEMMA4_RESPONSE_TEMPLATE), + TOOLS, + enable_auto_tools=True, + ) + + delta = parser.parse_delta( + TOOL_GENERATION, + [], + request(), + prompt_token_ids=PROMPT_TOKEN_IDS, + finished=True, + ) + + assert delta is not None + assert len(delta.tool_calls) == 1 + call = delta.tool_calls[0] + assert call.id is not None + assert call.function is not None + assert call.function.name == "set_alarm" + assert json.loads(call.function.arguments) == {"hour": 7, "label": "morning"} + + +def test_response_template_takes_name_from_region_content(): + template = { + "defaults": {"role": "assistant"}, + "start_anchor": PREFIX, + "fields": { + "tool_calls": { + "open": "", + "close": "", + "content": "json", + "transform": { + "type": "function", + "function": { + "name": "{content.name}", + "arguments": "{content.arguments}", + }, + }, + } + }, + } + chunks = [ + "", + '{"name":"set_', + 'alarm","arguments":{"hour":7}}', + "", + ] + parser = ResponseTemplateParser( + FakeTokenizer(template), + TOOLS, + response_template=template, + enable_auto_tools=True, + ) + + deltas = feed_chunks(parser, chunks, request()) + + assert all(not delta.tool_calls for delta in deltas[:-1]) + assert len(deltas[-1].tool_calls) == 1 + call = deltas[-1].tool_calls[0] + assert call.function is not None + assert call.function.name == "set_alarm" + assert json.loads(call.function.arguments) == {"hour": 7} + + +def test_response_template_applies_literal_transformed_name(): + template = { + "defaults": {"role": "assistant"}, + "start_anchor": PREFIX, + "fields": { + "tool_calls": { + "open": "", + "close": "", + "content": "json", + "transform": { + "type": "function", + "function": { + "name": "set_alarm", + "arguments": "{content}", + }, + }, + } + }, + } + parser = ResponseTemplateParser( + FakeTokenizer(template), + TOOLS, + response_template=template, + enable_auto_tools=True, + ) + + deltas = feed_chunks( + parser, + ["", '{"hour":7}', ""], + request(), + ) + + assert len(deltas) == 1 + assert deltas[0].tool_calls[0].function.name == "set_alarm" + assert json.loads(deltas[0].tool_calls[0].function.arguments) == {"hour": 7} + + +def test_gemma4_response_template_drops_incomplete_call(): + text = '<|tool_call>call:get_weather{city:<|"|>London' + req = request() + registered = Gemma4Parser(FakeTokenizer(GEMMA4_RESPONSE_TEMPLATE), TOOLS) + metadata = ResponseTemplateParser( + FakeTokenizer(GEMMA4_RESPONSE_TEMPLATE), + TOOLS, + ) + metadata.set_prompt_token_ids(PROMPT_TOKEN_IDS) + + _, _, registered_calls = registered.parse( + text, + req, + enable_auto_tools=True, + ) + _, metadata_content, metadata_calls = metadata.parse( + text, + req, + enable_auto_tools=True, + ) + + assert normalize_calls(registered_calls) == [("get_weather", {"city": "London"})] + assert metadata_content is None + assert metadata_calls is None + + +def test_gemma4_response_template_preserves_content_after_tool_call(): + text = TOOL_GENERATION + "Done." + req = request() + registered = Gemma4Parser(FakeTokenizer(GEMMA4_RESPONSE_TEMPLATE), TOOLS) + metadata = ResponseTemplateParser( + FakeTokenizer(GEMMA4_RESPONSE_TEMPLATE), + TOOLS, + ) + metadata.set_prompt_token_ids(PROMPT_TOKEN_IDS) + + _, registered_content, registered_calls = registered.parse( + text, + req, + enable_auto_tools=True, + ) + _, metadata_content, metadata_calls = metadata.parse( + text, + req, + enable_auto_tools=True, + ) + + assert registered_content is None + assert metadata_content == "Done." + assert normalize_calls(metadata_calls) == normalize_calls(registered_calls) + + +def test_gemma4_checkpoint_grammar_excludes_hyphenated_name(): + text = '<|tool_call>call:get-weather{city:<|"|>London<|"|>}' + req = request() + registered = Gemma4Parser(FakeTokenizer(GEMMA4_RESPONSE_TEMPLATE), TOOLS) + metadata = ResponseTemplateParser( + FakeTokenizer(GEMMA4_RESPONSE_TEMPLATE), + TOOLS, + ) + metadata.set_prompt_token_ids(PROMPT_TOKEN_IDS) + + _, _, registered_calls = registered.parse( + text, + req, + enable_auto_tools=True, + ) + _, metadata_content, metadata_calls = metadata.parse( + text, + req, + enable_auto_tools=True, + ) + + assert normalize_calls(registered_calls) == [("get-weather", {"city": "London"})] + assert metadata_content is None + assert metadata_calls is None + + +@pytest.mark.parametrize( + ( + "prefix_suffix", + "generated", + "enable_auto_tools", + "expected_reasoning", + "expected_content", + ), + [ + ( + "<|channel>thought\nprefill", + " generated", + False, + " generated", + None, + ), + ("prefill ", "generated", False, None, "generated"), + ("prefill", "", False, None, None), + (TOOL_GENERATION.split("{", 1)[0], "", True, None, None), + ("<|chan", "", False, None, None), + ("<|chan", "nel", False, None, "nel"), + ( + "<|chan", + "nel>thought\nreason", + False, + "reason", + None, + ), + ], + ids=[ + "reasoning", + "content", + "empty_content", + "empty_tool_open", + "empty_partial_delimiter", + "unresolved_partial_delimiter", + "completed_partial_delimiter", + ], +) +def test_streaming_prefill_is_state_only( + prefix_suffix, + generated, + enable_auto_tools, + expected_reasoning, + expected_content, +): + tokenizer = FakeTokenizer( + GEMMA4_RESPONSE_TEMPLATE, + prefix=PREFIX + prefix_suffix, + ) + parser = ResponseTemplateParser( + tokenizer, + TOOLS, + enable_auto_tools=enable_auto_tools, + ) + + delta = parser.parse_delta( + generated, + [], + request(), + prompt_token_ids=[1], + finished=True, + ) + + if expected_reasoning is None and expected_content is None: + assert delta is None + else: + assert delta is not None + assert delta.reasoning == expected_reasoning + assert delta.content == expected_content + + +def test_malformed_call_opened_across_prompt_boundary_is_dropped(): + parser = ResponseTemplateParser( + FakeTokenizer( + GEMMA4_RESPONSE_TEMPLATE, + prefix=PREFIX + "<|tool_call>call:set_", + ), + TOOLS, + enable_auto_tools=True, + ) + + opened = parser.parse_delta( + "alarm{", + [], + request(), + prompt_token_ids=[1], + finished=False, + ) + delta = parser.parse_delta( + "hour:7}unexpected", + [], + request(), + finished=True, + ) + + assert opened is None + assert delta is None + + +def test_responses_namespace_tools_use_vllm_flattened_names(): + namespace = NamespaceTool( + type="namespace", + name="calendar", + description="Calendar tools", + tools=[ + { + "type": "function", + "name": "set_alarm", + "description": "Set an alarm", + "parameters": TOOLS[0].function.parameters, + } + ], + ) + parser = ResponseTemplateParser( + FakeTokenizer(GEMMA4_RESPONSE_TEMPLATE), + [namespace], + ) + parser.set_prompt_token_ids([1]) + output = TOOL_GENERATION.replace( + "call:set_alarm", + "call:calendar__set_alarm", + ) + + _, content, calls = parser.parse( + output, + request(), + enable_auto_tools=True, + ) + + assert content is None + assert normalize_calls(calls) == [ + ("calendar__set_alarm", {"hour": 7, "label": "morning"}) + ] + + +def test_streaming_unknown_call_is_forwarded(): + text = TOOL_GENERATION.replace("call:set_alarm", "call:unknown") + parser = ResponseTemplateParser( + FakeTokenizer(GEMMA4_RESPONSE_TEMPLATE), + TOOLS, + enable_auto_tools=True, + ) + _, content, calls = collect_stream(parser, text, request(), 7) + + assert content == "" + assert calls == [("unknown", {"hour": 7, "label": "morning"})] + + +@pytest.mark.parametrize( + "text", + [ + TOOL_GENERATION.replace( + "", + "unexpected", + ), + TOOL_GENERATION.removesuffix("") + "", "unexpected") + + _, content, calls = collect_stream( + parser, malformed + TOOL_GENERATION, request(), 7 + ) + + assert content == "" + assert calls == [("set_alarm", {"hour": 7, "label": "morning"})] + + +def test_non_streaming_malformed_call_is_dropped(): + text = TOOL_GENERATION.replace( + "", + "unexpected", + ) + parser = ResponseTemplateParser( + FakeTokenizer(GEMMA4_RESPONSE_TEMPLATE), + TOOLS, + ) + parser.set_prompt_token_ids([1]) + + reasoning, content, calls = parser.parse( + text, + request(), + enable_auto_tools=True, + ) + + assert reasoning is None + assert content is None + assert calls is None + + +def test_streaming_malformed_call_is_dropped_before_emission(): + parser = ResponseTemplateParser( + FakeTokenizer(GEMMA4_RESPONSE_TEMPLATE), + TOOLS, + enable_auto_tools=True, + ) + first = parser.parse_delta( + "hello", + [], + request(), + prompt_token_ids=[1], + finished=False, + ) + malformed = TOOL_GENERATION.replace( + "", + "unexpected", + ) + failed = parser.parse_delta( + malformed, + [], + request(), + finished=False, + ) + + assert first is not None + assert first.content == "hello" + assert failed is None + + +def test_prefilled_malformed_call_is_dropped(): + opening, body = TOOL_GENERATION.split("{", 1) + malformed_body = "{" + body.replace( + "", + "unexpected", + ) + tokenizer = FakeTokenizer( + GEMMA4_RESPONSE_TEMPLATE, + prefix=PREFIX + opening, + ) + parser = ResponseTemplateParser( + tokenizer, + TOOLS, + enable_auto_tools=True, + ) + + delta = parser.parse_delta( + malformed_body, + [], + request(), + prompt_token_ids=[1], + finished=True, + ) + + assert delta is None + + +def test_tool_choice_none_preserves_raw_call_as_content(): + parser = ResponseTemplateParser( + FakeTokenizer(GEMMA4_RESPONSE_TEMPLATE), + TOOLS, + enable_auto_tools=True, + ) + _, content, calls = collect_stream( + parser, + TOOL_GENERATION, + request("none"), + 5, + ) + + assert content == TOOL_GENERATION + assert calls == [] + + +def test_adjust_request_preserves_parser_delimiters_without_forcing_stop_text(): + req = request() + parser = ResponseTemplateParser( + FakeTokenizer(GEMMA4_RESPONSE_TEMPLATE), + TOOLS, + ) + + assert parser.adjust_request(req) is req + assert req.skip_special_tokens is False + assert req.spaces_between_special_tokens is False + assert req.include_stop_str_in_output is False + + +def test_call_cut_off_before_its_closer_is_parsed(): + """A stop-token closer, such as a tool closer that ends generation, is + absent from the text; the open region is parsed at the end of the stream.""" + trimmed = TOOL_GENERATION.removesuffix("") + parser = ResponseTemplateParser(FakeTokenizer(GEMMA4_RESPONSE_TEMPLATE), TOOLS) + streaming = ResponseTemplateParser( + FakeTokenizer(GEMMA4_RESPONSE_TEMPLATE), + TOOLS, + enable_auto_tools=True, + ) + + _, content, calls = parser.parse(trimmed, request(), enable_auto_tools=True) + streamed = streaming.parse_delta( + trimmed, + [], + request(), + prompt_token_ids=[1], + finished=True, + ) + + assert content is None + assert normalize_calls(calls) == [("set_alarm", {"hour": 7, "label": "morning"})] + assert streamed is not None + assert [call.function.name for call in streamed.tool_calls] == ["set_alarm"] + + +@pytest.mark.parametrize( + ("req", "match"), + [ + (request("required"), "required tool choice"), + ( + request( + ChatCompletionNamedToolChoiceParam( + type="function", + function={"name": "set_alarm"}, + ) + ), + "named tool choice", + ), + (request("auto", parallel_tool_calls=False), "parallel_tool_calls=False"), + ( + request( + "auto", + tools=[ + { + "type": "function", + "name": "set_alarm", + "parameters": {"type": "object"}, + "strict": True, + } + ], + ), + "strict tools", + ), + ], +) +def test_unconstrained_tool_guarantees_are_rejected(req, match): + parser = ResponseTemplateParser( + FakeTokenizer(GEMMA4_RESPONSE_TEMPLATE), + req.tools, + enable_auto_tools=True, + ) + + with pytest.raises(VLLMValidationError, match=match): + parser.adjust_request(req) + + +def test_tool_choice_none_does_not_require_structural_guarantees(): + req = request( + "none", + tools=[ + { + "type": "function", + "name": "set_alarm", + "parameters": {"type": "object"}, + "strict": True, + } + ], + parallel_tool_calls=False, + ) + parser = ResponseTemplateParser( + FakeTokenizer(GEMMA4_RESPONSE_TEMPLATE), + req.tools, + enable_auto_tools=True, + ) + + assert parser.adjust_request(req) is req + + +def test_unsupported_semantic_field_is_rejected(): + template = deepcopy(GEMMA4_RESPONSE_TEMPLATE) + template["fields"]["citations"] = { + "open": "", + "close": "", + } + + with pytest.raises(ValueError, match="citations"): + ResponseTemplateParser( + FakeTokenizer(template), + TOOLS, + response_template=template, + ) + + +@pytest.mark.parametrize( + ("field_name", "field_update"), + [ + ("content", {"content": "json"}), + ("thinking", {"transform": "{content}"}), + ], +) +def test_unsupported_content_field_semantics_are_rejected( + field_name, + field_update, +): + template = deepcopy(GEMMA4_RESPONSE_TEMPLATE) + template["fields"][field_name].update(field_update) + + with pytest.raises(ValueError, match="cannot be streamed"): + ResponseTemplateParser( + FakeTokenizer(template), + TOOLS, + response_template=template, + ) + + +def test_delimited_template_keeps_plain_content_and_tolerates_required_fields(): + template = { + "start_anchor": "", + "fields": { + "thinking": {"open": "", "close": ""}, + "tool_calls": { + "open": "", + "close": "", + "content": "json", + "optional": False, + }, + }, + } + parser = ResponseTemplateParser( + FakeTokenizer(template, prefix=""), + TOOLS, + response_template=template, + ) + + assert parser.parse("plananswer", request()) == ( + "plan", + "answer", + None, + ) + + +def test_separate_parse_calls_use_distinct_tool_call_ids(): + parser = ResponseTemplateParser( + FakeTokenizer(GEMMA4_RESPONSE_TEMPLATE), + TOOLS, + enable_auto_tools=True, + ) + parsed_calls = [] + for _ in range(2): + _, _, calls = parser.parse( + TOOL_GENERATION, + request(), + enable_auto_tools=True, + ) + assert calls is not None + parsed_calls.append(calls[0]) + + assert parsed_calls[0].id is not None + assert parsed_calls[1].id is not None + assert parsed_calls[0].id != parsed_calls[1].id diff --git a/vllm/entrypoints/launchers/cli_args.py b/vllm/entrypoints/launchers/cli_args.py index 3a52cf2e2b..a800e1270d 100644 --- a/vllm/entrypoints/launchers/cli_args.py +++ b/vllm/entrypoints/launchers/cli_args.py @@ -22,6 +22,7 @@ from vllm.entrypoints.chat_utils import ( validate_chat_template, ) from vllm.entrypoints.openai.models.protocol import LoRAModulePath +from vllm.logger import init_logger from vllm.tool_parsers import ToolParserManager from vllm.tool_parsers.tool_strict_level import ToolStrictLevelName from vllm.utils.argparse_utils import FlexibleArgumentParser @@ -31,6 +32,8 @@ from .utils.constants import ( H11_MAX_INCOMPLETE_EVENT_SIZE_DEFAULT, ) +logger = init_logger(__name__) + class LoRAParserAction(argparse.Action): def __call__( @@ -466,6 +469,14 @@ def validate_parsed_serve_args(args: argparse.Namespace): # Ensure that the chat template is valid; raises if it likely isn't validate_chat_template(args.chat_template) + if args.chat_template is not None and "hf" in ( + args.tool_call_parser, + getattr(args, "reasoning_parser", None), + ): + logger.warning( + "--chat-template is set; the hf parser still expects " + "the checkpoint's output format." + ) # Enable auto tool needs a tool call parser to be valid if args.enable_auto_tool_choice and not args.tool_call_parser: diff --git a/vllm/entrypoints/openai/chat_completion/batch_serving.py b/vllm/entrypoints/openai/chat_completion/batch_serving.py index 3fde958a3d..8c7ae37647 100644 --- a/vllm/entrypoints/openai/chat_completion/batch_serving.py +++ b/vllm/entrypoints/openai/chat_completion/batch_serving.py @@ -129,17 +129,6 @@ class OpenAIServingChatBatch(OpenAIServingChat): return render_result single_requests, all_conversations, engine_prompts = render_result - parser: Parser | None = None - if self.parser_cls is not None: - chat_template_kwargs = self._effective_chat_template_kwargs( - single_requests[0] - ) - parser = self.parser_cls( - tokenizer, - None, # tools - chat_template_kwargs=chat_template_kwargs, - ) - request_id = ( f"chatcmpl-{self._base_request_id(raw_request, request.request_id)}" ) @@ -202,7 +191,7 @@ class OpenAIServingChatBatch(OpenAIServingChat): all_conversations, tokenizer, request_metadata, - parser, + self.parser_cls, ) async def chat_completion_full_generator_batch( @@ -214,7 +203,7 @@ class OpenAIServingChatBatch(OpenAIServingChat): all_conversations: list[list[ConversationMessage]], tokenizer: TokenizerLike, request_metadata: RequestResponseMetadata, - parser: Parser | None = None, + parser_cls: type[Parser] | None = None, ) -> ErrorResponse | ChatCompletionResponse: """Handle batched (non-streaming) chat completions. @@ -273,10 +262,21 @@ class OpenAIServingChatBatch(OpenAIServingChat): else: logprobs = None - if parser is not None: + if parser_cls is not None: + single_request = request.to_chat_completion_request( + request.messages[prompt_idx] + ) + parser = parser_cls( + tokenizer, + None, + chat_template_kwargs=self._effective_chat_template_kwargs( + single_request + ), + ) + parser.set_prompt_token_ids(final_res.prompt_token_ids) reasoning, content, _ = parser.parse( output.text, - request=request, # type: ignore[arg-type] + request=single_request, model_output_token_ids=output.token_ids, ) if not request.include_reasoning: diff --git a/vllm/entrypoints/openai/chat_completion/serving.py b/vllm/entrypoints/openai/chat_completion/serving.py index f22fed1e75..060d22b031 100644 --- a/vllm/entrypoints/openai/chat_completion/serving.py +++ b/vllm/entrypoints/openai/chat_completion/serving.py @@ -165,6 +165,7 @@ class OpenAIServingChat(GenerateBaseServing): tool_strict_level=tool_strict_level, model_name=self.model_config.model, is_harmony=self.model_config.hf_config.model_type == "gpt_oss", + tokenizer=self.renderer.tokenizer, ) self.exclude_tools_when_tool_choice_none = exclude_tools_when_tool_choice_none @@ -974,6 +975,8 @@ class OpenAIServingChat(GenerateBaseServing): parser = self._make_parser( request, tokenizer, self._effective_chat_template_kwargs(request) ) + if parser is not None and final_res.prompt_token_ids is not None: + parser.set_prompt_token_ids(final_res.prompt_token_ids) token_ids = output.token_ids out_logprobs = output.logprobs diff --git a/vllm/entrypoints/openai/responses/serving.py b/vllm/entrypoints/openai/responses/serving.py index 66e0f2640d..bd876e4916 100644 --- a/vllm/entrypoints/openai/responses/serving.py +++ b/vllm/entrypoints/openai/responses/serving.py @@ -141,6 +141,7 @@ class OpenAIServingResponses(GenerateBaseServing): tool_strict_level=tool_strict_level, model_name=self.model_config.model, is_harmony=self.model_config.hf_config.model_type == "gpt_oss", + tokenizer=self.renderer.tokenizer, ) self.enable_prompt_tokens_details = enable_prompt_tokens_details self.enable_force_include_usage = enable_force_include_usage @@ -446,6 +447,12 @@ class OpenAIServingResponses(GenerateBaseServing): response_parser = self._make_response_parser( request, tokenizer, chat_template_kwargs ) + if response_parser is not None: + prompt_token_ids = self._extract_prompt_components( + engine_input + ).token_ids + if prompt_token_ids is not None: + response_parser.set_prompt_token_ids(prompt_token_ids) context: ConversationContext function_tool_names = extract_function_tool_names(request.tools) @@ -703,6 +710,12 @@ class OpenAIServingResponses(GenerateBaseServing): context.request, context.response_messages, ) + if context.response_parser is not None: + prompt_token_ids = self._extract_prompt_components( + engine_input + ).token_ids + if prompt_token_ids is not None: + context.response_parser.set_prompt_token_ids(prompt_token_ids) sampling_params.max_tokens = get_max_tokens( max_model_len, @@ -1187,7 +1200,9 @@ class OpenAIServingResponses(GenerateBaseServing): ) -> AsyncGenerator[StreamingResponsesResponse, None]: processor = SimpleStreamingEventProcessor(tools=request.tools) - hide_stream_metadata = not request.include_reasoning and self.parser is not None + hide_stream_metadata = ( + not request.include_reasoning and context.response_parser is not None + ) def _get_logprobs(output: CompletionOutput) -> list[Logprob]: if not request.is_include_output_logprobs(): diff --git a/vllm/parser/abstract_parser.py b/vllm/parser/abstract_parser.py index 2a348697c7..eea957f90c 100644 --- a/vllm/parser/abstract_parser.py +++ b/vllm/parser/abstract_parser.py @@ -115,6 +115,7 @@ class Parser: tool_parser_cls: type[ToolParser] | None = None # Server-side floor for tool-call structural tags (--tool-strict-level). tool_strict_level: ToolStrictLevel = ToolStrictLevel.AUTO + always_adjust_request: bool = False def __init__( self, @@ -206,6 +207,10 @@ class Parser: """ return request + def set_prompt_token_ids(self, prompt_token_ids: Sequence[int]) -> None: + """Provide the exact rendered prompt to parsers that need prefix state.""" + return + @abstractmethod def is_reasoning_end(self, input_ids: list[int]) -> bool: """Check if the reasoning content ends in the input_ids. diff --git a/vllm/parser/parser_manager.py b/vllm/parser/parser_manager.py index 41b27967a6..719b396130 100644 --- a/vllm/parser/parser_manager.py +++ b/vllm/parser/parser_manager.py @@ -11,10 +11,13 @@ from vllm.tool_parsers.tool_strict_level import ToolStrictLevel if TYPE_CHECKING: from vllm.parser.abstract_parser import Parser from vllm.reasoning import ReasoningParser + from vllm.tokenizers import TokenizerLike from vllm.tool_parsers import ToolParser logger = init_logger(__name__) +HF_PARSER = "hf" + class ParserManager: """Provides a unified Parser by composing reasoning and tool parser adapters.""" @@ -79,6 +82,7 @@ class ParserManager: model_name: str | None = None, is_harmony: bool = False, tool_strict_level: str = "auto", + tokenizer: TokenizerLike | None = None, ) -> type[Parser] | None: """Get a Parser that handles both reasoning and tool parsing. @@ -93,6 +97,8 @@ class ParserManager: If True, HarmonyParser is always returned. tool_strict_level: Server-side floor for tool-call structural tags (``--tool-strict-level``). + tokenizer: Tokenizer whose `response_template` metadata is + validated when the `hf` parser is selected. Returns: A Parser class, or None if neither parser is specified. @@ -119,6 +125,40 @@ class ParserManager: HarmonyParser.tool_strict_level = strict_level return HarmonyParser + if HF_PARSER in (reasoning_parser_name, tool_parser_name): + if {reasoning_parser_name, tool_parser_name} - { + HF_PARSER, + None, + "", + }: + raise TypeError( + "The hf parser cannot be combined with other " + "reasoning or tool call parsers" + ) + from vllm.parser.response_template import ( + ResponseTemplateParser, + validate_tokenizer_response_template, + ) + + if tokenizer is not None: + validate_tokenizer_response_template( + tokenizer, + reasoning=reasoning_parser_cls is not None, + tools=tool_parser_cls is not None, + ) + + r_cls = reasoning_parser_cls + t_cls = tool_parser_cls + auto_tools = enable_auto_tools + + class _ResponseTemplateParser(ResponseTemplateParser): + reasoning_parser_cls = r_cls + tool_parser_cls = t_cls + tool_strict_level = strict_level + _enable_auto_tools = auto_tools + + return _ResponseTemplateParser + if reasoning_parser_name == "kimi_k3" or tool_parser_name == "kimi_k3": from vllm.parser.kimi_k3 import KimiK3Parser diff --git a/vllm/parser/response_template.py b/vllm/parser/response_template.py new file mode 100644 index 0000000000..985b76308a --- /dev/null +++ b/vllm/parser/response_template.py @@ -0,0 +1,642 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Unified vLLM parser backed by checkpoint `response_template` metadata.""" + +from __future__ import annotations + +import copy +import json +from collections.abc import Callable, Iterable, Sequence +from typing import TYPE_CHECKING, Any, cast + +from openai.types.responses import ToolChoiceFunction + +from vllm.entrypoints.chat_utils import make_tool_call_id +from vllm.entrypoints.generate.base.protocol import ( + DeltaFunctionCall, + DeltaMessage, + DeltaToolCall, + FunctionCall, +) +from vllm.entrypoints.openai.chat_completion.protocol import ( + ChatCompletionNamedToolChoiceParam, +) +from vllm.exceptions import VLLMValidationError +from vllm.logger import init_logger +from vllm.parser.abstract_parser import Parser +from vllm.parser.chat_parsing import ResponseParser +from vllm.parser.chat_parsing.response_templates import ( + ResponseTemplate, + load_response_template, +) +from vllm.reasoning.abs_reasoning_parsers import ReasoningParser +from vllm.tool_parsers.abstract_tool_parser import ToolParser +from vllm.tool_parsers.utils import iter_response_function_tool_dicts + +if TYPE_CHECKING: + from vllm.entrypoints.openai.chat_completion.protocol import ( + ChatCompletionRequest, + ) + from vllm.entrypoints.openai.responses.protocol import ResponsesRequest + from vllm.tokenizers import TokenizerLike + from vllm.tool_parsers.abstract_tool_parser import Tool + +logger = init_logger(__name__) + +THINKING_FIELD = "thinking" +CONTENT_FIELD = "content" +TOOL_FIELD = "tool_calls" +_SUPPORTED_FIELDS = frozenset({THINKING_FIELD, CONTENT_FIELD, TOOL_FIELD}) + + +def _tool_dict(tool: Any) -> dict[str, Any]: + if hasattr(tool, "model_dump"): + tool = tool.model_dump(exclude_none=True) + return tool if isinstance(tool, dict) else {} + + +def _as_tool_dicts(tools: Sequence[Any] | None) -> list[dict[str, Any]]: + """Responses function tools with flattened namespaces, plus chat tools.""" + tools = list(tools or ()) + result = iter_response_function_tool_dicts(tools) + result.extend( + tool_dict + for tool_dict in map(_tool_dict, tools) + if isinstance(tool_dict.get("function"), dict) + ) + return result + + +def _tool_is_strict(tool: Any) -> bool: + tool_dict = _tool_dict(tool) + function = tool_dict.get("function") + return bool( + tool_dict.get("strict") + or (isinstance(function, dict) and function.get("strict")) + ) + + +def _decode(tokenizer: TokenizerLike, token_ids: Sequence[int]) -> str: + try: + return cast(Any, tokenizer.decode)( + token_ids, + skip_special_tokens=False, + clean_up_tokenization_spaces=False, + spaces_between_special_tokens=False, + ) + except TypeError: + return tokenizer.decode(token_ids, skip_special_tokens=False) + + +def resolve_response_template( + tokenizer: Any | None, + template: dict[str, Any] | None = None, +) -> dict[str, Any] | None: + """Return `template` if given, otherwise the tokenizer's metadata.""" + if template is not None or tokenizer is None: + return template + template = getattr(tokenizer, "response_template", None) + if not isinstance(template, dict): + init_kwargs = getattr(tokenizer, "init_kwargs", None) or {} + template = init_kwargs.get("response_template") + return template if isinstance(template, dict) else None + + +def validate_response_template_for_serving( + template: dict[str, Any], +) -> ResponseTemplate: + """Validate the template and reject semantic fields serving cannot route.""" + loaded = load_response_template(template) + unsupported = (set(loaded.fields) - _SUPPORTED_FIELDS) | ( + set(loaded.defaults) - {"role"} + ) + if unsupported: + raise ValueError( + "response_template contains unsupported semantic fields: " + f"{sorted(unsupported)}. Supported fields are: " + f"{sorted(_SUPPORTED_FIELDS)}" + ) + for name in (THINKING_FIELD, CONTENT_FIELD): + field = loaded.fields.get(name) + if field is None: + continue + if ( + field.content != "text" + or field.transform is not None + or field.join is not None + ): + raise ValueError( + f"response_template field {name!r} uses semantics that cannot " + "be streamed by the OpenAI serving adapter" + ) + return loaded + + +def validate_tokenizer_response_template( + tokenizer: Any, + *, + reasoning: bool, + tools: bool, +) -> None: + """Fail at startup if the tokenizer cannot serve the requested parsing.""" + template = resolve_response_template(tokenizer) + if template is None: + raise TypeError( + "The hf parser requires `response_template` metadata " + "in the tokenizer configuration" + ) + try: + fields = validate_response_template_for_serving(template).fields + except (TypeError, ValueError) as exc: + raise TypeError(f"Invalid response_template metadata: {exc}") from exc + if reasoning and THINKING_FIELD not in fields: + raise TypeError( + "--reasoning-parser hf requires a response_template with a thinking field" + ) + if tools and TOOL_FIELD not in fields: + raise TypeError( + "--tool-call-parser hf requires a response_template with a tool_calls field" + ) + + +def _streaming_template(template: dict[str, Any]) -> ResponseTemplate: + result = copy.deepcopy(template) + fields = result["fields"] + if CONTENT_FIELD not in fields and all( + "open" in field or "open_pattern" in field for field in fields.values() + ): + fields[CONTENT_FIELD] = {} + for field in fields.values(): + field["optional"] = True + return load_response_template(result) + + +def _is_named_tool_choice(request: ChatCompletionRequest | ResponsesRequest) -> bool: + return isinstance( + request.tool_choice, (ToolChoiceFunction, ChatCompletionNamedToolChoiceParam) + ) + + +def _tool_parsing_enabled( + request: ChatCompletionRequest | ResponsesRequest, + *, + enable_auto_tools: bool, +) -> bool: + if request.tool_choice == "none": + return False + if request.tool_choice == "required" or _is_named_tool_choice(request): + return True + return enable_auto_tools and request.tool_choice in (None, "auto") + + +def _unsupported_tool_guarantee( + request: ChatCompletionRequest | ResponsesRequest, +) -> tuple[str, str] | None: + """The first requested tool guarantee this parser cannot enforce, with the + request parameter that asked for it.""" + if any(_tool_is_strict(tool) for tool in request.tools or ()): + return "strict tools", "tools" + if request.tool_choice == "required": + return "required tool choice", "tool_choice" + if _is_named_tool_choice(request): + return "named tool choice", "tool_choice" + if getattr(request, "parallel_tool_calls", True) is False: + return "parallel_tool_calls=False", "parallel_tool_calls" + return None + + +def _reasoning_has_ended( + parser: ResponseParser, + template: ResponseTemplate, + *, + thinking_disabled: bool = False, +) -> bool: + thinking = template.fields.get(THINKING_FIELD) + if thinking is None: + return True + if thinking.repeats: + return False + + active_field: str | None = None + thinking_closed = False + for event in parser.initial_events: + if event["type"] == "region_open": + active_field = event["field"] + elif event["type"] in {"region_close", "region_malformed"}: + thinking_closed |= event["field"] == THINKING_FIELD + active_field = None + if thinking_closed: + return True + if active_field in (CONTENT_FIELD, TOOL_FIELD): + return True + + # Resolve delimiters held at the prompt boundary. A NUL cannot occur in + # the model wire format and any implicit chunk containing it is a probe, + # not evidence that content generation has started. + for event in parser.feed("\0"): + if ( + event["type"] == "region_open" + and event["field"] in (CONTENT_FIELD, TOOL_FIELD) + and event["end"] > event["start"] + ): + return True + if event["field"] == THINKING_FIELD: + if event["type"] == "region_open": + return False + if event["type"] in {"region_close", "region_malformed"}: + return True + if active_field == THINKING_FIELD: + return False + return thinking_disabled + + +def _literal_ids( + tokenizer: TokenizerLike, vocab: dict[str, int], literals: Sequence[str] +) -> list[list[int]] | None: + """Token ids of each literal, or `None` when one encodes to nothing.""" + ids = [ + [vocab[literal]] + if literal in vocab + else tokenizer.encode(literal, add_special_tokens=False) + for literal in literals + ] + return ids if all(ids) else None + + +def _reasoning_end_ids( + tokenizer: TokenizerLike, vocab: dict[str, int], template: ResponseTemplate +) -> list[list[int]] | None: + """Token ids of each thinking closer, or `None` when they are not all + literal.""" + thinking = template.fields.get(THINKING_FIELD) + if thinking is None or thinking.repeats or not thinking.close_literals: + return None + return _literal_ids(tokenizer, vocab, thinking.close_literals) + + +def _ends_within(token_ids: Sequence[int], end_ids: list[int], num_new: int) -> bool: + """Whether `end_ids` completes within the last `num_new` of `token_ids`.""" + window = list(token_ids[max(0, len(token_ids) - num_new - len(end_ids) + 1) :]) + return any( + window[index : index + len(end_ids)] == end_ids + for index in range(len(window) - len(end_ids) + 1) + ) + + +def _serving_template( + tokenizer: Any, + template: dict[str, Any] | None, +) -> ResponseTemplate: + """The validated streaming form of `template` or the tokenizer's metadata.""" + template = resolve_response_template(tokenizer, template) + if template is None: + raise ValueError( + "The hf parser requires `response_template` " + "metadata in the tokenizer configuration" + ) + validate_response_template_for_serving(template) + return _streaming_template(template) + + +def _to_tool_calls(value: Any) -> list[tuple[str, str]]: + """`(name, arguments)` of each call in a tool region value, or nothing if + any call is incomplete.""" + values = value if isinstance(value, list) else [value] + calls: list[tuple[str, str]] = [] + for item in values: + function = item.get("function") if isinstance(item, dict) else None + if not isinstance(function, dict): + return [] + name = function.get("name") + arguments = function.get("arguments") + if not isinstance(name, str) or not name or arguments is None: + return [] + if not isinstance(arguments, str): + arguments = json.dumps(arguments, ensure_ascii=False) + calls.append((name, arguments)) + return calls + + +class ResponseTemplateReasoningParser(ReasoningParser): + """Reasoning end detection for `--reasoning-parser hf`. + + `ResponseTemplateParser` parses the output; this parser tells it and the + structured-output engine where reasoning ends. + """ + + def __init__( + self, + tokenizer: TokenizerLike, + *args, + response_template: dict[str, Any] | None = None, + **kwargs, + ) -> None: + super().__init__(tokenizer, *args, **kwargs) + self.response_template = _serving_template(tokenizer, response_template) + chat_template_kwargs = kwargs.get("chat_template_kwargs") or {} + self._thinking_disabled = not chat_template_kwargs.get("enable_thinking", True) + self._reasoning_end_ids = _reasoning_end_ids( + tokenizer, self.vocab, self.response_template + ) + + @property + def reasoning_start_str(self) -> str | None: + thinking = self.response_template.fields.get(THINKING_FIELD) + literals = thinking.open_literals if thinking else None + return literals[0] if literals else None + + @property + def reasoning_end_str(self) -> str | None: + thinking = self.response_template.fields.get(THINKING_FIELD) + literals = thinking.close_literals if thinking else None + return literals[0] if literals else None + + def is_reasoning_end(self, input_ids: Sequence[int]) -> bool: + try: + parser = ResponseParser( + self.response_template, + prefix=_decode(self.model_tokenizer, input_ids), + ) + except Exception: + return False + return _reasoning_has_ended( + parser, + self.response_template, + thinking_disabled=self._thinking_disabled, + ) + + def is_reasoning_end_streaming( + self, input_ids: Sequence[int], delta_ids: Iterable[int] + ) -> bool: + if self._reasoning_end_ids is None: + return self.is_reasoning_end(input_ids) + num_new = len(list(delta_ids)) + return any( + _ends_within(input_ids, end_ids, num_new) + for end_ids in self._reasoning_end_ids + ) + + def extract_content_ids(self, input_ids: list[int]) -> list[int]: + raise NotImplementedError("ResponseTemplateParser extracts content") + + def extract_reasoning( + self, + model_output: str, + request: ChatCompletionRequest | ResponsesRequest, + ) -> tuple[str | None, str | None]: + raise NotImplementedError("ResponseTemplateParser extracts reasoning") + + def extract_reasoning_streaming( + self, + previous_text: str, + current_text: str, + delta_text: str, + previous_token_ids: Sequence[int], + current_token_ids: Sequence[int], + delta_token_ids: Sequence[int], + ) -> DeltaMessage | None: + raise NotImplementedError("ResponseTemplateParser extracts reasoning") + + +class ResponseTemplateToolParser(ToolParser): + """Registers `--tool-call-parser hf`; `ResponseTemplateParser` extracts the + tool calls.""" + + +class _DeltaBuilder: + """Accumulates one `DeltaMessage`.""" + + def __init__(self) -> None: + self.reasoning: list[str] = [] + self.content: list[str] = [] + self.tool_calls: list[DeltaToolCall] = [] + + def build(self) -> DeltaMessage | None: + fields: dict[str, Any] = {} + if reasoning := "".join(self.reasoning): + fields["reasoning"] = reasoning + if content := "".join(self.content): + fields["content"] = content + if self.tool_calls: + fields["tool_calls"] = self.tool_calls + return DeltaMessage(**fields) if fields else None + + +class _ResponseStream: + """One generation fed through `ResponseParser`, routed to message deltas.""" + + def __init__( + self, + template: ResponseTemplate, + *, + prefix: str, + tools: list[dict[str, Any]], + parse_reasoning: bool, + include_reasoning: bool, + parse_tools: bool, + new_tool_call_id: Callable[[str], str], + ) -> None: + self.parse_reasoning = parse_reasoning + self.include_reasoning = include_reasoning + self.parse_tools = parse_tools + self.new_tool_call_id = new_tool_call_id + self.parser = ResponseParser(template, prefix=prefix, tools=tools) + self._prefix_end = len(self.parser.input_text) + self._next_tool_index = 0 + self._finalized = False + + def feed(self, text: str, *, finished: bool) -> DeltaMessage | None: + if self._finalized: + return None + delta = _DeltaBuilder() + self._route(self.parser.feed(text), delta) + if finished: + self._finalized = True + if len(self.parser.input_text) > self._prefix_end: + _, events = self.parser.finalize() + self._route(events, delta) + return delta.build() + + def _route(self, events: Sequence[dict[str, Any]], delta: _DeltaBuilder) -> None: + for event in events: + field = event.get("field") + if field == TOOL_FIELD: + self._route_tool_event(event, delta) + elif event["type"] == "region_chunk": + if field == THINKING_FIELD and self.parse_reasoning: + if self.include_reasoning: + delta.reasoning.append(event["text"]) + elif field in (THINKING_FIELD, CONTENT_FIELD): + delta.content.append(event["text"]) + + def _route_tool_event(self, event: dict[str, Any], delta: _DeltaBuilder) -> None: + """Emit the calls of a tool region once it ends and parses; a region cut + off at the end of the stream is parsed as generated. With tool parsing + disabled, the region passes through as content.""" + if not self.parse_tools: + if event["type"] == "region_chunk": + delta.content.append(event["text"]) + else: + start = max(event["start"], self._prefix_end) + delta.content.append(self.parser.input_text[start : event["end"]]) + return + if event["type"] == "region_malformed": + logger.warning("response_template: dropping a malformed tool call") + elif event["type"] == "region_close": + calls = _to_tool_calls(event.get("value")) + if not calls: + logger.warning("response_template: dropping an incomplete tool call") + for name, arguments in calls: + delta.tool_calls.append( + DeltaToolCall( + index=self._next_tool_index, + id=self.new_tool_call_id(name), + type="function", + function=DeltaFunctionCall(name=name, arguments=arguments), + ) + ) + self._next_tool_index += 1 + + +class ResponseTemplateParser(Parser): + """Parse reasoning, content, and tool calls with one response template.""" + + reasoning_parser_cls: type[ReasoningParser] | None = ResponseTemplateReasoningParser + tool_parser_cls: type[ToolParser] | None = ResponseTemplateToolParser + always_adjust_request = True + _enable_auto_tools = False + + def __init__( + self, + tokenizer: TokenizerLike, + tools: list[Tool] | None = None, + *, + response_template: dict[str, Any] | None = None, + enable_auto_tools: bool | None = None, + **kwargs, + ) -> None: + self.response_template = _serving_template(tokenizer, response_template) + super().__init__( + tokenizer, tools, response_template=response_template, **kwargs + ) + if enable_auto_tools is not None: + self._enable_auto_tools = enable_auto_tools + self._tools = _as_tool_dicts(tools) + self._prompt_token_ids: list[int] | None = None + self._prefix = "" + self._stream: _ResponseStream | None = None + + def set_prompt_token_ids(self, prompt_token_ids: Sequence[int]) -> None: + prompt_token_ids = list(prompt_token_ids) + if prompt_token_ids == self._prompt_token_ids: + return + self._prompt_token_ids = prompt_token_ids + self._prefix = _decode(self.model_tokenizer, prompt_token_ids) + self._stream = None + + def adjust_request( + self, + request: ChatCompletionRequest | ResponsesRequest, + ) -> ChatCompletionRequest | ResponsesRequest: + if getattr(request, "chat_template", None) is not None: + logger.warning_once( + "A request chat template is set; response_template parsing still " + "expects the checkpoint's output format." + ) + request.skip_special_tokens = False + if hasattr(request, "spaces_between_special_tokens"): + request.spaces_between_special_tokens = False + if ( + self._tool_parser is not None + and request.tools + and request.tool_choice != "none" + and (unsupported := _unsupported_tool_guarantee(request)) is not None + ): + guarantee, parameter = unsupported + raise VLLMValidationError( + f"response_template parsing cannot guarantee {guarantee} " + "without a format-compatible structural grammar", + parameter=parameter, + ) + return request + + def is_reasoning_end(self, input_ids: list[int]) -> bool: + return ( + self._reasoning_parser is None + or self._reasoning_parser.is_reasoning_end(input_ids) + ) + + def parse( + self, + model_output: str, + request: ChatCompletionRequest | ResponsesRequest, + enable_auto_tools: bool = False, + model_output_token_ids: Sequence[int] = (), + ) -> tuple[str | None, str | None, list[FunctionCall] | None]: + self._initialize_history_tool_call_cnt(request) + stream = self._new_stream( + request, + enable_auto_tools=enable_auto_tools, + include_reasoning=True, + ) + delta = stream.feed(model_output, finished=True) + if delta is None: + return None, None, None + tool_calls = [ + FunctionCall( + id=call.id, + name=call.function.name or "", + arguments=call.function.arguments or "{}", + ) + for call in delta.tool_calls + if call.function is not None + ] + return delta.reasoning, delta.content, tool_calls or None + + def parse_delta( + self, + delta_text: str, + delta_token_ids: list[int], + request: ChatCompletionRequest | ResponsesRequest, + prompt_token_ids: list[int] | None = None, + *, + finished: bool, + ) -> DeltaMessage | None: + self._initialize_history_tool_call_cnt(request) + if self._prompt_token_ids is None and prompt_token_ids is not None: + self.set_prompt_token_ids(prompt_token_ids) + if self._stream is None: + self._stream = self._new_stream( + request, + enable_auto_tools=self._enable_auto_tools, + include_reasoning=request.include_reasoning, + ) + return self._stream.feed(delta_text, finished=finished) + + def _new_stream( + self, + request: ChatCompletionRequest | ResponsesRequest, + *, + enable_auto_tools: bool, + include_reasoning: bool, + ) -> _ResponseStream: + return _ResponseStream( + self.response_template, + prefix=self._prefix, + tools=self._tools, + parse_reasoning=self._reasoning_parser is not None, + include_reasoning=include_reasoning, + parse_tools=self._tool_parser is not None + and _tool_parsing_enabled(request, enable_auto_tools=enable_auto_tools), + new_tool_call_id=self._new_tool_call_id, + ) + + def _new_tool_call_id(self, name: str) -> str: + state = self._stream_state + tool_call_id = make_tool_call_id( + id_type=state.tool_call_id_type, + func_name=name, + idx=state.history_tool_call_cnt, + ) + state.history_tool_call_cnt += 1 + return tool_call_id diff --git a/vllm/reasoning/__init__.py b/vllm/reasoning/__init__.py index 9a4413cbd9..f9ab1dc9a1 100644 --- a/vllm/reasoning/__init__.py +++ b/vllm/reasoning/__init__.py @@ -80,6 +80,10 @@ _REASONING_PARSERS_TO_REGISTER = { "granite_thinking_engine_reasoning_parser", "GraniteThinkingParserReasoningAdapter", ), + "hf": ( + "response_template_reasoning_parser", + "ResponseTemplateReasoningParser", + ), "holo2": ( "deepseek_v3_reasoning_parser", "DeepSeekV3ReasoningWithThinkingParser", diff --git a/vllm/reasoning/response_template_reasoning_parser.py b/vllm/reasoning/response_template_reasoning_parser.py new file mode 100644 index 0000000000..757dd73417 --- /dev/null +++ b/vllm/reasoning/response_template_reasoning_parser.py @@ -0,0 +1,6 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +from vllm.parser.response_template import ResponseTemplateReasoningParser + +__all__ = ["ResponseTemplateReasoningParser"] diff --git a/vllm/renderers/online_derenderer.py b/vllm/renderers/online_derenderer.py index e1f67397ce..c1f175798d 100644 --- a/vllm/renderers/online_derenderer.py +++ b/vllm/renderers/online_derenderer.py @@ -79,6 +79,7 @@ class OnlineDerenderer: tool_strict_level=tool_strict_level, model_name=model_config.model, is_harmony=self.use_harmony, + tokenizer=renderer.tokenizer, ) self.chat_template = chat_template @@ -160,6 +161,8 @@ class OnlineDerenderer: chat_template_kwargs=chat_template_kwargs, model_config=self.model_config, ) + if generate_response.prompt_token_ids is not None: + parser.set_prompt_token_ids(generate_response.prompt_token_ids) reasoning, content, tool_calls = parser.parse( decoded_text, chat_request, diff --git a/vllm/renderers/online_renderer.py b/vllm/renderers/online_renderer.py index ec8b58f73d..23352ef1cf 100644 --- a/vllm/renderers/online_renderer.py +++ b/vllm/renderers/online_renderer.py @@ -183,6 +183,7 @@ class OnlineRenderer: tool_strict_level=tool_strict_level, model_name=model_config.model, is_harmony=self.use_harmony, + tokenizer=renderer.tokenizer, ) self.chat_template = chat_template @@ -825,7 +826,8 @@ class OnlineRenderer: and tokenizer.supports_grammar ) should_adjust_request = ( - parser.reasoning_parser_cls is not None + parser.always_adjust_request + or parser.reasoning_parser_cls is not None or tool_choice != "none" or is_mistral_grammar_eligible ) diff --git a/vllm/tool_parsers/__init__.py b/vllm/tool_parsers/__init__.py index 5647a75291..8446aae815 100644 --- a/vllm/tool_parsers/__init__.py +++ b/vllm/tool_parsers/__init__.py @@ -86,6 +86,10 @@ _TOOL_PARSERS_TO_REGISTER = { "hermes_tool_parser", "Hermes2ProToolParser", ), + "hf": ( + "response_template_tool_parser", + "ResponseTemplateToolParser", + ), "poolside_v1": ( "poolside_v1_tool_parser", "PoolsideV1ToolParser", diff --git a/vllm/tool_parsers/response_template_tool_parser.py b/vllm/tool_parsers/response_template_tool_parser.py new file mode 100644 index 0000000000..32b4b44de9 --- /dev/null +++ b/vllm/tool_parsers/response_template_tool_parser.py @@ -0,0 +1,6 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +from vllm.parser.response_template import ResponseTemplateToolParser + +__all__ = ["ResponseTemplateToolParser"]