[Feature][Frontend] Add granite_thinking_parser reasoning parser for Granite 4.2 (#55957)
Signed-off-by: Yousaf shah <yousaf.shah@gmail.com> Co-authored-by: sfeng33 <4florafeng@gmail.com>
This commit is contained in:
@@ -25,12 +25,14 @@ vLLM currently supports the following reasoning models:
|
||||
| [Holo2 series](https://huggingface.co/collections/Hcompany/holo2) | `holo2` | `json`, `regex` | ✅ |
|
||||
| [Hunyuan A13B series](https://huggingface.co/collections/tencent/hunyuan-a13b-685ec38e5b46321e3ea7c4be) | `hunyuan_a13b` | `json`, `regex` | ✅ |
|
||||
| [IBM Granite 3.2 language models](https://huggingface.co/collections/ibm-granite/granite-32-language-models-67b3bc8c13508f6d064cff9a) | `granite` | ❌ | ❌ |
|
||||
| [IBM Granite 4.2 language models](https://huggingface.co/ibm-granite/granite-4.2-30b) | `granite_thinking_parser` | `json`, `regex` | ✅ |
|
||||
| [MiniMax-M2](https://huggingface.co/MiniMaxAI/MiniMax-M2) | `minimax_m2_append_think` | `json`, `regex` | ✅ |
|
||||
| [Qwen3 series](https://huggingface.co/collections/Qwen/qwen3-67dd247413f0e2e4f653967f) | `qwen3` | `json`, `regex` | ✅ |
|
||||
| [QwQ-32B](https://huggingface.co/Qwen/QwQ-32B) | `deepseek_r1` | `json`, `regex` | ✅ |
|
||||
|
||||
!!! note
|
||||
IBM Granite 3.2 and DeepSeek-V3.1 reasoning is disabled by default; to enable it, you must also pass `thinking=True` in your `chat_template_kwargs`.
|
||||
IBM Granite 4.2 reasoning is enabled by default. To disable it, you must pass `enable_thinking=False` in your `chat_template_kwargs`.
|
||||
The reasoning feature for the Qwen3 series is enabled by default. To disable it, you must pass `enable_thinking=False` in your `chat_template_kwargs`.
|
||||
Gemma 4 reasoning is disabled by default; to enable it, pass `enable_thinking=True` in your `chat_template_kwargs` or set `reasoning_effort` (which enables it automatically).
|
||||
DeepSeek-V3.1 tool calling is supported in non-thinking mode.
|
||||
|
||||
@@ -34,6 +34,7 @@ class Sample:
|
||||
tools: list[dict] | None = None
|
||||
chat_template_kwargs: dict | None = None
|
||||
prompt_token_ids: list[int] | None = None
|
||||
content_lstrip: str | None = None
|
||||
|
||||
|
||||
@dataclass
|
||||
|
||||
@@ -327,7 +327,18 @@ _TOOL_CALL_SAMPLES = [
|
||||
def _tool_suppression_expectations(
|
||||
sample, think_end: str, tool_start: str, *, include_tool_block: bool
|
||||
) -> tuple[str, str]:
|
||||
"""Expected (reasoning, content) when tool calls are not extracted.
|
||||
reasoning, content = _raw_tool_suppression_expectations(
|
||||
sample, think_end, tool_start, include_tool_block=include_tool_block
|
||||
)
|
||||
if sample.content_lstrip:
|
||||
content = content.lstrip(sample.content_lstrip)
|
||||
return reasoning, content
|
||||
|
||||
|
||||
def _raw_tool_suppression_expectations(
|
||||
sample, think_end: str, tool_start: str, *, include_tool_block: bool
|
||||
) -> tuple[str, str]:
|
||||
"""Expected (reasoning, content) from the raw token text.
|
||||
|
||||
With ``include_tool_block=True`` (skip_tool_parsing / reasoning
|
||||
adapter first pass), tool terminal text is preserved as content so
|
||||
|
||||
@@ -15,7 +15,7 @@ from __future__ import annotations
|
||||
|
||||
import functools
|
||||
import json
|
||||
from dataclasses import dataclass
|
||||
from dataclasses import dataclass, replace
|
||||
from typing import Any
|
||||
|
||||
from tests.parser.engine.replay_harness import (
|
||||
@@ -35,6 +35,7 @@ from vllm.parser.engine.registered_adapters import (
|
||||
Gemma4Parser,
|
||||
Glm47MoeParser,
|
||||
GraniteParser,
|
||||
GraniteThinkingParser,
|
||||
InklingParser,
|
||||
KimiK2Parser,
|
||||
MinimaxM2Parser,
|
||||
@@ -594,6 +595,30 @@ def _build_nemotron_v3(scenario: Scenario, validate: bool = True) -> Sample:
|
||||
)
|
||||
|
||||
|
||||
def _build_granite_thinking(scenario: Scenario, validate: bool = True) -> Sample:
|
||||
"""Granite 4.2: the Nemotron V3 grammar plus leading-newline stripping on
|
||||
content after ``</think>``."""
|
||||
sample = _build_qwen3(
|
||||
scenario,
|
||||
name="granite_thinking_parser",
|
||||
parser_cls=GraniteThinkingParser,
|
||||
strip_trailing_ws=True,
|
||||
validate=False,
|
||||
)
|
||||
sample = replace(
|
||||
sample,
|
||||
expected_content=(
|
||||
sample.expected_content.lstrip("\n")
|
||||
if sample.expected_content
|
||||
else sample.expected_content
|
||||
),
|
||||
content_lstrip="\n",
|
||||
)
|
||||
if validate:
|
||||
_validate_sample(sample, GraniteThinkingParser)
|
||||
return sample
|
||||
|
||||
|
||||
# ── Seed-OSS (Qwen3 XML grammar with Seed wrapper tokens) ────────────
|
||||
|
||||
_SEED_OSS_VOCAB: dict[str, int] = {
|
||||
@@ -1136,6 +1161,7 @@ _BUILDERS: dict[str, Any] = {
|
||||
"granite": _build_granite,
|
||||
"minimax_m2": _build_minimax_m2,
|
||||
"nemotron_v3": _build_nemotron_v3,
|
||||
"granite_thinking_parser": _build_granite_thinking,
|
||||
"seed_oss": _build_seed_oss,
|
||||
"glm47_moe": _build_glm47_moe,
|
||||
"kimi_k2": _build_kimi_k2,
|
||||
|
||||
@@ -0,0 +1,643 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from typing import TypedDict
|
||||
|
||||
import pytest
|
||||
import regex as re
|
||||
|
||||
from tests.reasoning.utils import run_reasoning_extraction
|
||||
from vllm.entrypoints.openai.chat_completion.protocol import (
|
||||
ChatCompletionRequest,
|
||||
)
|
||||
from vllm.parser.abstract_parser import DelegatingParser
|
||||
from vllm.parser.engine.registered_adapters import (
|
||||
GraniteThinkingParserReasoningAdapter,
|
||||
)
|
||||
from vllm.reasoning import ReasoningParser, ReasoningParserManager
|
||||
|
||||
parser_name = "granite_thinking_parser"
|
||||
|
||||
|
||||
class ReasoningCase(TypedDict):
|
||||
output: str
|
||||
reasoning: str | None
|
||||
content: str | None
|
||||
|
||||
|
||||
class GraniteThinkingTokenizer:
|
||||
def __init__(self):
|
||||
self._vocab = {
|
||||
"<think>": 1,
|
||||
"</think>": 2,
|
||||
}
|
||||
self._inv_vocab = {v: k for k, v in self._vocab.items()}
|
||||
self._pattern = re.compile(r"(<think>|</think>)")
|
||||
|
||||
def get_vocab(self) -> dict[str, int]:
|
||||
return self._vocab
|
||||
|
||||
def tokenize(self, text: str) -> list[str]:
|
||||
tokens: list[str] = []
|
||||
for part in self._pattern.split(text):
|
||||
if part:
|
||||
tokens.append(part)
|
||||
return tokens
|
||||
|
||||
def convert_tokens_to_string(self, tokens: list[str]) -> str:
|
||||
return "".join(tokens)
|
||||
|
||||
def decode(self, token_ids: list[int]) -> str:
|
||||
return "".join(self._inv_vocab.get(tid, f"<unk:{tid}>") for tid in token_ids)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def tokenizer():
|
||||
return GraniteThinkingTokenizer()
|
||||
|
||||
|
||||
# ── Basic reasoning extraction (non-streaming + streaming) ───────────
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"streaming,param_dict",
|
||||
[
|
||||
pytest.param(
|
||||
False,
|
||||
{
|
||||
"output": "<think>reasoning</think>\nHello",
|
||||
"reasoning": "reasoning",
|
||||
"content": "Hello",
|
||||
},
|
||||
id="leading_newline_stripped",
|
||||
),
|
||||
pytest.param(
|
||||
True,
|
||||
{
|
||||
"output": "<think>reasoning</think>\nHello",
|
||||
"reasoning": "reasoning",
|
||||
"content": "Hello",
|
||||
},
|
||||
id="leading_newline_stripped_streaming",
|
||||
),
|
||||
pytest.param(
|
||||
False,
|
||||
{
|
||||
"output": "<think>r</think>c",
|
||||
"reasoning": "r",
|
||||
"content": "c",
|
||||
},
|
||||
id="simple_reasoning",
|
||||
),
|
||||
pytest.param(
|
||||
True,
|
||||
{
|
||||
"output": "<think>r</think>c",
|
||||
"reasoning": "r",
|
||||
"content": "c",
|
||||
},
|
||||
id="simple_reasoning_streaming",
|
||||
),
|
||||
pytest.param(
|
||||
False,
|
||||
{
|
||||
"output": "This is a reasoning section</think>This is the rest",
|
||||
"reasoning": "This is a reasoning section",
|
||||
"content": "This is the rest",
|
||||
},
|
||||
id="without_start_token",
|
||||
),
|
||||
pytest.param(
|
||||
True,
|
||||
{
|
||||
"output": "This is a reasoning section</think>This is the rest",
|
||||
"reasoning": "This is a reasoning section",
|
||||
"content": "This is the rest",
|
||||
},
|
||||
id="without_start_token_streaming",
|
||||
),
|
||||
pytest.param(
|
||||
False,
|
||||
{
|
||||
"output": "<think>This is a reasoning section</think>This is the rest", # noqa: E501
|
||||
"reasoning": "This is a reasoning section",
|
||||
"content": "This is the rest",
|
||||
},
|
||||
id="with_start_token",
|
||||
),
|
||||
pytest.param(
|
||||
True,
|
||||
{
|
||||
"output": "<think>This is a reasoning section</think>This is the rest", # noqa: E501
|
||||
"reasoning": "This is a reasoning section",
|
||||
"content": "This is the rest",
|
||||
},
|
||||
id="with_start_token_streaming",
|
||||
),
|
||||
pytest.param(
|
||||
False,
|
||||
{
|
||||
"output": "<think>reasoning</think>\n\n\nHello",
|
||||
"reasoning": "reasoning",
|
||||
"content": "Hello",
|
||||
},
|
||||
id="multiple_leading_newlines_stripped",
|
||||
),
|
||||
pytest.param(
|
||||
True,
|
||||
{
|
||||
"output": "<think>reasoning</think>\n\n\nHello",
|
||||
"reasoning": "reasoning",
|
||||
"content": "Hello",
|
||||
},
|
||||
id="multiple_leading_newlines_stripped_streaming",
|
||||
),
|
||||
pytest.param(
|
||||
False,
|
||||
{
|
||||
"output": "<think>line1\nline2</think>\nresult1\nresult2",
|
||||
"reasoning": "line1\nline2",
|
||||
"content": "result1\nresult2",
|
||||
},
|
||||
id="multiline_reasoning_and_content",
|
||||
),
|
||||
pytest.param(
|
||||
True,
|
||||
{
|
||||
"output": "<think>line1\nline2</think>\nresult1\nresult2",
|
||||
"reasoning": "line1\nline2",
|
||||
"content": "result1\nresult2",
|
||||
},
|
||||
id="multiline_reasoning_and_content_streaming",
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_granite_thinking_reasoning(
|
||||
tokenizer: GraniteThinkingTokenizer,
|
||||
streaming: bool,
|
||||
param_dict: ReasoningCase,
|
||||
):
|
||||
output = tokenizer.tokenize(param_dict["output"])
|
||||
model_output = [tokenizer.convert_tokens_to_string([token]) for token in output]
|
||||
parser: ReasoningParser = ReasoningParserManager.get_reasoning_parser(parser_name)(
|
||||
tokenizer
|
||||
)
|
||||
|
||||
reasoning, content = run_reasoning_extraction(
|
||||
parser, model_output, streaming=streaming
|
||||
)
|
||||
|
||||
assert reasoning == param_dict["reasoning"]
|
||||
assert content == param_dict["content"]
|
||||
|
||||
|
||||
# ── No content after end token ───────────────────────────────────────
|
||||
|
||||
|
||||
def test_granite_thinking_no_content_after_end_token(
|
||||
tokenizer: GraniteThinkingTokenizer,
|
||||
):
|
||||
parser_cls = ReasoningParserManager.get_reasoning_parser(parser_name)
|
||||
parser = parser_cls(tokenizer)
|
||||
|
||||
reasoning, content = run_reasoning_extraction(
|
||||
parser,
|
||||
["<think>", "reasoning", "</think>"],
|
||||
streaming=False,
|
||||
)
|
||||
|
||||
assert reasoning == "reasoning"
|
||||
assert content is None
|
||||
|
||||
|
||||
# ── Whitespace-only content after end token ──────────────────────────
|
||||
|
||||
|
||||
@pytest.mark.parametrize("streaming", [False, True])
|
||||
def test_granite_thinking_whitespace_only_content(
|
||||
tokenizer: GraniteThinkingTokenizer,
|
||||
streaming: bool,
|
||||
):
|
||||
parser_cls = ReasoningParserManager.get_reasoning_parser(parser_name)
|
||||
parser = parser_cls(tokenizer)
|
||||
|
||||
reasoning, content = run_reasoning_extraction(
|
||||
parser,
|
||||
["<think>", "reasoning", "</think>", "\n\n"],
|
||||
streaming=streaming,
|
||||
)
|
||||
|
||||
assert reasoning == "reasoning"
|
||||
assert content is None
|
||||
|
||||
|
||||
# ── Unterminated think block ─────────────────────────────────────────
|
||||
|
||||
|
||||
def test_granite_thinking_unterminated_think_block(
|
||||
tokenizer: GraniteThinkingTokenizer,
|
||||
):
|
||||
parser_cls = ReasoningParserManager.get_reasoning_parser(parser_name)
|
||||
parser = parser_cls(tokenizer)
|
||||
request = ChatCompletionRequest(
|
||||
model="test-model",
|
||||
messages=[],
|
||||
chat_template_kwargs={"enable_thinking": True},
|
||||
)
|
||||
|
||||
reasoning, content = run_reasoning_extraction(
|
||||
parser,
|
||||
["<think>", "reasoning only"],
|
||||
request=request,
|
||||
streaming=False,
|
||||
)
|
||||
|
||||
assert reasoning == "reasoning only"
|
||||
assert content is None
|
||||
|
||||
|
||||
# ── enable_thinking=False ────────────────────────────────────────────
|
||||
|
||||
|
||||
def test_granite_thinking_disabled_moves_into_content(
|
||||
tokenizer: GraniteThinkingTokenizer,
|
||||
):
|
||||
parser_cls = ReasoningParserManager.get_reasoning_parser(parser_name)
|
||||
parser = parser_cls(tokenizer)
|
||||
request = ChatCompletionRequest(
|
||||
model="test-model",
|
||||
messages=[],
|
||||
chat_template_kwargs={"enable_thinking": False},
|
||||
)
|
||||
|
||||
reasoning, content = run_reasoning_extraction(
|
||||
parser,
|
||||
["This is plain content"],
|
||||
request=request,
|
||||
streaming=False,
|
||||
)
|
||||
|
||||
assert reasoning is None
|
||||
assert content == "This is plain content"
|
||||
|
||||
|
||||
# ── enable_thinking=False + leading newline (§6.2 ordering) ─────────
|
||||
|
||||
|
||||
def test_granite_thinking_disabled_with_leading_newline(
|
||||
tokenizer: GraniteThinkingTokenizer,
|
||||
):
|
||||
# With enable_thinking=False, model output goes through the swap
|
||||
# path: content is initially None (all text classified as
|
||||
# reasoning), so lstrip doesn't fire pre-swap. After the swap,
|
||||
# any leading \n in the original output is preserved. In practice,
|
||||
# enable_thinking=False output has no template-injected \n, so
|
||||
# this is a correctness check, not a realistic scenario.
|
||||
parser_cls = ReasoningParserManager.get_reasoning_parser(parser_name)
|
||||
parser = parser_cls(tokenizer)
|
||||
request = ChatCompletionRequest(
|
||||
model="test-model",
|
||||
messages=[],
|
||||
chat_template_kwargs={"enable_thinking": False},
|
||||
)
|
||||
|
||||
reasoning, content = run_reasoning_extraction(
|
||||
parser,
|
||||
["\nThis is plain content"],
|
||||
request=request,
|
||||
streaming=False,
|
||||
)
|
||||
|
||||
assert reasoning is None
|
||||
assert content == "\nThis is plain content"
|
||||
|
||||
|
||||
# ── force_nonempty_content=True ──────────────────────────────────────
|
||||
|
||||
|
||||
def test_granite_thinking_force_nonempty_content_moves_into_content(
|
||||
tokenizer: GraniteThinkingTokenizer,
|
||||
):
|
||||
parser_cls = ReasoningParserManager.get_reasoning_parser(parser_name)
|
||||
parser = parser_cls(tokenizer)
|
||||
request = ChatCompletionRequest(
|
||||
model="test-model",
|
||||
messages=[],
|
||||
chat_template_kwargs={"force_nonempty_content": True},
|
||||
)
|
||||
|
||||
reasoning, content = run_reasoning_extraction(
|
||||
parser,
|
||||
["<think>", "This is plain content"],
|
||||
request=request,
|
||||
streaming=False,
|
||||
)
|
||||
|
||||
assert reasoning is None
|
||||
assert content == "This is plain content"
|
||||
|
||||
|
||||
def test_granite_thinking_force_nonempty_no_swap_when_newlines_only(
|
||||
tokenizer: GraniteThinkingTokenizer,
|
||||
):
|
||||
# When </think> IS present and content is newlines-only, lstrip
|
||||
# removes them but the swap should NOT fire — content was present,
|
||||
# just whitespace. Matches the HF plugin behavior.
|
||||
parser_cls = ReasoningParserManager.get_reasoning_parser(parser_name)
|
||||
parser = parser_cls(tokenizer)
|
||||
request = ChatCompletionRequest(
|
||||
model="test-model",
|
||||
messages=[],
|
||||
chat_template_kwargs={"force_nonempty_content": True},
|
||||
)
|
||||
|
||||
reasoning, content = run_reasoning_extraction(
|
||||
parser,
|
||||
["<think>", "reasoning", "</think>", "\n\n"],
|
||||
request=request,
|
||||
streaming=False,
|
||||
)
|
||||
|
||||
assert reasoning == "reasoning"
|
||||
assert content is None
|
||||
|
||||
|
||||
def test_granite_thinking_force_nonempty_swaps_when_content_absent(
|
||||
tokenizer: GraniteThinkingTokenizer,
|
||||
):
|
||||
# When </think> IS present but content is truly absent (zero
|
||||
# characters after </think>, e.g. max_tokens cut), swap fires.
|
||||
parser_cls = ReasoningParserManager.get_reasoning_parser(parser_name)
|
||||
parser = parser_cls(tokenizer)
|
||||
request = ChatCompletionRequest(
|
||||
model="test-model",
|
||||
messages=[],
|
||||
chat_template_kwargs={"force_nonempty_content": True},
|
||||
)
|
||||
|
||||
reasoning, content = run_reasoning_extraction(
|
||||
parser,
|
||||
["<think>", "reasoning", "</think>"],
|
||||
request=request,
|
||||
streaming=False,
|
||||
)
|
||||
|
||||
assert reasoning is None
|
||||
assert content == "reasoning"
|
||||
|
||||
|
||||
def test_granite_thinking_force_nonempty_keeps_real_content(
|
||||
tokenizer: GraniteThinkingTokenizer,
|
||||
):
|
||||
parser_cls = ReasoningParserManager.get_reasoning_parser(parser_name)
|
||||
parser = parser_cls(tokenizer)
|
||||
request = ChatCompletionRequest(
|
||||
model="test-model",
|
||||
messages=[],
|
||||
chat_template_kwargs={"force_nonempty_content": True},
|
||||
)
|
||||
|
||||
reasoning, content = run_reasoning_extraction(
|
||||
parser,
|
||||
["<think>", "reasoning here", "</think>", "real answer"],
|
||||
request=request,
|
||||
streaming=False,
|
||||
)
|
||||
|
||||
assert reasoning == "reasoning here"
|
||||
assert content == "real answer"
|
||||
|
||||
|
||||
# ── Truncated reasoning with thinking on ─────────────────────────────
|
||||
|
||||
|
||||
def test_granite_thinking_keeps_truncated_reasoning(
|
||||
tokenizer: GraniteThinkingTokenizer,
|
||||
):
|
||||
parser_cls = ReasoningParserManager.get_reasoning_parser(parser_name)
|
||||
parser = parser_cls(tokenizer)
|
||||
request = ChatCompletionRequest(
|
||||
model="test-model",
|
||||
messages=[],
|
||||
chat_template_kwargs={"enable_thinking": True},
|
||||
)
|
||||
|
||||
reasoning, content = run_reasoning_extraction(
|
||||
parser,
|
||||
["This is truncated reasoning"],
|
||||
request=request,
|
||||
streaming=False,
|
||||
)
|
||||
|
||||
assert reasoning == "This is truncated reasoning"
|
||||
assert content is None
|
||||
|
||||
|
||||
# ── DelegatingParser / parse_delta streaming tests ───────────────────
|
||||
|
||||
_SPECIAL_TOKEN_IDS = {"<think>": 1, "</think>": 2}
|
||||
|
||||
|
||||
def _token_id(token: str) -> int:
|
||||
return _SPECIAL_TOKEN_IDS.get(token, 0)
|
||||
|
||||
|
||||
def _make_reasoning_parser(tokenizer):
|
||||
class _GraniteThinkingDelegating(DelegatingParser):
|
||||
reasoning_parser_cls = GraniteThinkingParserReasoningAdapter
|
||||
tool_parser_cls = None
|
||||
|
||||
return _GraniteThinkingDelegating(tokenizer)
|
||||
|
||||
|
||||
def _run_parse_delta(parser, tokenizer, text, request):
|
||||
tokens = tokenizer.tokenize(text)
|
||||
reasoning_parts: list[str] = []
|
||||
content_parts: list[str] = []
|
||||
for i, token in enumerate(tokens):
|
||||
delta = parser.parse_delta(
|
||||
delta_text=token,
|
||||
delta_token_ids=[_token_id(token)],
|
||||
request=request,
|
||||
prompt_token_ids=[] if i == 0 else None,
|
||||
finished=(i == len(tokens) - 1),
|
||||
)
|
||||
if delta is None:
|
||||
continue
|
||||
if delta.reasoning:
|
||||
reasoning_parts.append(delta.reasoning)
|
||||
if delta.content:
|
||||
content_parts.append(delta.content)
|
||||
return "".join(reasoning_parts), "".join(content_parts)
|
||||
|
||||
|
||||
def test_granite_thinking_streaming_enable_thinking_false(
|
||||
tokenizer: GraniteThinkingTokenizer,
|
||||
):
|
||||
# With enable_thinking=False, the parser (constructed without
|
||||
# kwargs) starts in REASONING state. All text streams as reasoning
|
||||
# AND is duplicated into content via the streaming fallback —
|
||||
# matching NemotronV3 behavior.
|
||||
request = ChatCompletionRequest(
|
||||
model="test-model",
|
||||
messages=[],
|
||||
chat_template_kwargs={"enable_thinking": False},
|
||||
)
|
||||
parser = _make_reasoning_parser(tokenizer)
|
||||
|
||||
reasoning, content = _run_parse_delta(
|
||||
parser, tokenizer, "This is plain content", request
|
||||
)
|
||||
|
||||
assert reasoning == "This is plain content"
|
||||
assert content == "This is plain content"
|
||||
|
||||
|
||||
def test_granite_thinking_streaming_strips_leading_newline(
|
||||
tokenizer: GraniteThinkingTokenizer,
|
||||
):
|
||||
request = ChatCompletionRequest(
|
||||
model="test-model",
|
||||
messages=[],
|
||||
)
|
||||
parser = _make_reasoning_parser(tokenizer)
|
||||
|
||||
reasoning, content = _run_parse_delta(
|
||||
parser, tokenizer, "<think>reason</think>\nHello", request
|
||||
)
|
||||
|
||||
assert reasoning == "reason"
|
||||
assert content == "Hello"
|
||||
|
||||
|
||||
def test_granite_thinking_streaming_promotes_reasoning_to_content(
|
||||
tokenizer: GraniteThinkingTokenizer,
|
||||
):
|
||||
request = ChatCompletionRequest(
|
||||
model="test-model",
|
||||
messages=[],
|
||||
chat_template_kwargs={"force_nonempty_content": True},
|
||||
)
|
||||
parser = _make_reasoning_parser(tokenizer)
|
||||
|
||||
reasoning, content = _run_parse_delta(parser, tokenizer, "<think>4", request)
|
||||
|
||||
assert reasoning == "4"
|
||||
assert content == "4"
|
||||
|
||||
|
||||
def test_granite_thinking_streaming_no_promotion_with_real_content(
|
||||
tokenizer: GraniteThinkingTokenizer,
|
||||
):
|
||||
request = ChatCompletionRequest(
|
||||
model="test-model",
|
||||
messages=[],
|
||||
chat_template_kwargs={"force_nonempty_content": True},
|
||||
)
|
||||
parser = _make_reasoning_parser(tokenizer)
|
||||
|
||||
reasoning, content = _run_parse_delta(
|
||||
parser,
|
||||
tokenizer,
|
||||
"<think>reason</think>real answer",
|
||||
request,
|
||||
)
|
||||
|
||||
assert reasoning == "reason"
|
||||
assert content == "real answer"
|
||||
|
||||
|
||||
def test_granite_thinking_streaming_no_promotion_without_opt_in(
|
||||
tokenizer: GraniteThinkingTokenizer,
|
||||
):
|
||||
request = ChatCompletionRequest(model="test-model", messages=[])
|
||||
parser = _make_reasoning_parser(tokenizer)
|
||||
|
||||
reasoning, content = _run_parse_delta(parser, tokenizer, "<think>4", request)
|
||||
|
||||
assert reasoning == "4"
|
||||
assert content == ""
|
||||
|
||||
|
||||
# ── Empty think block ────────────────────────────────────────────────
|
||||
|
||||
|
||||
@pytest.mark.parametrize("streaming", [False, True])
|
||||
def test_granite_thinking_empty_think_block(
|
||||
tokenizer: GraniteThinkingTokenizer,
|
||||
streaming: bool,
|
||||
):
|
||||
# <think></think>\nHello — empty reasoning, content after newline.
|
||||
parser_cls = ReasoningParserManager.get_reasoning_parser(parser_name)
|
||||
parser = parser_cls(tokenizer)
|
||||
|
||||
reasoning, content = run_reasoning_extraction(
|
||||
parser,
|
||||
["<think>", "</think>", "\n", "Hello"],
|
||||
streaming=streaming,
|
||||
)
|
||||
|
||||
assert reasoning is None or reasoning == ""
|
||||
assert content == "Hello"
|
||||
|
||||
|
||||
# ── Chunking-independence streaming test ─────────────────────────────
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"text,expected_reasoning,expected_content",
|
||||
[
|
||||
("<think>r</think>\nHello", "r", "Hello"),
|
||||
("<think></think>\nHello", "", "Hello"),
|
||||
],
|
||||
)
|
||||
def test_granite_thinking_streaming_chunking_independent(
|
||||
tokenizer: GraniteThinkingTokenizer,
|
||||
text: str,
|
||||
expected_reasoning: str,
|
||||
expected_content: str,
|
||||
):
|
||||
# Verify the same input produces identical results regardless
|
||||
# of how it's chunked into streaming deltas. Uses the
|
||||
# DelegatingParser path (_run_parse_delta) which matches the
|
||||
# real serving flow.
|
||||
tokens = tokenizer.tokenize(text)
|
||||
request = ChatCompletionRequest(model="test-model", messages=[])
|
||||
|
||||
chunk_patterns = [
|
||||
tokens,
|
||||
# Split at the </think> boundary
|
||||
[
|
||||
tokenizer.convert_tokens_to_string(tokens[: tokens.index("</think>") + 1]),
|
||||
tokenizer.convert_tokens_to_string(tokens[tokens.index("</think>") + 1 :]),
|
||||
],
|
||||
]
|
||||
|
||||
for chunks in chunk_patterns:
|
||||
parser = _make_reasoning_parser(tokenizer)
|
||||
reasoning_parts: list[str] = []
|
||||
content_parts: list[str] = []
|
||||
all_tokens = []
|
||||
for chunk in chunks:
|
||||
chunk_tokens = tokenizer.tokenize(chunk)
|
||||
all_tokens.extend(chunk_tokens)
|
||||
for i, token in enumerate(all_tokens):
|
||||
delta = parser.parse_delta(
|
||||
delta_text=token,
|
||||
delta_token_ids=[_token_id(token)],
|
||||
request=request,
|
||||
prompt_token_ids=[] if i == 0 else None,
|
||||
finished=(i == len(all_tokens) - 1),
|
||||
)
|
||||
if delta is None:
|
||||
continue
|
||||
if delta.reasoning:
|
||||
reasoning_parts.append(delta.reasoning)
|
||||
if delta.content:
|
||||
content_parts.append(delta.content)
|
||||
reasoning = "".join(reasoning_parts)
|
||||
content = "".join(content_parts)
|
||||
assert reasoning == expected_reasoning or (
|
||||
not expected_reasoning and not reasoning
|
||||
), f"reasoning mismatch with chunks={chunks}"
|
||||
assert content == expected_content, f"content mismatch with chunks={chunks}"
|
||||
@@ -14,6 +14,7 @@ from vllm.parser.engine.adapters import make_adapters
|
||||
from vllm.parser.gemma4 import Gemma4Parser
|
||||
from vllm.parser.glm47_moe import Glm47MoeParser
|
||||
from vllm.parser.granite import GraniteParser
|
||||
from vllm.parser.granite_thinking import GraniteThinkingParser
|
||||
from vllm.parser.inkling import InklingParser
|
||||
from vllm.parser.kimi_k2 import KimiK2Parser
|
||||
from vllm.parser.minimax_m2 import MinimaxM2Parser
|
||||
@@ -52,6 +53,11 @@ from vllm.parser.seed_oss import SeedOssParser
|
||||
GraniteParserToolAdapter,
|
||||
) = make_adapters(GraniteParser)
|
||||
|
||||
(
|
||||
GraniteThinkingParserReasoningAdapter,
|
||||
GraniteThinkingParserToolAdapter,
|
||||
) = make_adapters(GraniteThinkingParser)
|
||||
|
||||
(
|
||||
NemotronV3ParserReasoningAdapter,
|
||||
NemotronV3ParserToolAdapter,
|
||||
|
||||
@@ -0,0 +1,131 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Granite 4.2 thinking parser.
|
||||
|
||||
Granite 4.2 uses the same tool call and reasoning format as Nemotron V3
|
||||
(``<think>``/``</think>`` + ``<tool_call>`` XML). This config reuses
|
||||
:class:`NemotronV3Parser` with two Granite-specific behaviors:
|
||||
|
||||
1. Strip leading newlines from content — the Granite chat template writes
|
||||
``\\n</think>\\n``, so the first content character is always ``\\n``.
|
||||
2. Inherit the Nemotron V3 reasoning-to-content swap for
|
||||
``enable_thinking=False`` / ``force_nonempty_content=True``.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import dataclasses
|
||||
import functools
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from vllm.parser.nemotron_v3 import NemotronV3Parser, nemotron_v3_config
|
||||
from vllm.parser.qwen3 import Qwen3Parser
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.entrypoints.generate.base.protocol import DeltaMessage
|
||||
from vllm.entrypoints.openai.chat_completion.protocol import (
|
||||
ChatCompletionRequest,
|
||||
)
|
||||
from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
|
||||
from vllm.parser.engine.parser_engine import SemanticEvent
|
||||
from vllm.parser.engine.parser_engine_config import ParserEngineConfig
|
||||
from vllm.tokenizers import TokenizerLike
|
||||
from vllm.tool_parsers.abstract_tool_parser import Tool
|
||||
|
||||
|
||||
@functools.cache
|
||||
def granite_thinking_config(thinking: bool = True) -> ParserEngineConfig:
|
||||
return dataclasses.replace(
|
||||
nemotron_v3_config(thinking=thinking),
|
||||
name="granite_thinking_parser",
|
||||
)
|
||||
|
||||
|
||||
class GraniteThinkingParser(NemotronV3Parser):
|
||||
"""Granite 4.2 parser: same format as Nemotron V3, with
|
||||
leading-newline stripping on content after ``</think>``.
|
||||
|
||||
The ``reasoning_ended`` property is overridden to delay the
|
||||
reasoning-to-content transition reported to
|
||||
:class:`~vllm.parser.abstract_parser.DelegatingParser` until the
|
||||
template's leading newline has been consumed. Without this delay,
|
||||
``DelegatingParser.parse_delta`` passes newline tokens through a
|
||||
raw pass-through path that bypasses the engine's
|
||||
``_events_to_delta`` (and therefore our stripping logic).
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
tokenizer: TokenizerLike,
|
||||
tools: list[Tool] | None = None,
|
||||
**kwargs,
|
||||
) -> None:
|
||||
chat_kwargs = kwargs.get("chat_template_kwargs", {}) or {}
|
||||
thinking = chat_kwargs.get("enable_thinking", True)
|
||||
kwargs.pop("parser_engine_config", None)
|
||||
Qwen3Parser.__init__(
|
||||
self,
|
||||
tokenizer,
|
||||
tools,
|
||||
parser_engine_config=granite_thinking_config(thinking=thinking),
|
||||
**kwargs,
|
||||
)
|
||||
self._streamed_reasoning: list[str] = []
|
||||
self._content_started = False
|
||||
|
||||
@property
|
||||
def reasoning_ended(self) -> bool:
|
||||
# Internal engine logic uses _reasoning_ended directly; this
|
||||
# property is only read by DelegatingParser (via the adapter's
|
||||
# has_engine_confirmed_reasoning_end) to decide when to stop
|
||||
# routing deltas through extract_reasoning_streaming.
|
||||
if not self._reasoning_ended:
|
||||
return False
|
||||
return self._content_started
|
||||
|
||||
def _reset(self, initial_state=None) -> None:
|
||||
super()._reset(initial_state=initial_state)
|
||||
self._content_started = False
|
||||
|
||||
def _events_to_delta(
|
||||
self,
|
||||
events: list[SemanticEvent],
|
||||
finished: bool = False,
|
||||
) -> DeltaMessage | None:
|
||||
delta = super()._events_to_delta(events, finished=finished)
|
||||
if (
|
||||
delta is not None
|
||||
and delta.content is not None
|
||||
and not self._content_started
|
||||
):
|
||||
stripped = delta.content.lstrip("\n")
|
||||
if not stripped:
|
||||
delta.content = None
|
||||
if delta.reasoning is None and not delta.tool_calls:
|
||||
return None
|
||||
else:
|
||||
self._content_started = True
|
||||
delta.content = stripped
|
||||
return delta
|
||||
|
||||
def extract_reasoning(
|
||||
self,
|
||||
model_output: str,
|
||||
request: ChatCompletionRequest | ResponsesRequest,
|
||||
) -> tuple[str | None, str | None]:
|
||||
reasoning, content = Qwen3Parser.extract_reasoning(self, model_output, request)
|
||||
|
||||
# Track whether content was truly absent (None) vs present
|
||||
# but whitespace-only — the swap should only fire for absent.
|
||||
content_was_absent = content is None
|
||||
|
||||
if content is not None:
|
||||
content = content.lstrip("\n") or None
|
||||
|
||||
# Only swap when content was truly absent from model output
|
||||
# (truncated reasoning or max_tokens cut at </think>), not
|
||||
# when it was present but stripped to empty (newlines-only).
|
||||
if self._should_force_content(request) and content_was_absent:
|
||||
reasoning, content = content, reasoning
|
||||
|
||||
return reasoning, content
|
||||
@@ -76,6 +76,10 @@ _REASONING_PARSERS_TO_REGISTER = {
|
||||
"granite_reasoning_parser",
|
||||
"GraniteReasoningParser",
|
||||
),
|
||||
"granite_thinking_parser": (
|
||||
"granite_thinking_engine_reasoning_parser",
|
||||
"GraniteThinkingParserReasoningAdapter",
|
||||
),
|
||||
"holo2": (
|
||||
"deepseek_v3_reasoning_parser",
|
||||
"DeepSeekV3ReasoningWithThinkingParser",
|
||||
|
||||
@@ -0,0 +1,8 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from vllm.parser.engine.registered_adapters import (
|
||||
GraniteThinkingParserReasoningAdapter,
|
||||
)
|
||||
|
||||
__all__ = ["GraniteThinkingParserReasoningAdapter"]
|
||||
Reference in New Issue
Block a user