iHore/vLLM
8
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

[Frontend] Port Step-3.5 parsers to the streaming parser engine (#59321)

Browse Source 
Signed-off-by: sfeng33 <4florafeng@gmail.com>
This commit is contained in:
Flora Feng
2026-10-01 22:18:37 +00:00
committed by GitHub
parent ca65eb67d9
commit a5105dba05
12 changed files with 236 additions and 1647 deletions
Download Patch File Download Diff File Expand all files Collapse all files
docs/features/reasoning_outputs.md
+1
View File
@@ -29,6 +29,7 @@ vLLM currently supports the following reasoning models:
| [MiniMax-M2](https://huggingface.co/MiniMaxAI/MiniMax-M2) | `minimax_m2_append_think` | `json`, `regex` | ✅ |
| [Qwen3 series](https://huggingface.co/collections/Qwen/qwen3-67dd247413f0e2e4f653967f) | `qwen3` | `json`, `regex` | ✅ |
| [QwQ-32B](https://huggingface.co/Qwen/QwQ-32B) | `deepseek_r1` | `json`, `regex` | ✅ |
| [Step-3.5 / Step-3.7 Flash](https://huggingface.co/stepfun-ai/Step-3.5-Flash) | `step3p5` | `json`, `regex` | ✅ |
!!! note
IBM Granite 3.2 and DeepSeek-V3.1 reasoning is disabled by default; to enable it, you must also pass `thinking=True` in your `chat_template_kwargs`.
docs/features/tool_calling.md
+12
View File
@@ -469,6 +469,18 @@ Use the checkpoint's chat template with `--tool-call-parser mimo --reasoning-par
For schema-constrained tool arguments, set `strict: true` inside each tool's `function`, or use `--tool-strict-level parameter`.
The parser supports `auto`, `required`, named tools, and `parallel_tool_calls=False`, preserving leading/trailing whitespace in string arguments.
### Step-3.5 and Step-3.7 Models (`step3p5`)
Supported models:
* `stepfun-ai/Step-3.5-Flash`
* `stepfun-ai/Step-3.5-Flash-FP8`
* `stepfun-ai/Step-3.7-Flash`
Flags: `--tool-call-parser step3p5 --reasoning-parser step3p5 --enable-auto-tool-choice`
The parser supports `auto`, `required`, and named tools.
### Olmo 3 Models (`olmo3`)
Olmo 3 models output tool calls in a format that is very similar to the one expected by the `pythonic` parser (see below), with a few differences. Each tool call is a pythonic string, but the parallel tool calls are newline-delimited, and the calls are wrapped within XML tags as `<function_calls>..</function_calls>`. In addition, the parser also allows JSON boolean and null literals (`true`, `false`, and `null`) in addition to the pythonic ones (`True`, `False`, and `None`).
tests/parser/engine/test_step3p5.py
+122
View File
@@ -0,0 +1,122 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests for the engine-based Step-3.5 parser.
Step-3.5 reuses the Qwen3 grammar, which ``test_qwen3.py`` covers. These tests
cover what differs: thinking is always on, the newline the model emits before
``</think>`` is dropped, prior-turn reasoning does not end the current turn,
and the ``step3p5`` names resolve to the engine adapters.
"""
import json
import pytest
from tests.parser.engine.conftest import make_mock_tokenizer
from tests.parser.engine.streaming_helpers import simulate_reasoning_streaming
from vllm.parser.engine.registered_adapters import (
Step3p5ParserReasoningAdapter,
Step3p5ParserToolAdapter,
)
from vllm.parser.step3p5 import Step3p5Parser
from vllm.reasoning import ReasoningParserManager
from vllm.tool_parsers import ToolParserManager
_THINK_START_ID = 50
_THINK_END_ID = 51
_IM_START_ID = 70
_IM_END_ID = 71
_TEXT_ID = 100
_STEP3P5_VOCAB = {
"<think>": _THINK_START_ID,
"</think>": _THINK_END_ID,
"<tool_call>": 60,
"</tool_call>": 61,
"<|im_start|>": _IM_START_ID,
"<|im_end|>": _IM_END_ID,
}
@pytest.fixture
def mock_tokenizer():
return make_mock_tokenizer(_STEP3P5_VOCAB)
@pytest.fixture
def parser(mock_tokenizer):
return Step3p5Parser(mock_tokenizer)
def test_parser_names_resolve_to_engine_adapters():
tool_parser_cls = ToolParserManager.get_tool_parser("step3p5")
assert issubclass(tool_parser_cls, Step3p5ParserToolAdapter)
assert tool_parser_cls.structural_tag_model == "qwen_3_coder"
reasoning_parser_cls = ReasoningParserManager.get_reasoning_parser("step3p5")
assert reasoning_parser_cls is Step3p5ParserReasoningAdapter
def test_enable_thinking_false_is_ignored(mock_tokenizer):
# The chat template always prefills "<think>\n" and has no thinking switch.
parser = Step3p5Parser(
mock_tokenizer, chat_template_kwargs={"enable_thinking": False}
)
assert parser.extract_reasoning("plan</think>answer", None) == ("plan", "answer")
assert not parser.is_reasoning_end([_IM_START_ID, _TEXT_ID, _THINK_START_ID])
def test_newline_before_think_end_dropped(parser):
text = "Line one.\nLine two.\n</think>\nThe answer."
assert parser.extract_reasoning(text, None) == (
"Line one.\nLine two.",
"\nThe answer.",
)
def test_newline_before_think_end_dropped_streaming(parser):
reasoning, content = simulate_reasoning_streaming(
parser,
["Line one.\n", "Line two.\n", "</think>", "\nThe answer."],
[(_TEXT_ID,), (_TEXT_ID,), (_THINK_END_ID,), (_TEXT_ID,)],
)
assert reasoning == "Line one.\nLine two."
assert content == "\nThe answer."
class TestPriorTurnReasoning:
"""The template replays earlier assistant turns as
``<think>\\n...\\n</think>\\n``, so a ``</think>`` from history must not end
reasoning for the new turn (#34211)."""
_HISTORY = [
_IM_START_ID,
_TEXT_ID,
_THINK_START_ID,
_TEXT_ID,
_THINK_END_ID,
_TEXT_ID,
_IM_END_ID,
]
def test_prior_turn_think_end_not_end(self, parser):
assert not parser.is_reasoning_end([*self._HISTORY, _IM_START_ID, _TEXT_ID])
def test_think_end_in_current_turn_is_end(self, parser):
assert parser.is_reasoning_end(
[*self._HISTORY, _IM_START_ID, _THINK_START_ID, _TEXT_ID, _THINK_END_ID]
)
def test_reasoning_then_tool_call(parser, mock_request):
text = (
"Need the weather.\n</think>\n<tool_call>\n<function=get_weather>\n"
"<parameter=city>\nTokyo\n</parameter>\n</function>\n</tool_call>"
)
reasoning, content, tool_calls = parser.parse(
text, mock_request, enable_auto_tools=True
)
assert reasoning == "Need the weather."
assert content is None
assert tool_calls is not None
assert tool_calls[0].name == "get_weather"
assert json.loads(tool_calls[0].arguments) == {"city": "Tokyo"}
tests/parser/engine/trace_builder.py
+15
View File
@@ -46,6 +46,7 @@ from vllm.parser.engine.registered_adapters import (
Plamo3Parser,
Qwen3Parser,
SeedOssParser,
Step3p5Parser,
)
# ── Data structures ──────────────────────────────────────────────────
@@ -449,6 +450,19 @@ def _build_mimo(scenario: Scenario, validate: bool = True) -> Sample:
)
# ── Step-3.5 (Qwen3 XML, trailing reasoning whitespace stripped) ────
def _build_step3p5(scenario: Scenario, validate: bool = True) -> Sample:
return _build_qwen3(
scenario,
name="step3p5",
parser_cls=Step3p5Parser,
strip_trailing_ws=True,
validate=validate,
)
# ── MiniMax M2 (XML invoke format, starts in REASONING) ──────────────
_MINIMAX_M2_VOCAB: dict[str, int] = {
@@ -1268,6 +1282,7 @@ _BUILDERS: dict[str, Any] = {
"kimi_k2": _build_kimi_k2,
"qwen3": _build_qwen3,
"mimo": _build_mimo,
"step3p5": _build_step3p5,
"inkling": _build_inkling,
"plamo3": _build_plamo3,
}
tests/reasoning/test_step3p5_reasoning_parser.py
+9 -9
View File
@@ -4,8 +4,8 @@
import pytest
from tests.reasoning.utils import run_reasoning_extraction
from vllm.parser.engine.registered_adapters import Step3p5ParserReasoningAdapter
from vllm.reasoning import ReasoningParserManager
from vllm.reasoning.step3p5_reasoning_parser import Step3p5ReasoningParser
from vllm.tokenizers import get_tokenizer
parser_name = "step3p5"
@@ -112,7 +112,7 @@ EMPTY_STREAMING = {
}
NEW_LINE = {
"output": "\n<think>This is a reasoning section</think>\nThis is the rest",
"reasoning": "This is a reasoning section",
"reasoning": "\nThis is a reasoning section",
"content": "\nThis is the rest",
"is_reasoning_end": True,
}
@@ -126,7 +126,7 @@ NEW_LINE_STREAMING = {
NEW_LINE_STREAMING_COMPLEX_CONTENT = {
"output": "\n This is a \n reasoning section\n\n\n</think>\n\nThis is the rest",
"reasoning": "\n This is a \n reasoning section\n\n",
"reasoning": "\n This is a \n reasoning section",
"content": "\n\nThis is the rest",
"is_reasoning_end": True,
}
@@ -285,7 +285,7 @@ def test_reasoning(
step3p5_tokenizer.convert_tokens_to_string([token]) for token in output
]
parser = ReasoningParserManager.get_reasoning_parser(parser_name)(step3p5_tokenizer)
assert isinstance(parser, Step3p5ReasoningParser)
assert isinstance(parser, Step3p5ParserReasoningAdapter)
reasoning, content = run_reasoning_extraction(
parser, output_tokens, streaming=streaming
@@ -307,9 +307,10 @@ def test_reasoning(
current_ids: list[int] = []
for token_id in output_ids:
current_ids.append(token_id)
if token_id == parser.end_token_id:
if token_id in parser.reasoning_end_token_ids:
assert parser.is_reasoning_end_streaming(current_ids, [token_id])
assert parser.is_reasoning_end_streaming(output_ids, output_ids)
if param_dict["is_reasoning_end"]:
assert parser.is_reasoning_end_streaming(output_ids, output_ids)
break
assert not parser.is_reasoning_end_streaming(current_ids, [token_id])
else:
@@ -333,6 +334,5 @@ def test_reasoning(
step3p5_tokenizer.tokenize(param_dict["content"])
)
assert content_ids == expected_content_ids
else:
content_ids = parser.extract_content_ids(output_ids)
assert content_ids == []
elif param_dict["is_reasoning_end"]:
assert parser.extract_content_ids(output_ids) == []
tests/tool_parsers/test_step3p5_tool_parser.py
+12 -57
View File
@@ -847,7 +847,7 @@ TX
assert chunk.tool_calls[0].function.name == "get_current_weather"
assert chunk.tool_calls[0].type == "function"
# Empty initially
assert chunk.tool_calls[0].function.arguments == ""
assert not chunk.tool_calls[0].function.arguments
break
assert header_found
@@ -867,43 +867,6 @@ TX
assert parsed_args["state"] == "TX"
def test_extract_tool_calls_complex_type_with_single_quote(step3p5_tokenizer):
"""Test parameter type conversion based on tool schema."""
tools = [
ChatCompletionToolsParam(
type="function",
function={
"name": "test_types",
"parameters": {
"type": "object",
"properties": {
"int_param": {"type": "integer"},
"float_param": {"type": "float"},
"bool_param": {"type": "boolean"},
"str_param": {"type": "string"},
"obj_param": {"type": "object"},
},
},
},
)
]
model_output = """<tool_call>
<function=test_types>
<parameter=obj_param>
{'key': 'value'}
</parameter>
</function>
</tool_call>"""
parser = Step3p5ToolParser(step3p5_tokenizer, tools=tools)
request = ChatCompletionRequest(model=MODEL, messages=[], tools=tools)
extracted_tool_calls = parser.extract_tool_calls(model_output, request=request)
args = json.loads(extracted_tool_calls.tool_calls[0].function.arguments)
assert args["obj_param"] == {"key": "value"}
def test_extract_tool_calls_streaming_mixed_content_and_multiple_tool_calls(
step3p5_tool_parser, step3p5_tokenizer, sample_tools
):
@@ -1016,8 +979,8 @@ def test_extract_tool_calls_non_streaming_mixed_content_and_multiple_tool_calls(
"""Test non-streaming extraction with mixed content and multiple tool calls.
Scenario: Model outputs "hello" + complete tool call + "hi" + complete tool call.
Expected: "hello" as content, first tool call parsed (index=0), "hi" as content,
second tool call parsed (index=1)
Expected: "hello" as content, first tool call parsed (index=0), second tool call
parsed (index=1). Non-streaming content stops at the first tool call.
"""
# Model output: hello + complete tool call + hi + complete tool call
model_output = """hello<tool_call>
@@ -1066,17 +1029,7 @@ rectangle
assert args_dict_1["dimensions"]["width"] == 10
assert args_dict_1["dimensions"]["height"] == 5
# Verify content: should contain "hello", "hi"
assert extracted_tool_calls.content is not None
assert "hello" in extracted_tool_calls.content, "Should contain 'hello' as content"
assert "hi" in extracted_tool_calls.content, "Should contain 'hi' as content"
# Verify the order: hello should come first, then hi
hello_index = extracted_tool_calls.content.find("hello")
hi_index = extracted_tool_calls.content.find("hi")
assert hello_index >= 0, "'hello' should be in content"
assert hi_index > hello_index, "'hi' should come after 'hello'"
assert extracted_tool_calls.content == "hello"
# Verify that tool call tags are NOT in the content
assert "<function=get_current_weather>" not in extracted_tool_calls.content, (
@@ -1155,12 +1108,14 @@ rectangle
request=request,
)
# Process delta result
if delta_result:
if delta_result.content:
other_content += delta_result.content
if delta_result.tool_calls:
for tool_call in delta_result.tool_calls:
# Serving flushes the parser at the end of the stream.
for delta in (delta_result, step3p5_tool_parser.finish_streaming()):
if not delta:
continue
if delta.content:
other_content += delta.content
if delta.tool_calls:
for tool_call in delta.tool_calls:
idx = tool_call.index
if idx not in tool_states:
tool_states[idx] = {
vllm/parser/engine/registered_adapters.py
+6
View File
@@ -24,6 +24,7 @@ from vllm.parser.nemotron_v3 import NemotronV3Parser
from vllm.parser.plamo3 import Plamo3Parser
from vllm.parser.qwen3 import Qwen3Parser
from vllm.parser.seed_oss import SeedOssParser
from vllm.parser.step3p5 import Step3p5Parser
(
DeepSeekV32ParserReasoningAdapter,
@@ -80,6 +81,11 @@ from vllm.parser.seed_oss import SeedOssParser
SeedOssParserToolAdapter,
) = make_adapters(SeedOssParser)
(
Step3p5ParserReasoningAdapter,
Step3p5ParserToolAdapter,
) = make_adapters(Step3p5Parser)
(
Glm47MoeParserReasoningAdapter,
Glm47MoeParserToolAdapter,
vllm/parser/step3p5.py
+49
View File
@@ -0,0 +1,49 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Step-3.5 / Step-3.7 parser for tool calls and reasoning.
Both models use the Qwen3 XML tool-call grammar and ``<think>`` reasoning
unchanged. Two things differ from :class:`Qwen3Parser`:
* The chat template always prefills ``<think>\\n`` and has no switch to turn
thinking off, so ``enable_thinking`` is ignored.
* The model closes reasoning with ``\\n</think>\\n``, so trailing reasoning
whitespace is stripped.
"""
from __future__ import annotations
import functools
from dataclasses import replace
from typing import TYPE_CHECKING
from vllm.parser.engine.parser_engine_config import ParserEngineConfig
from vllm.parser.qwen3 import CHATML_TURN_BOUNDARIES, Qwen3Parser, qwen3_config
if TYPE_CHECKING:
from vllm.tokenizers import TokenizerLike
from vllm.tool_parsers.abstract_tool_parser import Tool
@functools.cache
def step3p5_config() -> ParserEngineConfig:
return replace(
qwen3_config(
thinking=True,
name="step3p5",
turn_boundary_tokens=CHATML_TURN_BOUNDARIES,
),
strip_trailing_reasoning_whitespace=True,
)
class Step3p5Parser(Qwen3Parser):
def __init__(
self,
tokenizer: TokenizerLike,
tools: list[Tool] | None = None,
**kwargs,
) -> None:
kwargs.setdefault("parser_engine_config", step3p5_config())
super().__init__(tokenizer, tools, **kwargs)
self.thinking_enabled = True
vllm/reasoning/__init__.py
+2 -2
View File
@@ -157,8 +157,8 @@ _REASONING_PARSERS_TO_REGISTER = {
"Step3ReasoningParser",
),
"step3p5": (
"step3p5_reasoning_parser",
"Step3p5ReasoningParser",
"step3p5_engine_reasoning_parser",
"Step3p5ParserReasoningAdapter",
),
"inkling": (
"inkling_reasoning_parser",
vllm/reasoning/step3p5_engine_reasoning_parser.py
+6
View File
@@ -0,0 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from vllm.parser.engine.registered_adapters import Step3p5ParserReasoningAdapter
__all__ = ["Step3p5ParserReasoningAdapter"]
vllm/reasoning/step3p5_reasoning_parser.py
-121
View File
@@ -1,121 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from collections.abc import Sequence
from typing import TYPE_CHECKING
from vllm.entrypoints.generate.base.protocol import DeltaMessage
from vllm.reasoning.basic_parsers import BaseThinkingReasoningParser
from vllm.tokenizers import TokenizerLike
if TYPE_CHECKING:
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
class Step3p5ReasoningParser(BaseThinkingReasoningParser):
"""Reasoning parser for Step3p5 model.
Step3p5 uses the <think>...</think> format, but it tends to emit an extra
newline immediately before and/or after the </think> token. This parser
trims the newline before </think> in extract_reasoning*.
Reasoning end detection doesn't wait for an extra token after </think> so
structured output can constrain the first post-think token.
"""
@property
def start_token(self) -> str:
return "<think>"
@property
def end_token(self) -> str:
return "</think>"
def __init__(self, tokenizer: TokenizerLike, *args, **kwargs):
super().__init__(tokenizer, *args, **kwargs)
# Used to hold a trailing "\n" from reasoning content so we can decide
# whether it is immediately before </think>.
self._pending_reasoning_newline = False
def extract_reasoning(
self,
model_output: str,
request: "ChatCompletionRequest | ResponsesRequest",
) -> tuple[str | None, str | None]:
reasoning, content = super().extract_reasoning(model_output, request)
if reasoning is not None:
reasoning = reasoning.removesuffix("\n")
return reasoning or None, content or None
def extract_reasoning_streaming(
self,
previous_text: str,
current_text: str,
delta_text: str,
previous_token_ids: Sequence[int],
current_token_ids: Sequence[int],
delta_token_ids: Sequence[int],
) -> DeltaMessage | None:
ret = super().extract_reasoning_streaming(
previous_text,
current_text,
delta_text,
previous_token_ids,
current_token_ids,
delta_token_ids,
)
if ret is None:
return None
# Compatibility path for models that don't generate the start token:
# treat everything before </think> as reasoning and everything after
# as content.
if (
self.start_token_id not in previous_token_ids
and self.start_token_id not in delta_token_ids
):
if self.end_token_id in delta_token_ids:
end_index = delta_text.find(self.end_token)
reasoning = delta_text[:end_index]
content = delta_text[end_index + len(self.end_token) :]
ret = DeltaMessage(reasoning=reasoning, content=content or None)
elif self.end_token_id in previous_token_ids:
ret = DeltaMessage(content=delta_text)
else:
ret = DeltaMessage(reasoning=delta_text)
reasoning_to_output = ret.reasoning
content_to_output = ret.content
# Reasoning: handle the newline immediately before </think>.
if reasoning_to_output is not None:
if self._pending_reasoning_newline:
reasoning_to_output = "\n" + reasoning_to_output
self._pending_reasoning_newline = False
if reasoning_to_output.endswith("\n"):
reasoning_to_output = reasoning_to_output.removesuffix("\n")
if self.end_token in delta_text:
# Trailing "\n" is right before </think>, drop it.
self._pending_reasoning_newline = False
else:
# Hold the trailing "\n" until we know whether </think> follows.
self._pending_reasoning_newline = True
# Content: handle the newline immediately after </think>.
if content_to_output is not None:
# If we have content, reasoning must have ended.
self._pending_reasoning_newline = False
if self.end_token in delta_text and content_to_output.startswith("\n"):
content_to_output = content_to_output.removeprefix("\n")
reasoning_to_output = reasoning_to_output or None
content_to_output = content_to_output or None
if reasoning_to_output is None and content_to_output is None:
return None
return DeltaMessage(reasoning=reasoning_to_output, content=content_to_output)
vllm/tool_parsers/step3p5_tool_parser.py
+2 -1458
View File
File diff suppressed because it is too large Load Diff