[Model][Frontend] Support DeepSeek-V4.1-Flash in Rust and Python frontends (#56208)
Signed-off-by: Bugen Zhao <i@bugenzhao.com> Co-authored-by: Isotr0py <Isotr0py@outlook.com>
This commit is contained in:
@@ -0,0 +1,146 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import json
|
||||
|
||||
import pytest
|
||||
|
||||
from tests.parser.engine.replay_harness import (
|
||||
DUMMY_TOOLS,
|
||||
MockTokenizer,
|
||||
_test_request,
|
||||
collect_output,
|
||||
replay_streaming,
|
||||
)
|
||||
from vllm.parser.deepseek_v41 import deepseek_v41_config
|
||||
from vllm.parser.parser_manager import ParserManager
|
||||
|
||||
CALLS = (
|
||||
'\n\n<|DSML| calls>\n<|DSML| invoke name="get_weather">\n'
|
||||
'<|DSML| parameter name="city" string="true">杭州</|DSML| parameter>\n'
|
||||
'<|DSML| parameter name="count" string="false">42</|DSML| parameter>\n'
|
||||
'</|DSML| invoke>\n<|DSML| invoke name="add">\n'
|
||||
'<|DSML| parameter name="x" string="false">1.5</|DSML| parameter>\n'
|
||||
'<|DSML| parameter name="y" string="false">2.25</|DSML| parameter>\n'
|
||||
"</|DSML| invoke>\n</|DSML| calls>"
|
||||
)
|
||||
|
||||
|
||||
def tokenizer_for(text, special_calls):
|
||||
vocab = {"<think>": 50, "</think>": 51}
|
||||
if special_calls:
|
||||
vocab |= {"<|DSML| calls>": 52, "</|DSML| calls>": 53}
|
||||
tokens: list[tuple[int, str]] = []
|
||||
while text:
|
||||
special = next((marker for marker in vocab if text.startswith(marker)), None)
|
||||
piece = special or text[0]
|
||||
tokens.append((vocab[piece] if special else 100 + len(tokens), piece))
|
||||
text = text[len(piece) :]
|
||||
return MockTokenizer(vocab, tokens), tokens
|
||||
|
||||
|
||||
def parser_for(tokenizer, controls):
|
||||
cls = ParserManager.get_parser(
|
||||
tool_parser_name="deepseek_v41",
|
||||
reasoning_parser_name="deepseek_v41",
|
||||
enable_auto_tools=True,
|
||||
)
|
||||
return cls(tokenizer, chat_template_kwargs=controls)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("thinking", [False, True])
|
||||
@pytest.mark.parametrize("special_calls", [False, True])
|
||||
@pytest.mark.parametrize("chunk_size", [1, 7, 10000])
|
||||
def test_registered_adapters_parse_parallel_calls_across_chunks(
|
||||
thinking,
|
||||
special_calls,
|
||||
chunk_size,
|
||||
):
|
||||
text = ("Plan.</think>" if thinking else "") + "Checking." + CALLS
|
||||
tokenizer, tokens = tokenizer_for(text, special_calls)
|
||||
parser = parser_for(tokenizer, {"thinking": thinking})
|
||||
output = collect_output(
|
||||
replay_streaming(
|
||||
parser,
|
||||
tokens,
|
||||
chunk_size=chunk_size,
|
||||
finished_on_last=True,
|
||||
tools=DUMMY_TOOLS,
|
||||
prompt_token_ids=[50 if thinking else 51],
|
||||
)
|
||||
)
|
||||
assert output.reasoning == ("Plan." if thinking else "")
|
||||
assert output.content.strip() == "Checking."
|
||||
assert [call["name"] for call in output.tool_calls] == ["get_weather", "add"]
|
||||
assert [json.loads(call["arguments"]) for call in output.tool_calls] == [
|
||||
{"city": "杭州", "count": 42},
|
||||
{"x": 1.5, "y": 2.25},
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("thinking", [False, True])
|
||||
def test_registered_adapters_parse_complete_output(thinking):
|
||||
text = ("Plan.</think>" if thinking else "") + CALLS
|
||||
tokenizer, tokens = tokenizer_for(text, False)
|
||||
parser = parser_for(tokenizer, {"thinking": thinking})
|
||||
reasoning, content, calls = parser.parse(
|
||||
text,
|
||||
_test_request(tools=DUMMY_TOOLS),
|
||||
enable_auto_tools=True,
|
||||
model_output_token_ids=[tid for tid, _ in tokens],
|
||||
)
|
||||
assert (reasoning or "") == ("Plan." if thinking else "")
|
||||
assert not (content or "").strip()
|
||||
assert [call.name for call in calls] == ["get_weather", "add"]
|
||||
assert json.loads(calls[0].arguments) == {"city": "杭州", "count": 42}
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("controls", "text", "expected_reasoning", "reasoning_tokens"),
|
||||
[
|
||||
({"thinking": False}, "12", "", 0),
|
||||
({"enable_thinking": False}, "12", "", 0),
|
||||
({"thinking": True, "reasoning_effort": "none"}, "12", "", 0),
|
||||
({}, "</think>12", "", 0),
|
||||
({}, "Plan.</think>12", "Plan.", 5),
|
||||
],
|
||||
)
|
||||
def test_reasoning_adapter_controls_and_usage(
|
||||
controls,
|
||||
text,
|
||||
expected_reasoning,
|
||||
reasoning_tokens,
|
||||
):
|
||||
tokenizer, tokens = tokenizer_for(text, False)
|
||||
parser = parser_for(tokenizer, controls)
|
||||
output = collect_output(
|
||||
replay_streaming(
|
||||
parser,
|
||||
tokens,
|
||||
chunk_size=1,
|
||||
finished_on_last=True,
|
||||
)
|
||||
)
|
||||
assert output.reasoning == expected_reasoning
|
||||
assert output.content == "12"
|
||||
assert (
|
||||
parser.reasoning_parser.count_reasoning_tokens([tid for tid, _ in tokens])
|
||||
== reasoning_tokens
|
||||
)
|
||||
assert parser.is_reasoning_end([50, 100, 51])
|
||||
assert not parser.is_reasoning_end([51, 100, 50])
|
||||
|
||||
|
||||
def test_python_argument_conversion_and_partial_values():
|
||||
converter = deepseek_v41_config().arg_converter
|
||||
raw = (
|
||||
'<|DSML| parameter name="object" string="false">'
|
||||
'{"a": [true, null]}</|DSML| parameter>'
|
||||
'<|DSML| parameter name="bad" string="false">[broken</|DSML| parameter>'
|
||||
'<|DSML| parameter name="text" string="true"> a<b'
|
||||
)
|
||||
assert json.loads(converter(raw, True)) == {
|
||||
"object": {"a": [True, None]},
|
||||
"bad": "[broken",
|
||||
"text": " a<b",
|
||||
}
|
||||
@@ -0,0 +1,38 @@
|
||||
<|begin▁of▁sentence|><|System|>Reasoning Effort: 50 (range 1-100, the higher the value, the more thorough the reasoning)
|
||||
|
||||
You are a helpful assistant.
|
||||
|
||||
## Tools
|
||||
|
||||
You have access to a set of tools to help answer the user's question. You can invoke tools by writing a "<|DSML| calls>" block like the following:
|
||||
|
||||
<|DSML| calls>
|
||||
<|DSML| invoke name="$TOOL_NAME">
|
||||
<|DSML| parameter name="$PARAMETER_NAME" string="true|false">$PARAMETER_VALUE</|DSML| parameter>
|
||||
...
|
||||
</|DSML| invoke>
|
||||
<|DSML| invoke name="$TOOL_NAME2">
|
||||
...
|
||||
</|DSML| invoke>
|
||||
</|DSML| calls>
|
||||
|
||||
String parameters should be specified as is and set `string="true"`. For all other types (numbers, booleans, arrays, objects), pass the value in JSON format and set `string="false"`.
|
||||
|
||||
If thinking_mode is enabled (triggered by <think>), you MUST output your complete reasoning inside <think>...</think> BEFORE any tool calls or final response.
|
||||
|
||||
Otherwise, output directly after </think> with tool calls or final response.
|
||||
|
||||
### Available Tool Schemas
|
||||
|
||||
{"name": "get_weather", "description": "Get the weather for a specific location", "parameters": {"type": "object", "properties": {"location": {"type": "string", "description": "The city name"}, "unit": {"type": "string", "enum": ["celsius", "fahrenheit"], "description": "Temperature unit"}}, "required": ["location"]}}
|
||||
{"name": "search", "description": "Search the web for information", "parameters": {"type": "object", "properties": {"query": {"type": "string", "description": "Search query"}, "num_results": {"type": "integer", "description": "Number of results to return"}}, "required": ["query"]}}
|
||||
|
||||
You MUST strictly follow the above defined tool name and parameter schemas to invoke tool calls.
|
||||
<|User|>What's the weather in Beijing?<|Assistant|><think>The user wants to know the weather in Beijing. I should use the get_weather tool.</think>
|
||||
|
||||
<|DSML| calls>
|
||||
<|DSML| invoke name="get_weather">
|
||||
<|DSML| parameter name="location" string="true">Beijing</|DSML| parameter>
|
||||
<|DSML| parameter name="unit" string="true">celsius</|DSML| parameter>
|
||||
</|DSML| invoke>
|
||||
</|DSML| calls><|end▁of▁sentence|><|User|><tool_result>{"temperature": 22, "condition": "sunny", "humidity": 45}</tool_result><|Assistant|><think>Got the weather data. Let me format a nice response.</think>The weather in Beijing is currently sunny with a temperature of 22°C and 45% humidity.<|end▁of▁sentence|>
|
||||
@@ -0,0 +1,3 @@
|
||||
<|begin▁of▁sentence|><|System|>Reasoning Effort: 50 (range 1-100, the higher the value, the more thorough the reasoning)
|
||||
|
||||
You are a helpful assistant.<|User|>Hello<|Assistant|></think>Hi there! How can I help you?<|end▁of▁sentence|><|User|>What is the capital of France?<|Assistant|><think>The user asks about the capital of France. It is Paris.</think>The capital of France is Paris.<|end▁of▁sentence|>
|
||||
@@ -0,0 +1,169 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import copy
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from tests.tokenizers_.test_deepseek_v4 import FakeHfTokenizer
|
||||
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
|
||||
from vllm.tokenizers.deepseek_v41 import get_deepseek_v41_tokenizer
|
||||
|
||||
FIXTURES = Path(__file__).parent / "fixtures"
|
||||
|
||||
|
||||
def render(messages, **kwargs):
|
||||
return get_deepseek_v41_tokenizer(FakeHfTokenizer()).apply_chat_template(
|
||||
messages, tokenize=False, **kwargs
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("case_id", [1, 2])
|
||||
def test_reference_encoder_fixtures(case_id):
|
||||
# Expected prompts come from the DeepSeek V4.1 reference encoder.
|
||||
data = json.loads(
|
||||
(FIXTURES / "deepseek_v4" / f"test_input_{case_id}.json").read_text()
|
||||
)
|
||||
messages = data["messages"] if isinstance(data, dict) else data
|
||||
tools = data.get("tools") if isinstance(data, dict) else None
|
||||
expected = (FIXTURES / "deepseek_v41" / f"test_output_{case_id}.txt").read_text()
|
||||
assert render(messages, tools=tools) == expected
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("effort", "budget"),
|
||||
[
|
||||
(None, 50),
|
||||
("low", 25),
|
||||
("high", 50),
|
||||
("xhigh", 75),
|
||||
("max", 100),
|
||||
(1, 1),
|
||||
(42, 42),
|
||||
(100, 100),
|
||||
],
|
||||
)
|
||||
def test_numeric_reasoning_effort(effort, budget):
|
||||
assert render(
|
||||
[{"role": "user", "content": "question"}], reasoning_effort=effort
|
||||
) == (
|
||||
"<|begin▁of▁sentence|><|System|>"
|
||||
f"Reasoning Effort: {budget} (range 1-100, the higher the value, "
|
||||
"the more thorough the reasoning)\n\n"
|
||||
"<|User|>question<|Assistant|><think>"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("effort", ["medium", "minimal", -1, 0, 101, True, 1.5, []])
|
||||
def test_invalid_effort_is_a_request_error(effort):
|
||||
with pytest.raises(ValueError, match="reasoning_effort"):
|
||||
render([{"role": "user", "content": "question"}], reasoning_effort=effort)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"controls",
|
||||
[
|
||||
{"thinking": False},
|
||||
{"enable_thinking": False},
|
||||
{"thinking": True, "reasoning_effort": "none"},
|
||||
{"enable_thinking": True, "reasoning_effort": "none"},
|
||||
],
|
||||
)
|
||||
def test_chat_mode_has_closed_thinking_prefix(controls):
|
||||
assert render([{"role": "user", "content": "question"}], **controls) == (
|
||||
"<|begin▁of▁sentence|><|User|>question<|Assistant|></think>"
|
||||
)
|
||||
|
||||
|
||||
def test_raw_text_parts_preserve_reference_separator():
|
||||
messages = [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "text", "text": "first"},
|
||||
{"type": "text", "text": "second"},
|
||||
],
|
||||
}
|
||||
]
|
||||
original = copy.deepcopy(messages)
|
||||
assert (
|
||||
render(
|
||||
messages,
|
||||
conversation=[{"role": "user", "content": "first\nsecond"}],
|
||||
thinking=False,
|
||||
)
|
||||
== "<|begin▁of▁sentence|><|User|>first\n\nsecond<|Assistant|></think>"
|
||||
)
|
||||
assert messages == original
|
||||
|
||||
|
||||
def test_mid_system_gets_its_own_marker_and_generation_header():
|
||||
assert render(
|
||||
[
|
||||
{"role": "user", "content": "question"},
|
||||
{"role": "system", "content": "answer briefly"},
|
||||
],
|
||||
thinking=False,
|
||||
) == (
|
||||
"<|begin▁of▁sentence|><|User|>question"
|
||||
"<|System|>answer briefly<|Assistant|></think>"
|
||||
)
|
||||
|
||||
|
||||
def test_top_level_effort_overrides_template_effort():
|
||||
request = ChatCompletionRequest(
|
||||
model="deepseek-ai/DeepSeek-V4.1-Flash",
|
||||
messages=[{"role": "user", "content": "question"}],
|
||||
reasoning_effort="low",
|
||||
chat_template_kwargs={"reasoning_effort": 100},
|
||||
)
|
||||
kwargs = request.build_chat_params(None, "auto").get_apply_chat_template_kwargs()
|
||||
assert "Reasoning Effort: 25 " in render(request.messages, **kwargs)
|
||||
|
||||
|
||||
def test_encode_uses_one_bos_and_forwards_truncation():
|
||||
tokenizer = get_deepseek_v41_tokenizer(FakeHfTokenizer())
|
||||
tokenizer.apply_chat_template(
|
||||
[{"role": "user", "content": "question"}],
|
||||
thinking=False,
|
||||
truncation=True,
|
||||
max_length=16,
|
||||
)
|
||||
text, add_special_tokens, kwargs = tokenizer.last_encode
|
||||
assert text.startswith("<|begin▁of▁sentence|>")
|
||||
assert add_special_tokens is False
|
||||
assert kwargs == {"truncation": True, "max_length": 16}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("image_type", ["image_url", "input_image", "image_pil"])
|
||||
def test_images_preserve_content_order_and_reference_separator(image_type):
|
||||
from PIL import Image
|
||||
|
||||
def image_part(color):
|
||||
if image_type == "image_pil":
|
||||
return {"type": image_type, "image_pil": Image.new("RGB", (2, 2), color)}
|
||||
url = f"https://example.com/{color}.png"
|
||||
return {
|
||||
"type": image_type,
|
||||
"image_url": {"url": url} if image_type == "image_url" else url,
|
||||
}
|
||||
|
||||
parts = [
|
||||
{"type": "text", "text": "first"},
|
||||
image_part("red"),
|
||||
{"type": "text", "text": "second"},
|
||||
image_part("blue"),
|
||||
{"type": "text", "text": "compare"},
|
||||
]
|
||||
assert render([{"role": "user", "content": parts}], thinking=False) == (
|
||||
"<|begin▁of▁sentence|><|User|>first\n\n<|deepseek_image|>\n\n"
|
||||
"second\n\n<|deepseek_image|>\n\ncompare<|Assistant|></think>"
|
||||
)
|
||||
assert parts[1]["type"] == parts[3]["type"] == image_type
|
||||
|
||||
|
||||
def test_unsupported_media_is_a_request_error():
|
||||
with pytest.raises(ValueError, match="text and image content only"):
|
||||
render([{"role": "user", "content": [{"type": "input_audio"}]}])
|
||||
@@ -80,6 +80,67 @@ def test_supported_structural_tag_models_include_vllm_builtins():
|
||||
assert "hermes" in VLLM_BUILTIN_STRUCTURAL_TAG_MODELS
|
||||
|
||||
|
||||
@pytest.mark.parametrize("choice", ["auto", "required", "get_weather"])
|
||||
def test_deepseek_v41_format_constraints_ignore_parameter_schema(
|
||||
choice,
|
||||
sample_tools_strict,
|
||||
):
|
||||
tool_choice = (
|
||||
ChatCompletionNamedToolChoiceParam(
|
||||
function=ChatCompletionNamedFunction(name=choice)
|
||||
)
|
||||
if choice == "get_weather"
|
||||
else choice
|
||||
)
|
||||
tag = get_model_structural_tag(
|
||||
"deepseek_v41", sample_tools_strict, tool_choice, reasoning=False
|
||||
)
|
||||
grammar = Grammar.from_structural_tag(tag)
|
||||
begin = '\n\n<|DSML| calls>\n<|DSML| invoke name="get_weather">\n'
|
||||
end = "</|DSML| invoke>\n</|DSML| calls>"
|
||||
parameter = (
|
||||
'<|DSML| parameter name="undeclared" string="false">'
|
||||
'{"nested": [true, null, 1.5]}</|DSML| parameter>\n'
|
||||
)
|
||||
# Required city is omitted; extra and repeated fields remain unconstrained.
|
||||
assert _is_grammar_accept_string(grammar, begin + end)
|
||||
assert _is_grammar_accept_string(grammar, begin + parameter * 2 + end)
|
||||
assert not _is_grammar_accept_string(
|
||||
grammar,
|
||||
begin + parameter.replace('{"nested": [true, null, 1.5]}', "invalid") + end,
|
||||
)
|
||||
assert not _is_grammar_accept_string(
|
||||
grammar, (begin + end).replace('name="get_weather"', 'name="unknown"')
|
||||
)
|
||||
assert not _is_grammar_accept_string(
|
||||
grammar, begin + parameter.replace("| parameter", "|parameter") + end
|
||||
)
|
||||
assert _is_grammar_accept_string(grammar, "Hello") == (choice == "auto")
|
||||
|
||||
|
||||
def test_deepseek_v41_named_choice_emits_one_call(sample_tools):
|
||||
tag = get_model_structural_tag(
|
||||
"deepseek_v41",
|
||||
sample_tools,
|
||||
ChatCompletionNamedToolChoiceParam(
|
||||
function=ChatCompletionNamedFunction(name="get_weather")
|
||||
),
|
||||
reasoning=False,
|
||||
)
|
||||
grammar = Grammar.from_structural_tag(tag)
|
||||
call = '<|DSML| invoke name="get_weather">\n</|DSML| invoke>\n'
|
||||
assert _is_grammar_accept_string(
|
||||
grammar, "\n\n<|DSML| calls>\n" + call + "</|DSML| calls>"
|
||||
)
|
||||
assert not _is_grammar_accept_string(
|
||||
grammar, "\n\n<|DSML| calls>\n" + call * 2 + "</|DSML| calls>"
|
||||
)
|
||||
assert (
|
||||
get_model_structural_tag("deepseek_v41", sample_tools, "auto", reasoning=False)
|
||||
is None
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model", sorted(XGRAMMAR_BUILTIN_STRUCTURAL_TAG_MODELS))
|
||||
def test_get_model_structural_tag_supports_all_xgrammar_builtins(
|
||||
model: str,
|
||||
|
||||
Reference in New Issue
Block a user