2406 lines
74 KiB
Python
2406 lines
74 KiB
Python
# SPDX-License-Identifier: Apache-2.0
|
||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||
|
||
import copy
|
||
from typing import Any
|
||
|
||
import llguidance
|
||
import pytest
|
||
from mistral_common.exceptions import InvalidMessageStructureException
|
||
from mistral_common.guidance.grammar_factory import GrammarFactory
|
||
from mistral_common.tokens.tokenizers.base import SpecialTokenPolicy, SpecialTokens
|
||
|
||
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
|
||
from vllm.exceptions import VLLMValidationError
|
||
from vllm.renderers.chat_utils import ChatCompletionMessageParam
|
||
from vllm.tokenizers.mistral import (
|
||
MistralTokenizer,
|
||
_validate_apply_chat_template_args,
|
||
validate_request_params,
|
||
)
|
||
|
||
|
||
def test_validate_apply_chat_template_args():
|
||
# add_generation_prompt with assistant last message → error
|
||
messages: list[ChatCompletionMessageParam] = [
|
||
{"role": "assistant", "content": "Hello"}
|
||
]
|
||
with pytest.raises(ValueError):
|
||
_validate_apply_chat_template_args(messages, add_generation_prompt=True)
|
||
|
||
# add_generation_prompt with user last message → ok
|
||
messages = [{"role": "user", "content": "Hello"}]
|
||
_validate_apply_chat_template_args(messages, add_generation_prompt=True)
|
||
|
||
# both add_generation_prompt and continue_final_message → error
|
||
with pytest.raises(ValueError):
|
||
_validate_apply_chat_template_args(
|
||
messages,
|
||
add_generation_prompt=True,
|
||
continue_final_message=True,
|
||
)
|
||
|
||
# continue_final_message with assistant last message → ok
|
||
messages = [{"role": "assistant", "content": "Hello"}]
|
||
_validate_apply_chat_template_args(messages, continue_final_message=True)
|
||
|
||
# continue_final_message with user last message → error
|
||
messages = [{"role": "user", "content": "Hello"}]
|
||
with pytest.raises(ValueError):
|
||
_validate_apply_chat_template_args(messages, continue_final_message=True)
|
||
|
||
|
||
@pytest.fixture(scope="module")
|
||
def mistral_tokenizer(request) -> MistralTokenizer:
|
||
return MistralTokenizer.from_pretrained(request.param)
|
||
|
||
|
||
@pytest.mark.parametrize(
|
||
"mistral_tokenizer",
|
||
["mistralai/Mistral-7B-Instruct-v0.3", "mistralai/Magistral-Small-2509"],
|
||
indirect=True,
|
||
)
|
||
class TestMistralTokenizer:
|
||
def test_all_special_tokens(self, mistral_tokenizer: MistralTokenizer):
|
||
if mistral_tokenizer.is_tekken:
|
||
assert mistral_tokenizer.all_special_tokens == [
|
||
"<unk>",
|
||
"<s>",
|
||
"</s>",
|
||
"[INST]",
|
||
"[/INST]",
|
||
"[AVAILABLE_TOOLS]",
|
||
"[/AVAILABLE_TOOLS]",
|
||
"[TOOL_RESULTS]",
|
||
"[/TOOL_RESULTS]",
|
||
"[TOOL_CALLS]",
|
||
"[IMG]",
|
||
"<pad>",
|
||
"[IMG_BREAK]",
|
||
"[IMG_END]",
|
||
"[PREFIX]",
|
||
"[MIDDLE]",
|
||
"[SUFFIX]",
|
||
"[SYSTEM_PROMPT]",
|
||
"[/SYSTEM_PROMPT]",
|
||
"[TOOL_CONTENT]",
|
||
] + [f"<SPECIAL_{i}>" for i in range(20, 32)] + [
|
||
"[ARGS]",
|
||
"[CALL_ID]",
|
||
"[THINK]",
|
||
"[/THINK]",
|
||
] + [f"<SPECIAL_{i}>" for i in range(36, 1000)]
|
||
else:
|
||
assert mistral_tokenizer.all_special_tokens == [
|
||
"<s>",
|
||
"</s>",
|
||
"[INST]",
|
||
"[/INST]",
|
||
"[TOOL_CALLS]",
|
||
"[AVAILABLE_TOOLS]",
|
||
"[/AVAILABLE_TOOLS]",
|
||
"[TOOL_RESULTS]",
|
||
"[/TOOL_RESULTS]",
|
||
] + [f"[control_{i}]" for i in range(8, 769)]
|
||
|
||
def get_vocab(self, mistral_tokenizer: MistralTokenizer):
|
||
assert (
|
||
mistral_tokenizer.get_vocab()
|
||
== mistral_tokenizer.transformers_tokenizer.get_vocab()
|
||
)
|
||
|
||
def test_get_added_vocab(self, mistral_tokenizer: MistralTokenizer):
|
||
assert mistral_tokenizer.get_added_vocab() == {}
|
||
|
||
def test_encode(self, mistral_tokenizer: MistralTokenizer):
|
||
token_ids = (
|
||
[1, 22177, 4304, 2662]
|
||
if mistral_tokenizer.is_tekken
|
||
else [1, 23325, 2294, 1686]
|
||
)
|
||
|
||
assert mistral_tokenizer.encode("Hello world !") == token_ids
|
||
assert mistral_tokenizer.encode("Hello world !", max_length=3) == token_ids[:-1]
|
||
assert (
|
||
mistral_tokenizer.encode("Hello world !", truncation=True, max_length=3)
|
||
== token_ids[:-1]
|
||
)
|
||
assert (
|
||
mistral_tokenizer.encode("Hello world !", truncation=False, max_length=3)
|
||
== token_ids
|
||
)
|
||
|
||
assert (
|
||
mistral_tokenizer.encode("Hello world !", add_special_tokens=True)
|
||
== token_ids
|
||
)
|
||
assert (
|
||
mistral_tokenizer.encode(
|
||
"Hello world !", add_special_tokens=True, max_length=3
|
||
)
|
||
== token_ids[:-1]
|
||
)
|
||
assert (
|
||
mistral_tokenizer.encode(
|
||
"Hello world !", add_special_tokens=True, truncation=False, max_length=3
|
||
)
|
||
== token_ids
|
||
)
|
||
assert (
|
||
mistral_tokenizer.encode("Hello world !", add_special_tokens=False)
|
||
== token_ids[1:]
|
||
)
|
||
assert mistral_tokenizer.encode("", add_special_tokens=False) == []
|
||
|
||
def test_call(self, mistral_tokenizer: MistralTokenizer):
|
||
token_ids = (
|
||
[1, 22177, 4304, 2662]
|
||
if mistral_tokenizer.is_tekken
|
||
else [1, 23325, 2294, 1686]
|
||
)
|
||
attn_mask = [1 for _ in range(len(token_ids))]
|
||
|
||
# Test 1: no special tokens
|
||
assert mistral_tokenizer("Hello world !", add_special_tokens=False) == {
|
||
"attention_mask": attn_mask[1:],
|
||
"input_ids": token_ids[1:],
|
||
}
|
||
# Test 2: special tokens
|
||
assert mistral_tokenizer("Hello world !", add_special_tokens=True) == {
|
||
"attention_mask": attn_mask,
|
||
"input_ids": token_ids,
|
||
}
|
||
# Test 3: special tokens + truncation
|
||
assert mistral_tokenizer(
|
||
"Hello world !", add_special_tokens=True, truncation=True, max_length=3
|
||
) == {
|
||
"attention_mask": attn_mask[:-1],
|
||
"input_ids": token_ids[:-1],
|
||
}
|
||
# Test 4: special tokens + no truncation + max length
|
||
assert mistral_tokenizer(
|
||
"Hello world !", add_special_tokens=True, max_length=3
|
||
) == {
|
||
"attention_mask": attn_mask,
|
||
"input_ids": token_ids,
|
||
}
|
||
# Test 5: empty string
|
||
assert mistral_tokenizer("", add_special_tokens=False) == {
|
||
"attention_mask": [],
|
||
"input_ids": [],
|
||
}
|
||
|
||
with pytest.raises(
|
||
ValueError,
|
||
match=(r"`text_pair` is not supported by `MistralTokenizer.__call__`."),
|
||
):
|
||
mistral_tokenizer("Hello world !", "invalid pair")
|
||
|
||
@pytest.mark.parametrize(
|
||
"openai_request,add_generation_prompt,continue_final_message,expected_output,decoded_expected_output",
|
||
[
|
||
(
|
||
{
|
||
"messages": [
|
||
{
|
||
"role": "user",
|
||
"content": "Hello world !",
|
||
}
|
||
],
|
||
},
|
||
True,
|
||
False,
|
||
([1, 3, 23325, 2294, 1686, 4], [1, 3, 22177, 4304, 2662, 4]),
|
||
("<s>[INST]▁Hello▁world▁![/INST]", ("<s>[INST]Hello world ![/INST]")),
|
||
),
|
||
(
|
||
{
|
||
"messages": [
|
||
{
|
||
"role": "system",
|
||
"content": "I am an AI",
|
||
},
|
||
{
|
||
"role": "user",
|
||
"content": "Hello world !",
|
||
},
|
||
],
|
||
},
|
||
True,
|
||
False,
|
||
(
|
||
[1, 3, 1083, 1605, 1164, 16875, 781, 781, 16998, 2294, 1686, 4],
|
||
[1, 17, 1073, 1855, 1420, 26554, 18, 3, 22177, 4304, 2662, 4],
|
||
),
|
||
(
|
||
"<s>[INST]▁I▁am▁an▁AI<0x0A><0x0A>Hello▁world▁![/INST]",
|
||
(
|
||
"<s>[SYSTEM_PROMPT]I am an AI[/SYSTEM_PROMPT][INST]Hello world ![/INST]" # noqa: E501
|
||
),
|
||
),
|
||
),
|
||
(
|
||
{
|
||
"messages": [
|
||
{
|
||
"role": "system",
|
||
"content": "I am an AI",
|
||
},
|
||
{
|
||
"role": "user",
|
||
"content": "Hello world !",
|
||
},
|
||
],
|
||
"tools": [
|
||
{
|
||
"type": "function",
|
||
"function": {
|
||
"name": "get_weather",
|
||
"description": "Gets the current weather in a city.",
|
||
"parameters": {
|
||
"type": "object",
|
||
"properties": {
|
||
"city": {
|
||
"type": "string",
|
||
"description": "The city name",
|
||
}
|
||
},
|
||
"required": ["city"],
|
||
},
|
||
},
|
||
}
|
||
],
|
||
},
|
||
True,
|
||
False,
|
||
(
|
||
[
|
||
1,
|
||
6,
|
||
1501,
|
||
7567,
|
||
1891,
|
||
2032,
|
||
1113,
|
||
3396,
|
||
1316,
|
||
1113,
|
||
3396,
|
||
2032,
|
||
10598,
|
||
1629,
|
||
2032,
|
||
1113,
|
||
1295,
|
||
29498,
|
||
1537,
|
||
1991,
|
||
1316,
|
||
1113,
|
||
7286,
|
||
2032,
|
||
1113,
|
||
2226,
|
||
29481,
|
||
1040,
|
||
2636,
|
||
8854,
|
||
1065,
|
||
1032,
|
||
3758,
|
||
9959,
|
||
1113,
|
||
12206,
|
||
2032,
|
||
10598,
|
||
1891,
|
||
2032,
|
||
1113,
|
||
3582,
|
||
1316,
|
||
1113,
|
||
11491,
|
||
2032,
|
||
10598,
|
||
19141,
|
||
2032,
|
||
10598,
|
||
1891,
|
||
2032,
|
||
1113,
|
||
2195,
|
||
1316,
|
||
1113,
|
||
7286,
|
||
2032,
|
||
1113,
|
||
1782,
|
||
3758,
|
||
1909,
|
||
29507,
|
||
11549,
|
||
1113,
|
||
11661,
|
||
2032,
|
||
8135,
|
||
19141,
|
||
3010,
|
||
1743,
|
||
10925,
|
||
7,
|
||
3,
|
||
1083,
|
||
1605,
|
||
1164,
|
||
16875,
|
||
781,
|
||
781,
|
||
16998,
|
||
2294,
|
||
1686,
|
||
4,
|
||
],
|
||
[
|
||
1,
|
||
17,
|
||
1073,
|
||
1855,
|
||
1420,
|
||
26554,
|
||
18,
|
||
5,
|
||
1091,
|
||
19227,
|
||
4994,
|
||
2811,
|
||
1429,
|
||
5165,
|
||
1897,
|
||
1429,
|
||
5165,
|
||
2811,
|
||
16753,
|
||
2391,
|
||
2811,
|
||
1429,
|
||
1689,
|
||
1095,
|
||
45629,
|
||
1897,
|
||
1429,
|
||
14653,
|
||
2811,
|
||
1429,
|
||
1071,
|
||
3083,
|
||
1278,
|
||
3519,
|
||
17253,
|
||
1294,
|
||
1261,
|
||
5970,
|
||
39249,
|
||
1429,
|
||
26204,
|
||
2811,
|
||
16753,
|
||
4994,
|
||
2811,
|
||
1429,
|
||
6371,
|
||
1897,
|
||
1429,
|
||
48649,
|
||
2811,
|
||
16753,
|
||
29363,
|
||
2811,
|
||
16753,
|
||
4994,
|
||
2811,
|
||
1429,
|
||
3607,
|
||
1897,
|
||
1429,
|
||
14653,
|
||
2811,
|
||
1429,
|
||
1784,
|
||
5970,
|
||
2564,
|
||
1034,
|
||
47579,
|
||
1429,
|
||
15760,
|
||
2811,
|
||
12161,
|
||
29363,
|
||
4964,
|
||
2821,
|
||
27028,
|
||
6,
|
||
3,
|
||
22177,
|
||
4304,
|
||
2662,
|
||
4,
|
||
],
|
||
),
|
||
(
|
||
'<s>[AVAILABLE_TOOLS]▁[{"type":▁"function",▁"function":▁{"name":▁"get_weather",▁"description":▁"Gets▁the▁current▁weather▁in▁a▁city.",▁"parameters":▁{"type":▁"object",▁"properties":▁{"city":▁{"type":▁"string",▁"description":▁"The▁city▁name"}},▁"required":▁["city"]}}}][/AVAILABLE_TOOLS][INST]▁I▁am▁an▁AI<0x0A><0x0A>Hello▁world▁![/INST]',
|
||
(
|
||
'<s>[SYSTEM_PROMPT]I am an AI[/SYSTEM_PROMPT][AVAILABLE_TOOLS][{"type": "function", "function": {"name": "get_weather", "description": "Gets the current weather in a city.", "parameters": {"type": "object", "properties": {"city": {"type": "string", "description": "The city name"}}, "required": ["city"]}}}][/AVAILABLE_TOOLS][INST]Hello world ![/INST]' # noqa: E501
|
||
),
|
||
),
|
||
),
|
||
(
|
||
{
|
||
"messages": [
|
||
{
|
||
"role": "system",
|
||
"content": "I am an AI",
|
||
},
|
||
{
|
||
"role": "user",
|
||
"content": "Hello world !",
|
||
},
|
||
{
|
||
"role": "assistant",
|
||
"content": "",
|
||
"tool_calls": [
|
||
{
|
||
"id": "123456789",
|
||
"type": "function",
|
||
"function": {
|
||
"name": "get_weather",
|
||
"arguments": '{"city": "Paris"}',
|
||
},
|
||
}
|
||
],
|
||
},
|
||
{
|
||
"role": "tool",
|
||
"tool_call_id": "123456789",
|
||
"content": '{"temperature": 20, "unit": "celsius"}',
|
||
},
|
||
],
|
||
"tools": [
|
||
{
|
||
"type": "function",
|
||
"function": {
|
||
"name": "get_weather",
|
||
"description": "Gets the current weather in a city.",
|
||
"parameters": {
|
||
"type": "object",
|
||
"properties": {
|
||
"city": {
|
||
"type": "string",
|
||
"description": "The city name",
|
||
}
|
||
},
|
||
"required": ["city"],
|
||
},
|
||
},
|
||
}
|
||
],
|
||
},
|
||
True,
|
||
False,
|
||
(
|
||
[
|
||
1,
|
||
6,
|
||
1501,
|
||
7567,
|
||
1891,
|
||
2032,
|
||
1113,
|
||
3396,
|
||
1316,
|
||
1113,
|
||
3396,
|
||
2032,
|
||
10598,
|
||
1629,
|
||
2032,
|
||
1113,
|
||
1295,
|
||
29498,
|
||
1537,
|
||
1991,
|
||
1316,
|
||
1113,
|
||
7286,
|
||
2032,
|
||
1113,
|
||
2226,
|
||
29481,
|
||
1040,
|
||
2636,
|
||
8854,
|
||
1065,
|
||
1032,
|
||
3758,
|
||
9959,
|
||
1113,
|
||
12206,
|
||
2032,
|
||
10598,
|
||
1891,
|
||
2032,
|
||
1113,
|
||
3582,
|
||
1316,
|
||
1113,
|
||
11491,
|
||
2032,
|
||
10598,
|
||
19141,
|
||
2032,
|
||
10598,
|
||
1891,
|
||
2032,
|
||
1113,
|
||
2195,
|
||
1316,
|
||
1113,
|
||
7286,
|
||
2032,
|
||
1113,
|
||
1782,
|
||
3758,
|
||
1909,
|
||
29507,
|
||
11549,
|
||
1113,
|
||
11661,
|
||
2032,
|
||
8135,
|
||
19141,
|
||
3010,
|
||
1743,
|
||
10925,
|
||
7,
|
||
3,
|
||
1083,
|
||
1605,
|
||
1164,
|
||
16875,
|
||
781,
|
||
781,
|
||
16998,
|
||
2294,
|
||
1686,
|
||
4,
|
||
5,
|
||
1501,
|
||
7567,
|
||
1629,
|
||
2032,
|
||
1113,
|
||
1295,
|
||
29498,
|
||
1537,
|
||
1991,
|
||
1316,
|
||
1113,
|
||
17452,
|
||
2032,
|
||
10598,
|
||
19141,
|
||
2032,
|
||
1113,
|
||
4684,
|
||
1046,
|
||
8474,
|
||
1113,
|
||
1081,
|
||
2032,
|
||
1113,
|
||
29508,
|
||
29518,
|
||
29538,
|
||
29549,
|
||
29550,
|
||
29552,
|
||
29555,
|
||
29551,
|
||
29542,
|
||
29507,
|
||
10925,
|
||
2,
|
||
8,
|
||
10598,
|
||
4557,
|
||
2032,
|
||
10598,
|
||
29475,
|
||
17329,
|
||
2032,
|
||
29473,
|
||
29518,
|
||
29502,
|
||
29493,
|
||
1113,
|
||
6074,
|
||
2032,
|
||
1113,
|
||
29485,
|
||
1958,
|
||
3938,
|
||
8474,
|
||
1113,
|
||
3613,
|
||
29498,
|
||
1081,
|
||
2032,
|
||
1113,
|
||
29508,
|
||
29518,
|
||
29538,
|
||
29549,
|
||
29550,
|
||
29552,
|
||
29555,
|
||
29551,
|
||
29542,
|
||
18163,
|
||
9,
|
||
],
|
||
[
|
||
1,
|
||
17,
|
||
1073,
|
||
1855,
|
||
1420,
|
||
26554,
|
||
18,
|
||
5,
|
||
1091,
|
||
19227,
|
||
4994,
|
||
2811,
|
||
1429,
|
||
5165,
|
||
1897,
|
||
1429,
|
||
5165,
|
||
2811,
|
||
16753,
|
||
2391,
|
||
2811,
|
||
1429,
|
||
1689,
|
||
1095,
|
||
45629,
|
||
1897,
|
||
1429,
|
||
14653,
|
||
2811,
|
||
1429,
|
||
1071,
|
||
3083,
|
||
1278,
|
||
3519,
|
||
17253,
|
||
1294,
|
||
1261,
|
||
5970,
|
||
39249,
|
||
1429,
|
||
26204,
|
||
2811,
|
||
16753,
|
||
4994,
|
||
2811,
|
||
1429,
|
||
6371,
|
||
1897,
|
||
1429,
|
||
48649,
|
||
2811,
|
||
16753,
|
||
29363,
|
||
2811,
|
||
16753,
|
||
4994,
|
||
2811,
|
||
1429,
|
||
3607,
|
||
1897,
|
||
1429,
|
||
14653,
|
||
2811,
|
||
1429,
|
||
1784,
|
||
5970,
|
||
2564,
|
||
1034,
|
||
47579,
|
||
1429,
|
||
15760,
|
||
2811,
|
||
12161,
|
||
29363,
|
||
4964,
|
||
2821,
|
||
27028,
|
||
6,
|
||
3,
|
||
22177,
|
||
4304,
|
||
2662,
|
||
4,
|
||
9,
|
||
1689,
|
||
1095,
|
||
45629,
|
||
32,
|
||
19227,
|
||
29363,
|
||
2811,
|
||
1429,
|
||
42572,
|
||
46005,
|
||
2,
|
||
7,
|
||
19227,
|
||
113824,
|
||
2811,
|
||
1032,
|
||
1050,
|
||
1048,
|
||
1044,
|
||
1429,
|
||
8979,
|
||
2811,
|
||
1429,
|
||
1099,
|
||
79092,
|
||
46005,
|
||
8,
|
||
],
|
||
),
|
||
(
|
||
'<s>[AVAILABLE_TOOLS]▁[{"type":▁"function",▁"function":▁{"name":▁"get_weather",▁"description":▁"Gets▁the▁current▁weather▁in▁a▁city.",▁"parameters":▁{"type":▁"object",▁"properties":▁{"city":▁{"type":▁"string",▁"description":▁"The▁city▁name"}},▁"required":▁["city"]}}}][/AVAILABLE_TOOLS][INST]▁I▁am▁an▁AI<0x0A><0x0A>Hello▁world▁![/INST][TOOL_CALLS]▁[{"name":▁"get_weather",▁"arguments":▁{"city":▁"Paris"},▁"id":▁"123456789"}]</s>[TOOL_RESULTS]▁{"content":▁{"temperature":▁20,▁"unit":▁"celsius"},▁"call_id":▁"123456789"}[/TOOL_RESULTS]',
|
||
(
|
||
'<s>[SYSTEM_PROMPT]I am an AI[/SYSTEM_PROMPT][AVAILABLE_TOOLS][{"type": "function", "function": {"name": "get_weather", "description": "Gets the current weather in a city.", "parameters": {"type": "object", "properties": {"city": {"type": "string", "description": "The city name"}}, "required": ["city"]}}}][/AVAILABLE_TOOLS][INST]Hello world ![/INST][TOOL_CALLS]get_weather[ARGS]{"city": "Paris"}</s>[TOOL_RESULTS]{"temperature": 20, "unit": "celsius"}[/TOOL_RESULTS]' # noqa: E501
|
||
),
|
||
),
|
||
),
|
||
(
|
||
{
|
||
"messages": [
|
||
{
|
||
"role": "user",
|
||
"content": "Hello world !",
|
||
},
|
||
{
|
||
"role": "assistant",
|
||
"content": "Hello ",
|
||
},
|
||
],
|
||
},
|
||
False,
|
||
True,
|
||
(
|
||
[1, 3, 23325, 2294, 1686, 4, 23325],
|
||
[1, 3, 22177, 4304, 2662, 4, 22177],
|
||
),
|
||
(
|
||
"<s>[INST]▁Hello▁world▁![/INST]▁Hello",
|
||
"<s>[INST]Hello world ![/INST]Hello",
|
||
),
|
||
),
|
||
],
|
||
)
|
||
def test_apply_chat_template(
|
||
self,
|
||
mistral_tokenizer: MistralTokenizer,
|
||
openai_request: dict[str, Any],
|
||
add_generation_prompt: bool,
|
||
continue_final_message: bool,
|
||
expected_output: tuple[list[int], list[int]],
|
||
decoded_expected_output: tuple[str, str],
|
||
):
|
||
actual_output = mistral_tokenizer.apply_chat_template(
|
||
openai_request["messages"],
|
||
tools=openai_request.get("tools", []),
|
||
add_generation_prompt=add_generation_prompt,
|
||
continue_final_message=continue_final_message,
|
||
)
|
||
decoded_actual_output = mistral_tokenizer.tokenizer.decode(
|
||
actual_output, SpecialTokenPolicy.KEEP
|
||
)
|
||
|
||
assert actual_output == expected_output[mistral_tokenizer.is_tekken]
|
||
assert (
|
||
decoded_actual_output
|
||
== decoded_expected_output[mistral_tokenizer.is_tekken]
|
||
)
|
||
|
||
def test_apply_chat_template_error(self, mistral_tokenizer: MistralTokenizer):
|
||
messages: list[ChatCompletionMessageParam] = [
|
||
{"role": "user", "content": "Hello world !"}
|
||
]
|
||
|
||
with pytest.raises(ValueError):
|
||
mistral_tokenizer.apply_chat_template(
|
||
messages,
|
||
tools=[],
|
||
add_generation_prompt=True,
|
||
continue_final_message=True,
|
||
)
|
||
|
||
with pytest.raises(ValueError):
|
||
mistral_tokenizer.apply_chat_template(
|
||
messages,
|
||
tools=[],
|
||
add_generation_prompt=False,
|
||
continue_final_message=True,
|
||
)
|
||
|
||
messages = [
|
||
{"role": "user", "content": "Hello world !"},
|
||
{"role": "assistant", "content": "Hello "},
|
||
]
|
||
with pytest.raises(ValueError):
|
||
mistral_tokenizer.apply_chat_template(
|
||
messages,
|
||
tools=[],
|
||
add_generation_prompt=True,
|
||
continue_final_message=False,
|
||
)
|
||
|
||
messages = [
|
||
{"role": "user", "content": "Hello world !"},
|
||
{"role": "assistant", "content": "Hello "},
|
||
]
|
||
with pytest.raises(InvalidMessageStructureException):
|
||
mistral_tokenizer.apply_chat_template(
|
||
messages,
|
||
tools=[],
|
||
add_generation_prompt=False,
|
||
continue_final_message=False,
|
||
)
|
||
|
||
@pytest.mark.parametrize(
|
||
"skip_special_tokens,expected_tokens",
|
||
(
|
||
(
|
||
False,
|
||
(
|
||
"<s>[INST]▁Hello▁world▁![/INST]▁Hello</s>",
|
||
"<s>[INST]Hello world ![/INST]Hello</s>",
|
||
),
|
||
),
|
||
(True, ("Hello world ! Hello", "Hello world !Hello")),
|
||
),
|
||
)
|
||
def test_decode(
|
||
self,
|
||
mistral_tokenizer: MistralTokenizer,
|
||
skip_special_tokens: bool,
|
||
expected_tokens: tuple[str, str],
|
||
):
|
||
ids = (
|
||
[1, 3, 23325, 2294, 1686, 4, 23325, 2],
|
||
[1, 3, 22177, 4304, 2662, 4, 22177, 2],
|
||
)
|
||
assert (
|
||
mistral_tokenizer.decode(
|
||
ids[mistral_tokenizer.is_tekken],
|
||
skip_special_tokens=skip_special_tokens,
|
||
)
|
||
== expected_tokens[mistral_tokenizer.is_tekken]
|
||
)
|
||
|
||
def test_decode_empty(
|
||
self,
|
||
mistral_tokenizer: MistralTokenizer,
|
||
):
|
||
assert (
|
||
mistral_tokenizer.decode(
|
||
[],
|
||
)
|
||
== ""
|
||
)
|
||
|
||
def test_decode_int(
|
||
self,
|
||
mistral_tokenizer: MistralTokenizer,
|
||
):
|
||
ids = 1
|
||
assert (
|
||
mistral_tokenizer.decode(
|
||
ids,
|
||
skip_special_tokens=False,
|
||
)
|
||
== "<s>"
|
||
)
|
||
|
||
@pytest.mark.parametrize(
|
||
"skip_special_tokens,expected_tokens",
|
||
(
|
||
(
|
||
False,
|
||
(
|
||
["<s>[INST]▁Hello▁world▁![/INST]▁Hello</s>"],
|
||
["<s>[INST]Hello world ![/INST]Hello</s>"],
|
||
),
|
||
),
|
||
(True, (["Hello world ! Hello"], ["Hello world !Hello"])),
|
||
),
|
||
)
|
||
def test_batch_decode(
|
||
self,
|
||
mistral_tokenizer: MistralTokenizer,
|
||
skip_special_tokens: bool,
|
||
expected_tokens: tuple[str, str],
|
||
):
|
||
ids = (
|
||
[[1, 3, 23325, 2294, 1686, 4, 23325, 2]],
|
||
[[1, 3, 22177, 4304, 2662, 4, 22177, 2]],
|
||
)
|
||
assert (
|
||
mistral_tokenizer.batch_decode(
|
||
ids[mistral_tokenizer.is_tekken],
|
||
skip_special_tokens=skip_special_tokens,
|
||
)
|
||
== expected_tokens[mistral_tokenizer.is_tekken]
|
||
)
|
||
|
||
def test_batch_decode_empty(
|
||
self,
|
||
mistral_tokenizer: MistralTokenizer,
|
||
):
|
||
assert mistral_tokenizer.batch_decode(
|
||
[[]],
|
||
) == [""]
|
||
|
||
def test_convert_tokens_to_string(self, mistral_tokenizer: MistralTokenizer):
|
||
tokens = (
|
||
[
|
||
"<s>",
|
||
"[AVAILABLE_TOOLS]",
|
||
"▁[",
|
||
'{"',
|
||
"type",
|
||
'":',
|
||
'▁"',
|
||
"function",
|
||
'",',
|
||
'▁"',
|
||
"function",
|
||
'":',
|
||
'▁{"',
|
||
"name",
|
||
'":',
|
||
'▁"',
|
||
"get",
|
||
"_",
|
||
"we",
|
||
"ather",
|
||
'",',
|
||
'▁"',
|
||
"description",
|
||
'":',
|
||
'▁"',
|
||
"Get",
|
||
"s",
|
||
"▁the",
|
||
"▁current",
|
||
"▁weather",
|
||
"▁in",
|
||
"▁a",
|
||
"▁city",
|
||
'.",',
|
||
'▁"',
|
||
"parameters",
|
||
'":',
|
||
'▁{"',
|
||
"type",
|
||
'":',
|
||
'▁"',
|
||
"object",
|
||
'",',
|
||
'▁"',
|
||
"properties",
|
||
'":',
|
||
'▁{"',
|
||
"city",
|
||
'":',
|
||
'▁{"',
|
||
"type",
|
||
'":',
|
||
'▁"',
|
||
"string",
|
||
'",',
|
||
'▁"',
|
||
"description",
|
||
'":',
|
||
'▁"',
|
||
"The",
|
||
"▁city",
|
||
"▁name",
|
||
'"',
|
||
"}},",
|
||
'▁"',
|
||
"required",
|
||
'":',
|
||
'▁["',
|
||
"city",
|
||
'"]',
|
||
"}}",
|
||
"}]",
|
||
"[/AVAILABLE_TOOLS]",
|
||
"[INST]",
|
||
"▁I",
|
||
"▁am",
|
||
"▁an",
|
||
"▁AI",
|
||
"<0x0A>",
|
||
"<0x0A>",
|
||
"Hello",
|
||
"▁world",
|
||
"▁!",
|
||
"[/INST]",
|
||
"[TOOL_CALLS]",
|
||
"▁[",
|
||
'{"',
|
||
"name",
|
||
'":',
|
||
'▁"',
|
||
"get",
|
||
"_",
|
||
"we",
|
||
"ather",
|
||
'",',
|
||
'▁"',
|
||
"arguments",
|
||
'":',
|
||
'▁{"',
|
||
"city",
|
||
'":',
|
||
'▁"',
|
||
"Par",
|
||
"is",
|
||
'"},',
|
||
'▁"',
|
||
"id",
|
||
'":',
|
||
'▁"',
|
||
"1",
|
||
"2",
|
||
"3",
|
||
"4",
|
||
"5",
|
||
"6",
|
||
"7",
|
||
"8",
|
||
"9",
|
||
'"',
|
||
"}]",
|
||
"</s>",
|
||
"[TOOL_RESULTS]",
|
||
'▁{"',
|
||
"content",
|
||
'":',
|
||
'▁{"',
|
||
"t",
|
||
"emperature",
|
||
'":',
|
||
"▁",
|
||
"2",
|
||
"0",
|
||
",",
|
||
'▁"',
|
||
"unit",
|
||
'":',
|
||
'▁"',
|
||
"c",
|
||
"els",
|
||
"ius",
|
||
'"},',
|
||
'▁"',
|
||
"call",
|
||
"_",
|
||
"id",
|
||
'":',
|
||
'▁"',
|
||
"1",
|
||
"2",
|
||
"3",
|
||
"4",
|
||
"5",
|
||
"6",
|
||
"7",
|
||
"8",
|
||
"9",
|
||
'"}',
|
||
"[/TOOL_RESULTS]",
|
||
],
|
||
[
|
||
"<s>",
|
||
"[SYSTEM_PROMPT]",
|
||
"I",
|
||
" am",
|
||
" an",
|
||
" AI",
|
||
"[/SYSTEM_PROMPT]",
|
||
"[AVAILABLE_TOOLS]",
|
||
"[",
|
||
'{"',
|
||
"type",
|
||
'":',
|
||
' "',
|
||
"function",
|
||
'",',
|
||
' "',
|
||
"function",
|
||
'":',
|
||
' {"',
|
||
"name",
|
||
'":',
|
||
' "',
|
||
"get",
|
||
"_",
|
||
"weather",
|
||
'",',
|
||
' "',
|
||
"description",
|
||
'":',
|
||
' "',
|
||
"G",
|
||
"ets",
|
||
" the",
|
||
" current",
|
||
" weather",
|
||
" in",
|
||
" a",
|
||
" city",
|
||
'.",',
|
||
' "',
|
||
"parameters",
|
||
'":',
|
||
' {"',
|
||
"type",
|
||
'":',
|
||
' "',
|
||
"object",
|
||
'",',
|
||
' "',
|
||
"properties",
|
||
'":',
|
||
' {"',
|
||
"city",
|
||
'":',
|
||
' {"',
|
||
"type",
|
||
'":',
|
||
' "',
|
||
"string",
|
||
'",',
|
||
' "',
|
||
"description",
|
||
'":',
|
||
' "',
|
||
"The",
|
||
" city",
|
||
" name",
|
||
'"',
|
||
"}},",
|
||
' "',
|
||
"required",
|
||
'":',
|
||
' ["',
|
||
"city",
|
||
'"]',
|
||
"}}",
|
||
"}]",
|
||
"[/AVAILABLE_TOOLS]",
|
||
"[INST]",
|
||
"Hello",
|
||
" world",
|
||
" !",
|
||
"[/INST]",
|
||
"[TOOL_CALLS]",
|
||
"get",
|
||
"_",
|
||
"weather",
|
||
"[ARGS]",
|
||
'{"',
|
||
"city",
|
||
'":',
|
||
' "',
|
||
"Paris",
|
||
'"}',
|
||
"</s>",
|
||
"[TOOL_RESULTS]",
|
||
'{"',
|
||
"temperature",
|
||
'":',
|
||
" ",
|
||
"2",
|
||
"0",
|
||
",",
|
||
' "',
|
||
"unit",
|
||
'":',
|
||
' "',
|
||
"c",
|
||
"elsius",
|
||
'"}',
|
||
"[/TOOL_RESULTS]",
|
||
],
|
||
)
|
||
|
||
expected_strings = (
|
||
'[{"type": "function", "function": {"name": "get_weather", "description": "Gets the current weather in a city.", "parameters": {"type": "object", "properties": {"city": {"type": "string", "description": "The city name"}}, "required": ["city"]}}}] I am an AI\n\nHello world ![TOOL_CALLS][{"name": "get_weather", "arguments": {"city": "Paris"}, "id": "123456789"}] {"content": {"temperature": 20, "unit": "celsius"}, "call_id": "123456789"}', # noqa: E501
|
||
# v11+ tool-call decode emits the explicit [ARGS] separator between
|
||
# the function name and its JSON arguments (get_weather[ARGS]{...}).
|
||
'I am an AI[{"type": "function", "function": {"name": "get_weather", "description": "Gets the current weather in a city.", "parameters": {"type": "object", "properties": {"city": {"type": "string", "description": "The city name"}}, "required": ["city"]}}}]Hello world ![TOOL_CALLS]get_weather[ARGS]{"city": "Paris"}{"temperature": 20, "unit": "celsius"}', # noqa: E501
|
||
)
|
||
|
||
assert (
|
||
mistral_tokenizer.convert_tokens_to_string(
|
||
tokens[mistral_tokenizer.is_tekken]
|
||
)
|
||
== expected_strings[mistral_tokenizer.is_tekken]
|
||
)
|
||
|
||
assert mistral_tokenizer.convert_tokens_to_string([]) == ""
|
||
|
||
@pytest.mark.parametrize(
|
||
"skip_special_tokens,tuple_expected_tokens",
|
||
(
|
||
(
|
||
True,
|
||
(
|
||
[
|
||
"▁[",
|
||
'{"',
|
||
"type",
|
||
'":',
|
||
'▁"',
|
||
"function",
|
||
'",',
|
||
'▁"',
|
||
"function",
|
||
'":',
|
||
'▁{"',
|
||
"name",
|
||
'":',
|
||
'▁"',
|
||
"get",
|
||
"_",
|
||
"we",
|
||
"ather",
|
||
'",',
|
||
'▁"',
|
||
"description",
|
||
'":',
|
||
'▁"',
|
||
"Get",
|
||
"s",
|
||
"▁the",
|
||
"▁current",
|
||
"▁weather",
|
||
"▁in",
|
||
"▁a",
|
||
"▁city",
|
||
'.",',
|
||
'▁"',
|
||
"parameters",
|
||
'":',
|
||
'▁{"',
|
||
"type",
|
||
'":',
|
||
'▁"',
|
||
"object",
|
||
'",',
|
||
'▁"',
|
||
"properties",
|
||
'":',
|
||
'▁{"',
|
||
"city",
|
||
'":',
|
||
'▁{"',
|
||
"type",
|
||
'":',
|
||
'▁"',
|
||
"string",
|
||
'",',
|
||
'▁"',
|
||
"description",
|
||
'":',
|
||
'▁"',
|
||
"The",
|
||
"▁city",
|
||
"▁name",
|
||
'"',
|
||
"}},",
|
||
'▁"',
|
||
"required",
|
||
'":',
|
||
'▁["',
|
||
"city",
|
||
'"]',
|
||
"}}",
|
||
"}]",
|
||
"▁I",
|
||
"▁am",
|
||
"▁an",
|
||
"▁AI",
|
||
"<0x0A>",
|
||
"<0x0A>",
|
||
"Hello",
|
||
"▁world",
|
||
"▁!",
|
||
"[TOOL_CALLS]",
|
||
"▁[",
|
||
'{"',
|
||
"name",
|
||
'":',
|
||
'▁"',
|
||
"get",
|
||
"_",
|
||
"we",
|
||
"ather",
|
||
'",',
|
||
'▁"',
|
||
"arguments",
|
||
'":',
|
||
'▁{"',
|
||
"city",
|
||
'":',
|
||
'▁"',
|
||
"Par",
|
||
"is",
|
||
'"},',
|
||
'▁"',
|
||
"id",
|
||
'":',
|
||
'▁"',
|
||
"1",
|
||
"2",
|
||
"3",
|
||
"4",
|
||
"5",
|
||
"6",
|
||
"7",
|
||
"8",
|
||
"9",
|
||
'"',
|
||
"}]",
|
||
'▁{"',
|
||
"content",
|
||
'":',
|
||
'▁{"',
|
||
"t",
|
||
"emperature",
|
||
'":',
|
||
"▁",
|
||
"2",
|
||
"0",
|
||
",",
|
||
'▁"',
|
||
"unit",
|
||
'":',
|
||
'▁"',
|
||
"c",
|
||
"els",
|
||
"ius",
|
||
'"},',
|
||
'▁"',
|
||
"call",
|
||
"_",
|
||
"id",
|
||
'":',
|
||
'▁"',
|
||
"1",
|
||
"2",
|
||
"3",
|
||
"4",
|
||
"5",
|
||
"6",
|
||
"7",
|
||
"8",
|
||
"9",
|
||
'"}',
|
||
],
|
||
[
|
||
"I",
|
||
" am",
|
||
" an",
|
||
" AI",
|
||
"[",
|
||
'{"',
|
||
"type",
|
||
'":',
|
||
' "',
|
||
"function",
|
||
'",',
|
||
' "',
|
||
"function",
|
||
'":',
|
||
' {"',
|
||
"name",
|
||
'":',
|
||
' "',
|
||
"get",
|
||
"_",
|
||
"weather",
|
||
'",',
|
||
' "',
|
||
"description",
|
||
'":',
|
||
' "',
|
||
"G",
|
||
"ets",
|
||
" the",
|
||
" current",
|
||
" weather",
|
||
" in",
|
||
" a",
|
||
" city",
|
||
'.",',
|
||
' "',
|
||
"parameters",
|
||
'":',
|
||
' {"',
|
||
"type",
|
||
'":',
|
||
' "',
|
||
"object",
|
||
'",',
|
||
' "',
|
||
"properties",
|
||
'":',
|
||
' {"',
|
||
"city",
|
||
'":',
|
||
' {"',
|
||
"type",
|
||
'":',
|
||
' "',
|
||
"string",
|
||
'",',
|
||
' "',
|
||
"description",
|
||
'":',
|
||
' "',
|
||
"The",
|
||
" city",
|
||
" name",
|
||
'"',
|
||
"}},",
|
||
' "',
|
||
"required",
|
||
'":',
|
||
' ["',
|
||
"city",
|
||
'"]',
|
||
"}}",
|
||
"}]",
|
||
"Hello",
|
||
" world",
|
||
" !",
|
||
"[TOOL_CALLS]",
|
||
"get",
|
||
"_",
|
||
"weather",
|
||
"[ARGS]",
|
||
'{"',
|
||
"city",
|
||
'":',
|
||
' "',
|
||
"Paris",
|
||
'"}',
|
||
'{"',
|
||
"temperature",
|
||
'":',
|
||
" ",
|
||
"2",
|
||
"0",
|
||
",",
|
||
' "',
|
||
"unit",
|
||
'":',
|
||
' "',
|
||
"c",
|
||
"elsius",
|
||
'"}',
|
||
],
|
||
),
|
||
),
|
||
(
|
||
False,
|
||
(
|
||
[
|
||
"<s>",
|
||
"[AVAILABLE_TOOLS]",
|
||
"▁[",
|
||
'{"',
|
||
"type",
|
||
'":',
|
||
'▁"',
|
||
"function",
|
||
'",',
|
||
'▁"',
|
||
"function",
|
||
'":',
|
||
'▁{"',
|
||
"name",
|
||
'":',
|
||
'▁"',
|
||
"get",
|
||
"_",
|
||
"we",
|
||
"ather",
|
||
'",',
|
||
'▁"',
|
||
"description",
|
||
'":',
|
||
'▁"',
|
||
"Get",
|
||
"s",
|
||
"▁the",
|
||
"▁current",
|
||
"▁weather",
|
||
"▁in",
|
||
"▁a",
|
||
"▁city",
|
||
'.",',
|
||
'▁"',
|
||
"parameters",
|
||
'":',
|
||
'▁{"',
|
||
"type",
|
||
'":',
|
||
'▁"',
|
||
"object",
|
||
'",',
|
||
'▁"',
|
||
"properties",
|
||
'":',
|
||
'▁{"',
|
||
"city",
|
||
'":',
|
||
'▁{"',
|
||
"type",
|
||
'":',
|
||
'▁"',
|
||
"string",
|
||
'",',
|
||
'▁"',
|
||
"description",
|
||
'":',
|
||
'▁"',
|
||
"The",
|
||
"▁city",
|
||
"▁name",
|
||
'"',
|
||
"}},",
|
||
'▁"',
|
||
"required",
|
||
'":',
|
||
'▁["',
|
||
"city",
|
||
'"]',
|
||
"}}",
|
||
"}]",
|
||
"[/AVAILABLE_TOOLS]",
|
||
"[INST]",
|
||
"▁I",
|
||
"▁am",
|
||
"▁an",
|
||
"▁AI",
|
||
"<0x0A>",
|
||
"<0x0A>",
|
||
"Hello",
|
||
"▁world",
|
||
"▁!",
|
||
"[/INST]",
|
||
"[TOOL_CALLS]",
|
||
"▁[",
|
||
'{"',
|
||
"name",
|
||
'":',
|
||
'▁"',
|
||
"get",
|
||
"_",
|
||
"we",
|
||
"ather",
|
||
'",',
|
||
'▁"',
|
||
"arguments",
|
||
'":',
|
||
'▁{"',
|
||
"city",
|
||
'":',
|
||
'▁"',
|
||
"Par",
|
||
"is",
|
||
'"},',
|
||
'▁"',
|
||
"id",
|
||
'":',
|
||
'▁"',
|
||
"1",
|
||
"2",
|
||
"3",
|
||
"4",
|
||
"5",
|
||
"6",
|
||
"7",
|
||
"8",
|
||
"9",
|
||
'"',
|
||
"}]",
|
||
"</s>",
|
||
"[TOOL_RESULTS]",
|
||
'▁{"',
|
||
"content",
|
||
'":',
|
||
'▁{"',
|
||
"t",
|
||
"emperature",
|
||
'":',
|
||
"▁",
|
||
"2",
|
||
"0",
|
||
",",
|
||
'▁"',
|
||
"unit",
|
||
'":',
|
||
'▁"',
|
||
"c",
|
||
"els",
|
||
"ius",
|
||
'"},',
|
||
'▁"',
|
||
"call",
|
||
"_",
|
||
"id",
|
||
'":',
|
||
'▁"',
|
||
"1",
|
||
"2",
|
||
"3",
|
||
"4",
|
||
"5",
|
||
"6",
|
||
"7",
|
||
"8",
|
||
"9",
|
||
'"}',
|
||
"[/TOOL_RESULTS]",
|
||
],
|
||
[
|
||
"<s>",
|
||
"[SYSTEM_PROMPT]",
|
||
"I",
|
||
" am",
|
||
" an",
|
||
" AI",
|
||
"[/SYSTEM_PROMPT]",
|
||
"[AVAILABLE_TOOLS]",
|
||
"[",
|
||
'{"',
|
||
"type",
|
||
'":',
|
||
' "',
|
||
"function",
|
||
'",',
|
||
' "',
|
||
"function",
|
||
'":',
|
||
' {"',
|
||
"name",
|
||
'":',
|
||
' "',
|
||
"get",
|
||
"_",
|
||
"weather",
|
||
'",',
|
||
' "',
|
||
"description",
|
||
'":',
|
||
' "',
|
||
"G",
|
||
"ets",
|
||
" the",
|
||
" current",
|
||
" weather",
|
||
" in",
|
||
" a",
|
||
" city",
|
||
'.",',
|
||
' "',
|
||
"parameters",
|
||
'":',
|
||
' {"',
|
||
"type",
|
||
'":',
|
||
' "',
|
||
"object",
|
||
'",',
|
||
' "',
|
||
"properties",
|
||
'":',
|
||
' {"',
|
||
"city",
|
||
'":',
|
||
' {"',
|
||
"type",
|
||
'":',
|
||
' "',
|
||
"string",
|
||
'",',
|
||
' "',
|
||
"description",
|
||
'":',
|
||
' "',
|
||
"The",
|
||
" city",
|
||
" name",
|
||
'"',
|
||
"}},",
|
||
' "',
|
||
"required",
|
||
'":',
|
||
' ["',
|
||
"city",
|
||
'"]',
|
||
"}}",
|
||
"}]",
|
||
"[/AVAILABLE_TOOLS]",
|
||
"[INST]",
|
||
"Hello",
|
||
" world",
|
||
" !",
|
||
"[/INST]",
|
||
"[TOOL_CALLS]",
|
||
"get",
|
||
"_",
|
||
"weather",
|
||
"[ARGS]",
|
||
'{"',
|
||
"city",
|
||
'":',
|
||
' "',
|
||
"Paris",
|
||
'"}',
|
||
"</s>",
|
||
"[TOOL_RESULTS]",
|
||
'{"',
|
||
"temperature",
|
||
'":',
|
||
" ",
|
||
"2",
|
||
"0",
|
||
",",
|
||
' "',
|
||
"unit",
|
||
'":',
|
||
' "',
|
||
"c",
|
||
"elsius",
|
||
'"}',
|
||
"[/TOOL_RESULTS]",
|
||
],
|
||
),
|
||
),
|
||
),
|
||
)
|
||
def test_convert_ids_to_tokens(
|
||
self,
|
||
mistral_tokenizer: MistralTokenizer,
|
||
skip_special_tokens: bool,
|
||
tuple_expected_tokens: tuple[list[str], list[str]],
|
||
):
|
||
tuple_ids = (
|
||
[
|
||
1,
|
||
6,
|
||
1501,
|
||
7567,
|
||
1891,
|
||
2032,
|
||
1113,
|
||
3396,
|
||
1316,
|
||
1113,
|
||
3396,
|
||
2032,
|
||
10598,
|
||
1629,
|
||
2032,
|
||
1113,
|
||
1295,
|
||
29498,
|
||
1537,
|
||
1991,
|
||
1316,
|
||
1113,
|
||
7286,
|
||
2032,
|
||
1113,
|
||
2226,
|
||
29481,
|
||
1040,
|
||
2636,
|
||
8854,
|
||
1065,
|
||
1032,
|
||
3758,
|
||
9959,
|
||
1113,
|
||
12206,
|
||
2032,
|
||
10598,
|
||
1891,
|
||
2032,
|
||
1113,
|
||
3582,
|
||
1316,
|
||
1113,
|
||
11491,
|
||
2032,
|
||
10598,
|
||
19141,
|
||
2032,
|
||
10598,
|
||
1891,
|
||
2032,
|
||
1113,
|
||
2195,
|
||
1316,
|
||
1113,
|
||
7286,
|
||
2032,
|
||
1113,
|
||
1782,
|
||
3758,
|
||
1909,
|
||
29507,
|
||
11549,
|
||
1113,
|
||
11661,
|
||
2032,
|
||
8135,
|
||
19141,
|
||
3010,
|
||
1743,
|
||
10925,
|
||
7,
|
||
3,
|
||
1083,
|
||
1605,
|
||
1164,
|
||
16875,
|
||
781,
|
||
781,
|
||
16998,
|
||
2294,
|
||
1686,
|
||
4,
|
||
5,
|
||
1501,
|
||
7567,
|
||
1629,
|
||
2032,
|
||
1113,
|
||
1295,
|
||
29498,
|
||
1537,
|
||
1991,
|
||
1316,
|
||
1113,
|
||
17452,
|
||
2032,
|
||
10598,
|
||
19141,
|
||
2032,
|
||
1113,
|
||
4684,
|
||
1046,
|
||
8474,
|
||
1113,
|
||
1081,
|
||
2032,
|
||
1113,
|
||
29508,
|
||
29518,
|
||
29538,
|
||
29549,
|
||
29550,
|
||
29552,
|
||
29555,
|
||
29551,
|
||
29542,
|
||
29507,
|
||
10925,
|
||
2,
|
||
8,
|
||
10598,
|
||
4557,
|
||
2032,
|
||
10598,
|
||
29475,
|
||
17329,
|
||
2032,
|
||
29473,
|
||
29518,
|
||
29502,
|
||
29493,
|
||
1113,
|
||
6074,
|
||
2032,
|
||
1113,
|
||
29485,
|
||
1958,
|
||
3938,
|
||
8474,
|
||
1113,
|
||
3613,
|
||
29498,
|
||
1081,
|
||
2032,
|
||
1113,
|
||
29508,
|
||
29518,
|
||
29538,
|
||
29549,
|
||
29550,
|
||
29552,
|
||
29555,
|
||
29551,
|
||
29542,
|
||
18163,
|
||
9,
|
||
],
|
||
[
|
||
1,
|
||
17,
|
||
1073,
|
||
1855,
|
||
1420,
|
||
26554,
|
||
18,
|
||
5,
|
||
1091,
|
||
19227,
|
||
4994,
|
||
2811,
|
||
1429,
|
||
5165,
|
||
1897,
|
||
1429,
|
||
5165,
|
||
2811,
|
||
16753,
|
||
2391,
|
||
2811,
|
||
1429,
|
||
1689,
|
||
1095,
|
||
45629,
|
||
1897,
|
||
1429,
|
||
14653,
|
||
2811,
|
||
1429,
|
||
1071,
|
||
3083,
|
||
1278,
|
||
3519,
|
||
17253,
|
||
1294,
|
||
1261,
|
||
5970,
|
||
39249,
|
||
1429,
|
||
26204,
|
||
2811,
|
||
16753,
|
||
4994,
|
||
2811,
|
||
1429,
|
||
6371,
|
||
1897,
|
||
1429,
|
||
48649,
|
||
2811,
|
||
16753,
|
||
29363,
|
||
2811,
|
||
16753,
|
||
4994,
|
||
2811,
|
||
1429,
|
||
3607,
|
||
1897,
|
||
1429,
|
||
14653,
|
||
2811,
|
||
1429,
|
||
1784,
|
||
5970,
|
||
2564,
|
||
1034,
|
||
47579,
|
||
1429,
|
||
15760,
|
||
2811,
|
||
12161,
|
||
29363,
|
||
4964,
|
||
2821,
|
||
27028,
|
||
6,
|
||
3,
|
||
22177,
|
||
4304,
|
||
2662,
|
||
4,
|
||
9,
|
||
1689,
|
||
1095,
|
||
45629,
|
||
32,
|
||
19227,
|
||
29363,
|
||
2811,
|
||
1429,
|
||
42572,
|
||
46005,
|
||
2,
|
||
7,
|
||
19227,
|
||
113824,
|
||
2811,
|
||
1032,
|
||
1050,
|
||
1048,
|
||
1044,
|
||
1429,
|
||
8979,
|
||
2811,
|
||
1429,
|
||
1099,
|
||
79092,
|
||
46005,
|
||
8,
|
||
],
|
||
)
|
||
|
||
ids = tuple_ids[mistral_tokenizer.is_tekken]
|
||
expected_tokens = tuple_expected_tokens[mistral_tokenizer.is_tekken]
|
||
actual_tokens = mistral_tokenizer.convert_ids_to_tokens(
|
||
ids, skip_special_tokens=skip_special_tokens
|
||
)
|
||
assert actual_tokens == expected_tokens
|
||
|
||
assert mistral_tokenizer.convert_ids_to_tokens([]) == []
|
||
|
||
def test_grammar_factory(self, mistral_tokenizer: MistralTokenizer) -> None:
|
||
# works in this case cause Mistral 7B is < v11 and SPM
|
||
if not mistral_tokenizer.is_tekken:
|
||
with pytest.raises(AttributeError):
|
||
mistral_tokenizer.grammar_factory # noqa: B018
|
||
return
|
||
factory = mistral_tokenizer.grammar_factory
|
||
assert isinstance(factory, GrammarFactory)
|
||
|
||
# Test caching
|
||
factory_2 = mistral_tokenizer.grammar_factory
|
||
assert factory is factory_2
|
||
|
||
def test_llg_tokenizer(self, mistral_tokenizer: MistralTokenizer) -> None:
|
||
if not mistral_tokenizer.is_tekken:
|
||
with pytest.raises(ValueError):
|
||
mistral_tokenizer.llg_tokenizer # noqa: B018
|
||
return
|
||
|
||
llg_tokenizer = mistral_tokenizer.llg_tokenizer
|
||
assert isinstance(llg_tokenizer, llguidance.LLTokenizer)
|
||
|
||
# Test caching
|
||
llg_tokenizer_2 = mistral_tokenizer.llg_tokenizer
|
||
assert llg_tokenizer is llg_tokenizer_2
|
||
|
||
@pytest.mark.parametrize(
|
||
"messages,tools,tekken_expected_substrings,spm_expected_substrings",
|
||
[
|
||
pytest.param(
|
||
[{"role": "user", "content": "Hello"}],
|
||
[{"type": "function", "function": {"name": "do_nothing"}}],
|
||
["do_nothing", '"description": ""', '"parameters": {}'],
|
||
["do_nothing", '"description":▁""', '"parameters":▁{}'],
|
||
id="tool_without_description_and_parameters",
|
||
),
|
||
pytest.param(
|
||
[
|
||
{"role": "user", "content": "Do nothing"},
|
||
{
|
||
"role": "assistant",
|
||
"content": "",
|
||
"tool_calls": [
|
||
{
|
||
"id": "123456789",
|
||
"type": "function",
|
||
"function": {
|
||
"name": "do_nothing",
|
||
"arguments": None,
|
||
},
|
||
}
|
||
],
|
||
},
|
||
{
|
||
"role": "tool",
|
||
"tool_call_id": "123456789",
|
||
"content": "done",
|
||
},
|
||
],
|
||
[{"type": "function", "function": {"name": "do_nothing"}}],
|
||
["do_nothing"],
|
||
["do_nothing"],
|
||
id="tool_call_with_none_arguments",
|
||
),
|
||
],
|
||
)
|
||
def test_apply_chat_template_tool_optional_fields(
|
||
self,
|
||
mistral_tokenizer: MistralTokenizer,
|
||
messages: list[ChatCompletionMessageParam],
|
||
tools: list[dict[str, Any]],
|
||
tekken_expected_substrings: list[str],
|
||
spm_expected_substrings: list[str],
|
||
) -> None:
|
||
output = mistral_tokenizer.apply_chat_template(
|
||
messages,
|
||
tools=tools,
|
||
add_generation_prompt=True,
|
||
)
|
||
decoded = mistral_tokenizer.tokenizer.decode(output, SpecialTokenPolicy.KEEP)
|
||
|
||
expected = (
|
||
tekken_expected_substrings
|
||
if mistral_tokenizer.is_tekken
|
||
else spm_expected_substrings
|
||
)
|
||
for substring in expected:
|
||
assert substring in decoded
|
||
|
||
def test_apply_chat_template_tools_not_mutated(
|
||
self, mistral_tokenizer: MistralTokenizer
|
||
) -> None:
|
||
messages: list[ChatCompletionMessageParam] = [
|
||
{"role": "user", "content": "Hello"},
|
||
]
|
||
tools: list[dict[str, Any]] = [
|
||
{
|
||
"type": "function",
|
||
"function": {
|
||
"name": "get_weather",
|
||
"description": "Gets weather.",
|
||
"parameters": {
|
||
"type": "object",
|
||
"properties": {
|
||
"city": {"type": "string"},
|
||
},
|
||
},
|
||
},
|
||
},
|
||
]
|
||
original_tools = copy.deepcopy(tools)
|
||
|
||
mistral_tokenizer.apply_chat_template(
|
||
messages,
|
||
tools=tools,
|
||
add_generation_prompt=True,
|
||
)
|
||
|
||
assert tools == original_tools
|
||
|
||
@pytest.mark.parametrize(
|
||
"reasoning_key",
|
||
["reasoning", "reasoning_content"],
|
||
)
|
||
def test_apply_chat_template_reasoning_assistant(
|
||
self, mistral_tokenizer: MistralTokenizer, reasoning_key: str
|
||
) -> None:
|
||
if not mistral_tokenizer.is_tekken:
|
||
pytest.skip("Reasoning tokens only supported on tekken tokenizers")
|
||
|
||
messages: list[dict[str, Any]] = [
|
||
{"role": "user", "content": "What is 2+2?"},
|
||
{
|
||
"role": "assistant",
|
||
"content": "4",
|
||
reasoning_key: "2+2 equals 4",
|
||
},
|
||
{"role": "user", "content": "Are you sure?"},
|
||
]
|
||
|
||
output = mistral_tokenizer.apply_chat_template(
|
||
messages,
|
||
add_generation_prompt=True,
|
||
)
|
||
decoded = mistral_tokenizer.tokenizer.decode(output, SpecialTokenPolicy.KEEP)
|
||
|
||
assert "[THINK]2+2 equals 4[/THINK]" in decoded
|
||
|
||
|
||
def test_convert_ids_to_tokens_pre_args_tekken():
|
||
"""convert_ids_to_tokens must not raise on pre-[ARGS] Tekken tokenizers.
|
||
|
||
Tekken tokenizers before v11 (e.g. Ministral-8B-Instruct-2410) have no
|
||
[ARGS] special token. This guards the is_special([ARGS]) gate in
|
||
MistralTokenizer.convert_ids_to_tokens, which must skip [ARGS] rather than
|
||
call get_special_token(args) unconditionally (which raises on Tekken).
|
||
"""
|
||
tokenizer = MistralTokenizer.from_pretrained("mistralai/Ministral-8B-Instruct-2410")
|
||
assert tokenizer.is_tekken
|
||
assert not tokenizer.tokenizer.is_special(SpecialTokens.args)
|
||
|
||
ids = tokenizer.encode("Hello world !", add_special_tokens=False)
|
||
tokens = tokenizer.convert_ids_to_tokens(ids, skip_special_tokens=True)
|
||
assert len(tokens) > 0
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# validate_request_params – reasoning_effort validation (no tokenizer needed)
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
@pytest.mark.parametrize("reasoning_effort", [None, "none", "high"])
|
||
def test_validate_request_params_valid_effort(reasoning_effort: str | None) -> None:
|
||
request = ChatCompletionRequest(
|
||
model="test-model",
|
||
messages=[],
|
||
reasoning_effort=reasoning_effort,
|
||
)
|
||
assert validate_request_params(request) is None
|
||
|
||
|
||
def test_validate_request_params_rejects_unsupported_effort() -> None:
|
||
request = ChatCompletionRequest(
|
||
model="test-model",
|
||
messages=[],
|
||
reasoning_effort="medium",
|
||
)
|
||
with pytest.raises(VLLMValidationError, match="reasoning_effort") as exc_info:
|
||
validate_request_params(request)
|
||
assert exc_info.value.parameter == "reasoning_effort"
|
||
|
||
|
||
def test_validate_request_params_rejects_chat_template() -> None:
|
||
request = ChatCompletionRequest(
|
||
model="test-model",
|
||
messages=[],
|
||
chat_template="{{ messages }}",
|
||
)
|
||
with pytest.raises(VLLMValidationError, match="chat_template") as exc_info:
|
||
validate_request_params(request)
|
||
assert exc_info.value.parameter == "chat_template"
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# apply_chat_template reasoning_effort passthrough (v15 tokenizer)
|
||
# ---------------------------------------------------------------------------
|
||
|
||
_PASSTHROUGH_MESSAGES = [{"role": "user", "content": "Hello"}]
|
||
|
||
|
||
@pytest.fixture(scope="module")
|
||
def v15_mistral_tokenizer() -> MistralTokenizer:
|
||
"""Load the v15 Mistral tokenizer; skip if unavailable."""
|
||
try:
|
||
return MistralTokenizer.from_pretrained("mistralai/Mistral-Small-4-119B-2603")
|
||
except Exception:
|
||
pytest.skip("v15 tokenizer unavailable")
|
||
raise AssertionError("unreachable")
|
||
|
||
|
||
@pytest.fixture(scope="module")
|
||
def v13_mistral_tokenizer() -> MistralTokenizer:
|
||
"""Load the v13 Mistral tokenizer; skip if unavailable."""
|
||
try:
|
||
return MistralTokenizer.from_pretrained("mistralai/Magistral-Small-2509")
|
||
except Exception:
|
||
pytest.skip("v13 tokenizer unavailable")
|
||
raise AssertionError("unreachable")
|
||
|
||
|
||
def test_v15_apply_chat_template_passes_reasoning_effort_high(
|
||
monkeypatch: pytest.MonkeyPatch,
|
||
v15_mistral_tokenizer: MistralTokenizer,
|
||
) -> None:
|
||
captured_kwargs: list[dict] = []
|
||
|
||
def fake_apply_chat_template(**kwargs):
|
||
captured_kwargs.append(kwargs)
|
||
return [0]
|
||
|
||
monkeypatch.setattr(
|
||
v15_mistral_tokenizer.transformers_tokenizer,
|
||
"apply_chat_template",
|
||
fake_apply_chat_template,
|
||
)
|
||
|
||
v15_mistral_tokenizer.apply_chat_template(
|
||
messages=_PASSTHROUGH_MESSAGES,
|
||
reasoning_effort="high",
|
||
)
|
||
|
||
assert captured_kwargs[-1]["reasoning_effort"] == "high"
|
||
|
||
|
||
def test_v15_apply_chat_template_passes_reasoning_effort_none_by_default(
|
||
monkeypatch: pytest.MonkeyPatch,
|
||
v15_mistral_tokenizer: MistralTokenizer,
|
||
) -> None:
|
||
captured_kwargs: list[dict] = []
|
||
|
||
def fake_apply_chat_template(**kwargs):
|
||
captured_kwargs.append(kwargs)
|
||
return [0]
|
||
|
||
monkeypatch.setattr(
|
||
v15_mistral_tokenizer.transformers_tokenizer,
|
||
"apply_chat_template",
|
||
fake_apply_chat_template,
|
||
)
|
||
|
||
v15_mistral_tokenizer.apply_chat_template(messages=_PASSTHROUGH_MESSAGES)
|
||
|
||
assert "reasoning_effort" in captured_kwargs[-1]
|
||
assert captured_kwargs[-1]["reasoning_effort"] is None
|
||
|
||
|
||
def test_pre_v15_apply_chat_template_omits_reasoning_effort(
|
||
monkeypatch: pytest.MonkeyPatch,
|
||
v13_mistral_tokenizer: MistralTokenizer,
|
||
) -> None:
|
||
captured_kwargs: list[dict] = []
|
||
|
||
def fake_apply_chat_template(**kwargs):
|
||
captured_kwargs.append(kwargs)
|
||
return [0]
|
||
|
||
monkeypatch.setattr(
|
||
v13_mistral_tokenizer.transformers_tokenizer,
|
||
"apply_chat_template",
|
||
fake_apply_chat_template,
|
||
)
|
||
|
||
v13_mistral_tokenizer.apply_chat_template(
|
||
messages=_PASSTHROUGH_MESSAGES,
|
||
reasoning_effort="high",
|
||
)
|
||
|
||
assert "reasoning_effort" not in captured_kwargs[-1]
|