[Model] Add Kimi K3 support: Python frontend [2/2] (#50093)

Signed-off-by: Bugen Zhao <i@bugenzhao.com>
This commit is contained in:
Bugen Zhao
2026-07-29 01:06:01 -07:00
committed by GitHub
parent 6370e53f24
commit f5a7cce9b6
20 changed files with 3056 additions and 5 deletions
@@ -0,0 +1,250 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
from vllm.entrypoints.openai.chat_completion.protocol import (
ChatCompletionRequest,
)
from vllm.entrypoints.openai.engine.protocol import DeltaMessage
from vllm.parser.kimi_k3 import KimiK3Parser
from vllm.parser.parser_manager import ParserManager
from vllm.reasoning.kimi_k3_reasoning_parser import KimiK3ReasoningParser
pytestmark = pytest.mark.skip_global_cleanup
OPEN = "<|open|>"
CLOSE = "<|close|>"
SEP = "<|sep|>"
THINK_OPEN = f"{OPEN}think{SEP}"
THINK_CLOSE = f"{CLOSE}think{SEP}"
RESPONSE_OPEN = f"{OPEN}response{SEP}"
class DummyTokenizer:
def get_vocab(self) -> dict[str, int]:
return {}
def encode(self, text: str, add_special_tokens: bool = False) -> list[int]:
if text == THINK_OPEN:
return [1, 2, 3]
if text == THINK_CLOSE:
return [4, 2, 3]
return [ord(ch) for ch in text]
class ReasoningOnlyParser(KimiK3Parser):
reasoning_parser_cls = KimiK3ReasoningParser
def test_parser_manager_selects_kimi_k3_parser_for_reasoning_only():
parser_cls = ParserManager.get_parser(reasoning_parser_name="kimi_k3")
assert parser_cls is not None
assert issubclass(parser_cls, KimiK3Parser)
assert parser_cls.reasoning_parser_cls is KimiK3ReasoningParser
assert parser_cls.tool_parser_cls is None
def test_parser_selection_thinking_disabled():
parser = KimiK3ReasoningParser(
DummyTokenizer(), chat_template_kwargs={"thinking": False}
)
assert parser._thinking_enabled is False
def test_extract_reasoning_with_xtml_tags():
parser = KimiK3ReasoningParser(DummyTokenizer())
request = ChatCompletionRequest(model="test-model", messages=[])
reasoning, content = parser.extract_reasoning_content(
f"{THINK_OPEN}step{THINK_CLOSE}{RESPONSE_OPEN}answer",
request,
)
assert reasoning == "step"
assert content == "answer"
def test_extract_reasoning_with_generation_prefix_consumed():
parser = KimiK3ReasoningParser(DummyTokenizer())
request = ChatCompletionRequest(model="test-model", messages=[])
reasoning, content = parser.extract_reasoning_content(
f"step{THINK_CLOSE}{RESPONSE_OPEN}answer",
request,
)
assert reasoning == "step"
assert content == "answer"
def test_delegating_parser_strips_response_wrapper_without_tool_parser():
parser = ReasoningOnlyParser(DummyTokenizer())
request = ChatCompletionRequest(model="test-model", messages=[])
reasoning, content, tool_calls = parser.parse(
f"{THINK_OPEN}step{THINK_CLOSE}{RESPONSE_OPEN}answer",
request,
)
assert reasoning == "step"
assert content == "answer"
assert tool_calls == []
def test_is_reasoning_end_uses_full_input_ids():
parser = KimiK3ReasoningParser(DummyTokenizer())
assert not parser.is_reasoning_end([4, 2])
assert parser.is_reasoning_end([4, 2, 3])
def test_is_reasoning_end_ignores_stale_close_from_prior_turn():
# DummyTokenizer: THINK_OPEN -> [1, 2, 3], THINK_CLOSE -> [4, 2, 3].
# Multi-turn / agent continuation: a prior turn's think channel (its close
# marker) is kept in the prompt, then the current turn opens a new think
# block that has not closed yet. Reasoning must read as NOT ended, otherwise
# the structured-output gate constrains the current turn's reasoning.
parser = KimiK3ReasoningParser(DummyTokenizer())
stale_close = [4, 2, 3]
new_open = [1, 2, 3]
# prior close, then current-turn open still unclosed -> not ended
assert not parser.is_reasoning_end([*stale_close, *new_open])
# ...then the current turn emits its own close -> ended
assert parser.is_reasoning_end([*stale_close, *new_open, *stale_close])
# open with no close yet -> not ended
assert not parser.is_reasoning_end([*new_open])
def test_streaming_split_open_marker_is_held_back():
parser = KimiK3ReasoningParser(DummyTokenizer())
first = parser.extract_reasoning_content_streaming(
previous_text="",
current_text=OPEN,
delta_text=OPEN,
previous_token_ids=[],
current_token_ids=[1],
delta_token_ids=[1],
)
second = parser.extract_reasoning_content_streaming(
previous_text=OPEN,
current_text=f"{OPEN}think",
delta_text="think",
previous_token_ids=[1],
current_token_ids=[1, 2],
delta_token_ids=[2],
)
third = parser.extract_reasoning_content_streaming(
previous_text=f"{OPEN}think",
current_text=THINK_OPEN + "step",
delta_text=f"{SEP}step",
previous_token_ids=[1, 2],
current_token_ids=[1, 2, 3, 9],
delta_token_ids=[3, 9],
)
assert first is None
assert second is None
assert isinstance(third, DeltaMessage)
assert third.reasoning == "step"
def test_streaming_split_close_marker_hands_content_downstream():
parser = KimiK3ReasoningParser(DummyTokenizer())
previous_text = f"{THINK_OPEN}step"
partial_close = parser.extract_reasoning_content_streaming(
previous_text=previous_text,
current_text=previous_text + CLOSE,
delta_text=CLOSE,
previous_token_ids=[1, 2, 3, 9],
current_token_ids=[1, 2, 3, 9, 4],
delta_token_ids=[4],
)
closed = parser.extract_reasoning_content_streaming(
previous_text=previous_text + CLOSE,
current_text=previous_text + f"{THINK_CLOSE}{RESPONSE_OPEN}answer",
delta_text=f"think{SEP}{RESPONSE_OPEN}answer",
previous_token_ids=[1, 2, 3, 9, 4],
current_token_ids=[1, 2, 3, 9, 4, 2, 3, 10],
delta_token_ids=[2, 3, 10],
)
assert partial_close is None
assert isinstance(closed, DeltaMessage)
assert closed.reasoning is None
assert closed.content == f"{RESPONSE_OPEN}answer"
assert parser.extract_content_ids([2, 3, 10]) == [10]
def test_thinking_disabled_streams_content():
parser = KimiK3ReasoningParser(
DummyTokenizer(), chat_template_kwargs={"enable_thinking": False}
)
delta = parser.extract_reasoning_content_streaming(
previous_text="",
current_text=f"{RESPONSE_OPEN}answer",
delta_text=f"{RESPONSE_OPEN}answer",
previous_token_ids=[],
current_token_ids=[1],
delta_token_ids=[1],
)
assert isinstance(delta, DeltaMessage)
assert delta.content == f"{RESPONSE_OPEN}answer"
assert delta.reasoning is None
def test_delegating_parser_thinking_false_streams_response_content():
parser = ReasoningOnlyParser(
DummyTokenizer(), chat_template_kwargs={"thinking": False}
)
request = ChatCompletionRequest(
model="test-model",
messages=[],
chat_template_kwargs={"thinking": False},
)
first = parser.parse_delta(
delta_text="OK",
delta_token_ids=[10],
request=request,
prompt_token_ids=[1],
finished=False,
)
partial_close = parser.parse_delta(
delta_text=CLOSE,
delta_token_ids=[2],
request=request,
prompt_token_ids=[1],
finished=False,
)
closed = parser.parse_delta(
delta_text=f"response{SEP}",
delta_token_ids=[3, 4],
request=request,
prompt_token_ids=[1],
finished=False,
)
assert first is not None
assert first.content == "OK"
assert first.reasoning is None
assert partial_close is None
assert closed is None
def test_adjust_request_keeps_xtml_markers_contiguous():
parser = KimiK3ReasoningParser(DummyTokenizer())
request = ChatCompletionRequest(model="test-model", messages=[])
adjusted = parser.adjust_request(request)
assert adjusted.skip_special_tokens is False
if hasattr(adjusted, "spaces_between_special_tokens"):
assert adjusted.spaces_between_special_tokens is False
+353
View File
@@ -0,0 +1,353 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from dataclasses import dataclass, field
from typing import Any
import pytest
from vllm.exceptions import VLLMValidationError
from vllm.renderers import ChatParams
from vllm.renderers.kimi_k3 import KimiK3Renderer, _merge_k3_media_io_kwargs
from vllm.renderers.registry import RENDERER_REGISTRY
from vllm.tokenizers.registry import TokenizerRegistry
class StubTokenizer:
"""Stands in for the model's TikTokenTokenizer.
Records the kwargs it is called with and returns fixed token ids, so tests
can assert how the renderer drives ``apply_chat_template`` without the real
(git-LFS) tiktoken vocabulary.
"""
def __init__(self, token_ids: list[int]) -> None:
self.token_ids = token_ids
self.calls: list[dict[str, Any]] = []
self.conversations: list[list[dict[str, Any]]] = []
def apply_chat_template(self, conversation, **kwargs) -> list[int]:
self.conversations.append(conversation)
self.calls.append(kwargs)
return list(self.token_ids)
@dataclass
class MockHFConfig:
model_type: str = "kimi_k3"
@dataclass
class MockModelConfig:
runner_type: str = "generate"
is_multimodal_model: bool = False
multimodal_config: Any = None
hf_config: MockHFConfig = field(default_factory=MockHFConfig)
allowed_local_media_path: str = ""
allowed_media_domains: Any = None
enable_prompt_embeds: bool = False
renderer_num_workers: int = 1
@dataclass
class MockParallelConfig:
_api_process_rank: int = 0
@dataclass
class MockVllmConfig:
model_config: MockModelConfig
parallel_config: MockParallelConfig
def _make_renderer(tokenizer: StubTokenizer) -> KimiK3Renderer:
config = MockVllmConfig(MockModelConfig(), MockParallelConfig())
return KimiK3Renderer(config, tokenizer)
def test_kimi_k3_registered():
assert RENDERER_REGISTRY.load_renderer_cls("kimi_k3").__name__ == "KimiK3Renderer"
assert (
TokenizerRegistry.load_tokenizer_cls("kimi_k3").__name__ == "CachedHfTokenizer"
)
def test_k3_media_io_defaults_preserve_original_mode():
# Default: K3 keeps the original image mode (no background flattening).
assert _merge_k3_media_io_kwargs(None) == {"image": {"image_mode": None}}
# Server-/request-level values take precedence over the K3 default.
assert _merge_k3_media_io_kwargs({"image": {"image_mode": "RGB"}}) == {
"image": {"image_mode": "RGB"}
}
# Unrelated image kwargs are merged with the default.
assert _merge_k3_media_io_kwargs(
{"image": {"rgba_background_color": (0, 0, 0)}}
) == {"image": {"image_mode": None, "rgba_background_color": (0, 0, 0)}}
def test_apply_chat_template_forces_tokenize_and_pins_return_dict():
tokenizer = StubTokenizer([7, 8, 9])
renderer = _make_renderer(tokenizer)
tools = [{"type": "function", "function": {"name": "search"}}]
params = ChatParams(
chat_template_kwargs={"tools": tools, "tokenize": False, "thinking": True}
)
token_ids = renderer._apply_chat_template(
[{"role": "user", "content": "hi"}], params
)
assert token_ids == [7, 8, 9]
kwargs = tokenizer.calls[-1]
# tokenize is forced on even though the request asked for False, so K3 keeps
# the special-vs-ordinary token distinction instead of re-tokenizing a string.
assert kwargs["tokenize"] is True
# return_dict is pinned False so we always get a flat list of ids.
assert kwargs["return_dict"] is False
assert kwargs["tools"] == tools
assert kwargs["thinking"] is True
def test_apply_chat_template_translates_standard_thinking_kwargs():
# Standard enable_thinking/reasoning_effort kwargs must be translated
# to K3's native thinking/thinking_effort.
tokenizer = StubTokenizer([7, 8, 9])
renderer = _make_renderer(tokenizer)
params = ChatParams(
chat_template_kwargs={"enable_thinking": False, "reasoning_effort": "none"}
)
renderer._apply_chat_template([{"role": "user", "content": "hi"}], params)
kwargs = tokenizer.calls[-1]
assert kwargs["thinking"] is False
assert "thinking_effort" not in kwargs
assert "enable_thinking" not in kwargs
assert "reasoning_effort" not in kwargs
@pytest.mark.parametrize("reasoning_effort", ["low", "high", "max"])
def test_apply_chat_template_translates_supported_reasoning_effort(
reasoning_effort: str,
):
tokenizer = StubTokenizer([7, 8, 9])
renderer = _make_renderer(tokenizer)
params = ChatParams(chat_template_kwargs={"reasoning_effort": reasoning_effort})
renderer._apply_chat_template([{"role": "user", "content": "hi"}], params)
kwargs = tokenizer.calls[-1]
assert kwargs["thinking_effort"] == reasoning_effort
assert "reasoning_effort" not in kwargs
@pytest.mark.parametrize("reasoning_effort", ["minimal", "medium", "xhigh"])
def test_apply_chat_template_rejects_unsupported_reasoning_effort(
reasoning_effort: str,
):
tokenizer = StubTokenizer([7, 8, 9])
renderer = _make_renderer(tokenizer)
params = ChatParams(chat_template_kwargs={"reasoning_effort": reasoning_effort})
with pytest.raises(VLLMValidationError, match="thinking_effort") as exc_info:
renderer._apply_chat_template([{"role": "user", "content": "hi"}], params)
assert exc_info.value.parameter == "thinking_effort"
assert exc_info.value.value == reasoning_effort
assert tokenizer.calls == []
def test_apply_chat_template_validates_canonical_native_thinking_effort():
tokenizer = StubTokenizer([7, 8, 9])
renderer = _make_renderer(tokenizer)
params = ChatParams(
chat_template_kwargs={
"thinking_effort": "low",
"reasoning_effort": "medium",
}
)
renderer._apply_chat_template([{"role": "user", "content": "hi"}], params)
assert tokenizer.calls[-1]["thinking_effort"] == "low"
@pytest.mark.parametrize("thinking_effort", ["none", "minimal", "medium", "xhigh"])
def test_apply_chat_template_rejects_unsupported_native_thinking_effort(
thinking_effort: str,
):
tokenizer = StubTokenizer([7, 8, 9])
renderer = _make_renderer(tokenizer)
params = ChatParams(chat_template_kwargs={"thinking_effort": thinking_effort})
with pytest.raises(VLLMValidationError, match="thinking_effort") as exc_info:
renderer._apply_chat_template([{"role": "user", "content": "hi"}], params)
assert exc_info.value.parameter == "thinking_effort"
assert exc_info.value.value == thinking_effort
assert tokenizer.calls == []
def test_apply_chat_template_native_k3_kwargs_take_precedence():
tokenizer = StubTokenizer([7, 8, 9])
renderer = _make_renderer(tokenizer)
params = ChatParams(
chat_template_kwargs={
"thinking": True,
"enable_thinking": False,
"thinking_effort": "low",
"reasoning_effort": "high",
}
)
renderer._apply_chat_template([{"role": "user", "content": "hi"}], params)
kwargs = tokenizer.calls[-1]
assert kwargs["thinking"] is True
assert kwargs["thinking_effort"] == "low"
def test_apply_chat_template_adds_k3_api_metadata():
tokenizer = StubTokenizer([7, 8, 9])
renderer = _make_renderer(tokenizer)
response_format = {"type": "json_object"}
params = ChatParams(
tool_choice="required",
response_format=response_format,
)
renderer._apply_chat_template([{"role": "user", "content": "hi"}], params)
kwargs = tokenizer.calls[-1]
assert kwargs["tool_choice"] == "required"
assert kwargs["response_format"] == response_format
def test_apply_chat_template_auto_tool_choice_keeps_template_kwarg():
tokenizer = StubTokenizer([7, 8, 9])
renderer = _make_renderer(tokenizer)
params = ChatParams(
chat_template_kwargs={"tool_choice": "required"},
tool_choice="auto",
)
renderer._apply_chat_template([{"role": "user", "content": "hi"}], params)
assert tokenizer.calls[-1]["tool_choice"] == "required"
def test_apply_chat_template_omits_tool_choice_without_tools():
tokenizer = StubTokenizer([7, 8, 9])
renderer = _make_renderer(tokenizer)
renderer._apply_chat_template(
[{"role": "user", "content": "hi"}], ChatParams(tool_choice=None)
)
assert "tool_choice" not in tokenizer.calls[-1]
def test_render_messages_returns_token_prompt():
renderer = _make_renderer(StubTokenizer([1, 2, 3]))
conversation, prompt = renderer.render_messages(
[{"role": "user", "content": "hi"}], ChatParams()
)
assert prompt == {"prompt_token_ids": [1, 2, 3]}
assert "multi_modal_data" not in prompt
assert conversation[0]["role"] == "user"
def test_render_messages_derives_private_xtml_tool_attrs():
tokenizer = StubTokenizer([1, 2, 3])
renderer = _make_renderer(tokenizer)
conversation, _ = renderer.render_messages(
[
{
"role": "assistant",
"content": "",
"tool_calls": [
{
"id": "lookup:0",
"type": "function",
"function": {"name": "lookup", "arguments": "{}"},
},
{
"id": "lookup:1",
"type": "function",
"function": {"name": "lookup", "arguments": "{}"},
},
],
},
{
"role": "tool",
"tool_call_id": "lookup:1",
"tool": "client-supplied-name",
"index": 99,
"content": "second",
},
{
"role": "tool",
"tool_call_id": "lookup:0",
"content": "first",
},
],
ChatParams(),
)
assert [message["content"] for message in conversation[1:]] == [
"first",
"second",
]
assert conversation[1]["tool"] == "lookup"
assert conversation[1]["index"] == 1
assert conversation[2]["tool"] == "lookup"
assert conversation[2]["index"] == 2
assert tokenizer.conversations[-1] == conversation
def test_render_messages_ignores_client_supplied_xtml_tool_attrs():
tokenizer = StubTokenizer([1, 2, 3])
renderer = _make_renderer(tokenizer)
conversation, _ = renderer.render_messages(
[
{
"role": "assistant",
"content": "",
"tool_calls": [
{
"id": "lookup:0",
"type": "function",
"function": {"name": "lookup", "arguments": "{}"},
}
],
},
{
"role": "tool",
"tool_call_id": "unknown",
"tool": "lookup",
"index": 1,
"content": "result",
},
],
ChatParams(),
)
assert "tool" not in conversation[1]
assert "index" not in conversation[1]
@pytest.mark.asyncio
async def test_render_messages_async_returns_token_prompt():
renderer = _make_renderer(StubTokenizer([4, 5]))
conversation, prompt = await renderer.render_messages_async(
[{"role": "user", "content": "hi"}], ChatParams()
)
assert prompt == {"prompt_token_ids": [4, 5]}
assert conversation[0]["role"] == "user"
@@ -0,0 +1,62 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Named tool choice for Kimi K3: allowed when the XTML structural tag is
attached (strict tool calling), rejected otherwise."""
import pytest
from vllm.entrypoints.openai.chat_completion.protocol import (
ChatCompletionRequest,
)
from vllm.exceptions import VLLMValidationError
from vllm.sampling_params import StructuredOutputsParams
class _DummyTokenizer:
def get_vocab(self):
return {}
def encode(self, text, add_special_tokens=False):
return [ord(c) for c in text]
def _request(with_tag: bool) -> ChatCompletionRequest:
req = ChatCompletionRequest(
model="k3",
messages=[{"role": "user", "content": "hi"}],
tools=[
{
"type": "function",
"function": {
"name": "get_weather",
"parameters": {
"type": "object",
"properties": {"city": {"type": "string"}},
"required": ["city"],
},
},
}
],
tool_choice={"type": "function", "function": {"name": "get_weather"}},
)
if with_tag:
req.structured_outputs = StructuredOutputsParams(
structural_tag='{"type": "structural_tag", "format": {}}'
)
return req
def _parser():
from vllm.tool_parsers.kimi_k3_tool_parser import KimiK3ToolParser
return KimiK3ToolParser(_DummyTokenizer())
def test_named_choice_allowed_with_structural_tag():
req = _parser().adjust_request(_request(with_tag=True))
assert req.skip_special_tokens is False
def test_named_choice_rejected_without_structural_tag():
with pytest.raises(VLLMValidationError):
_parser().adjust_request(_request(with_tag=False))
@@ -5,7 +5,8 @@ from types import SimpleNamespace
from unittest.mock import MagicMock
import pytest
from xgrammar import StructuralTag
from xgrammar import Grammar, StructuralTag
from xgrammar.testing import _is_grammar_accept_string
from vllm.entrypoints.openai.chat_completion.protocol import (
ChatCompletionNamedFunction,
@@ -24,6 +25,7 @@ from vllm.tool_parsers.deepseekv32_engine_tool_parser import (
from vllm.tool_parsers.glm47_moe_tool_parser import Glm47MoeModelToolParser
from vllm.tool_parsers.hermes_tool_parser import Hermes2ProToolParser
from vllm.tool_parsers.kimi_k2_tool_parser import KimiK2ToolParser
from vllm.tool_parsers.kimi_k3_tool_parser import KimiK3ToolParser
from vllm.tool_parsers.llama_tool_parser import Llama3JsonToolParser
from vllm.tool_parsers.minimax_m2_tool_parser import MinimaxM2ToolParser
from vllm.tool_parsers.qwen3_engine_tool_parser import Qwen3EngineToolParser
@@ -165,6 +167,196 @@ def test_hermes_required_tool_calls_use_empty_separator():
assert tag.format.separator == ""
# ---------------------------------------------------------------------------
# Kimi K3 (XTML channel format) structural tag
# ---------------------------------------------------------------------------
_K3_RESPONSE_OPEN = "<|open|>response<|sep|>"
_K3_RESPONSE_CLOSE = "<|close|>response<|sep|>"
_K3_TOOLS_OPEN = "<|open|>tools<|sep|>"
_K3_TOOLS_CLOSE = "<|close|>tools<|sep|>"
_K3_CALL_CLOSE = "<|close|>call<|sep|>"
_K3_ARG_CLOSE = "<|close|>argument<|sep|>"
_K3_MESSAGE_CLOSE = "<|close|>message<|sep|>"
def _k3_tools_by_name() -> list[ChatCompletionToolsParam]:
return [
ChatCompletionToolsParam(
type="function",
function={
"name": "get_weather",
"parameters": {
"type": "object",
"properties": {
"city": {"type": "string"},
"days": {"type": "integer"},
},
"required": ["city"],
},
},
),
ChatCompletionToolsParam(
type="function",
function={
"name": "run_command",
"parameters": {
"type": "object",
"properties": {"command": {"type": "string"}},
"required": ["command"],
},
},
),
]
def _k3_arg(key: str, typ: str, val: str) -> str:
return f'<|open|>argument key="{key}" type="{typ}"<|sep|>{val}{_K3_ARG_CLOSE}'
def _k3_call(name: str, args: str, idx: int = 1) -> str:
return f'<|open|>call tool="{name}" index="{idx}"<|sep|>{args}{_K3_CALL_CLOSE}'
def _k3_response(content: str = "") -> str:
return f"{_K3_RESPONSE_OPEN}{content}{_K3_RESPONSE_CLOSE}"
def _k3_tools(*calls: str) -> str:
return f"{_K3_TOOLS_OPEN}{''.join(calls)}{_K3_TOOLS_CLOSE}"
def _k3_grammar(tool_choice, tools=None):
tag = get_model_structural_tag(
model="kimi_k3",
tools=tools if tools is not None else _k3_tools_by_name(),
tool_choice=tool_choice,
reasoning=False,
)
assert isinstance(tag, StructuralTag)
return Grammar.from_structural_tag(tag)
def test_kimi_k3_registered_as_vllm_builtin():
assert "kimi_k3" in VLLM_BUILTIN_STRUCTURAL_TAG_MODELS
assert KimiK3ToolParser.structural_tag_model == "kimi_k3"
def test_kimi_k3_auto_without_strict_is_unconstrained():
# auto + no strict tool => no structural tag (matches the strict gate).
tag = get_model_structural_tag(
model="kimi_k3",
tools=_k3_tools_by_name(),
tool_choice="auto",
reasoning=False,
)
assert tag is None
@pytest.mark.parametrize(
"body",
[
# single required arg
_k3_response()
+ _k3_tools(_k3_call("get_weather", _k3_arg("city", "string", "Paris"))),
# response content + two args (string + number)
_k3_response("Checking.")
+ _k3_tools(
_k3_call(
"get_weather",
_k3_arg("city", "string", "Paris") + _k3_arg("days", "number", "3"),
)
),
# args in reverse order (parser is order-agnostic)
_k3_response()
+ _k3_tools(
_k3_call(
"get_weather",
_k3_arg("days", "number", "3") + _k3_arg("city", "string", "Paris"),
)
),
# two calls, second tool
_k3_response()
+ _k3_tools(
_k3_call("get_weather", _k3_arg("city", "string", "Paris"), 1),
_k3_call("run_command", _k3_arg("command", "string", "ls -la"), 2),
),
# string value with regex metacharacters / spaces
_k3_response()
+ _k3_tools(
_k3_call(
"run_command", _k3_arg("command", "string", "grep -E 'a|b{2,}' x.py")
)
),
# trailing message-close marker (model's natural turn terminator)
_k3_response()
+ _k3_tools(_k3_call("get_weather", _k3_arg("city", "string", "Paris")))
+ _K3_MESSAGE_CLOSE,
# non-thinking mode: response-open is the prompt prefix, so it is absent
_K3_RESPONSE_CLOSE
+ _k3_tools(_k3_call("get_weather", _k3_arg("city", "string", "Paris"))),
],
)
def test_kimi_k3_required_accepts_valid_tool_calls(body: str):
assert _is_grammar_accept_string(_k3_grammar("required"), body)
@pytest.mark.parametrize(
"body",
[
# unknown tool name
_k3_response()
+ _k3_tools(_k3_call("get_temperature", _k3_arg("city", "string", "x"))),
# number arg given a non-numeric JSON value
_k3_response()
+ _k3_tools(_k3_call("get_weather", _k3_arg("days", "number", "abc"))),
# undeclared argument key
_k3_response()
+ _k3_tools(
_k3_call(
"get_weather",
_k3_arg("city", "string", "Paris") + _k3_arg("zzz", "string", "x"),
)
),
# required schema but no argument tags
_k3_response() + _k3_tools(_k3_call("get_weather", "")),
# missing tools close marker
_k3_response()
+ _K3_TOOLS_OPEN
+ _k3_call("get_weather", _k3_arg("city", "string", "Paris")),
# required but no tool call
_k3_response("hello"),
],
)
def test_kimi_k3_required_rejects_invalid(body: str):
assert not _is_grammar_accept_string(_k3_grammar("required"), body)
def test_kimi_k3_schema_without_required_accepts_empty_call():
tools = [
ChatCompletionToolsParam(
type="function",
function={
"name": "get_weather",
"parameters": {
"type": "object",
"properties": {"city": {"type": "string"}},
},
},
)
]
grammar = _k3_grammar("required", tools=tools)
body = _k3_response() + _k3_tools(_k3_call("get_weather", ""))
assert _is_grammar_accept_string(grammar, body)
def test_kimi_k3_auto_strict_allows_response_only(sample_tools_strict):
# With a strict tool the tag is built; the tools channel is optional so a
# plain response (no tool call) is still valid.
grammar = _k3_grammar("auto", tools=sample_tools_strict)
assert _is_grammar_accept_string(grammar, _k3_response("Just answering."))
@pytest.mark.parametrize("model", sorted(XGRAMMAR_BUILTIN_STRUCTURAL_TAG_MODELS))
def test_get_model_structural_tag_supports_named_tool_choice(
model: str,
@@ -339,3 +531,151 @@ def test_get_function_parameters_relaxes_function_strict_false():
)
assert _get_function_parameters(function) is True
def _k3_tools_with_root_defs() -> list[ChatCompletionToolsParam]:
return [
ChatCompletionToolsParam(
type="function",
function={
"name": "make_config",
"parameters": {
"type": "object",
"properties": {
"config": {
"type": "object",
"properties": {
"build": {"$ref": "#/$defs/build"},
"index": {"type": "string"},
},
"required": ["index"],
"additionalProperties": False,
},
},
"required": ["config"],
"$defs": {
"build": {
"type": "object",
"properties": {"outDir": {"type": "string"}},
"additionalProperties": False,
}
},
},
},
)
]
def test_kimi_k3_property_ref_to_root_defs_compiles_and_accepts():
# Root-level $defs referenced from inside a property schema (the walle
# TestReferences shape). Slicing the property out of the parameters
# document orphans "#/$defs/..." unless the builder re-attaches $defs;
# before the fix Grammar.from_structural_tag raised on the dangling ref.
grammar = _k3_grammar("required", tools=_k3_tools_with_root_defs())
body = _k3_response() + _k3_tools(
_k3_call(
"make_config",
_k3_arg(
"config",
"object",
'{"build": {"outDir": "dist"}, "index": "a.html"}',
),
)
)
assert _is_grammar_accept_string(grammar, body)
def _k3_tools_with_string_enum() -> list[ChatCompletionToolsParam]:
return [
ChatCompletionToolsParam(
type="function",
function={
"name": "set_unit",
"parameters": {
"type": "object",
"properties": {
"unit": {
"type": "string",
"enum": ["celsius", " fahrenheit", "\tkelvin"],
},
},
"required": ["unit"],
},
},
)
]
@pytest.mark.parametrize("value", ["celsius", " fahrenheit", "\tkelvin"])
def test_kimi_k3_string_enum_accepts_exact_values(value: str):
# Raw string channel with enum: constrained to the exact enum values,
# including leading-whitespace variants the model otherwise flubs.
grammar = _k3_grammar("required", tools=_k3_tools_with_string_enum())
body = _k3_response() + _k3_tools(
_k3_call("set_unit", _k3_arg("unit", "string", value))
)
assert _is_grammar_accept_string(grammar, body)
@pytest.mark.parametrize("value", ["kelvin", "Celsius", "celsius ", ""])
def test_kimi_k3_string_enum_rejects_non_members(value: str):
grammar = _k3_grammar("required", tools=_k3_tools_with_string_enum())
body = _k3_response() + _k3_tools(
_k3_call("set_unit", _k3_arg("unit", "string", value))
)
assert not _is_grammar_accept_string(grammar, body)
def _k3_tools_with_maxlen() -> list[ChatCompletionToolsParam]:
return [
ChatCompletionToolsParam(
type="function",
function={
"name": "set_note",
"parameters": {
"type": "object",
"properties": {
"note": {"type": "string", "maxLength": 8, "minLength": 2},
},
"required": ["note"],
},
},
)
]
def test_kimi_k3_string_maxlength_bounds_raw_channel():
# Raw string channel with maxLength/minLength: enforced via a bounded
# regex that keeps the "<|" marker prefix unambiguous but still allows
# a bare '<' inside values.
grammar = _k3_grammar("required", tools=_k3_tools_with_maxlen())
def body(val: str) -> str:
return _k3_response() + _k3_tools(
_k3_call("set_note", _k3_arg("note", "string", val))
)
assert _is_grammar_accept_string(grammar, body("ab"))
assert _is_grammar_accept_string(grammar, body("a<b then"))
assert not _is_grammar_accept_string(grammar, body("way too long note"))
assert not _is_grammar_accept_string(grammar, body("a")) # under minLength
def test_kimi_k3_forced_tool_choice_builds_single_mandatory_call():
# Named tool choice normalizes to "forced": the tag must require exactly
# the named tool's call (no response-only escape).
grammar = _k3_grammar(
ChatCompletionNamedToolChoiceParam(
type="function",
function=ChatCompletionNamedFunction(name="get_weather"),
),
tools=_k3_tools_by_name(),
)
ok = _k3_response() + _k3_tools(
_k3_call("get_weather", _k3_arg("city", "string", "Paris"))
)
response_only = _k3_response("no call here")
assert _is_grammar_accept_string(grammar, ok)
assert not _is_grammar_accept_string(grammar, response_only)
+600
View File
@@ -0,0 +1,600 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import json
import pytest
from vllm.entrypoints.openai.chat_completion.protocol import (
ChatCompletionRequest,
)
from vllm.entrypoints.openai.engine.protocol import DeltaMessage
from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
from vllm.entrypoints.openai.responses.utils import build_response_output_items
from vllm.exceptions import VLLMValidationError
from vllm.parser.kimi_k3 import KimiK3Parser
from vllm.parser.parser_manager import ParserManager
from vllm.reasoning.kimi_k3_reasoning_parser import KimiK3ReasoningParser
from vllm.tool_parsers.kimi_k3_tool_parser import KimiK3ToolParser
OPEN = "<|open|>"
CLOSE = "<|close|>"
SEP = "<|sep|>"
THINK_OPEN = f"{OPEN}think{SEP}"
THINK_CLOSE = f"{CLOSE}think{SEP}"
RESPONSE_CLOSE = f"{CLOSE}response{SEP}"
class DummyTokenizer:
def get_vocab(self) -> dict[str, int]:
return {}
def encode(self, text: str, add_special_tokens: bool = False) -> list[int]:
if text == THINK_OPEN:
return [1, 2, 3]
if text == THINK_CLOSE:
return [4, 2, 3]
return [ord(ch) for ch in text]
class KimiK3DelegatingParser(KimiK3Parser):
reasoning_parser_cls = KimiK3ReasoningParser
tool_parser_cls = KimiK3ToolParser
def test_parser_manager_selects_kimi_k3_parser():
parser_cls = ParserManager.get_parser(
tool_parser_name="kimi_k3",
reasoning_parser_name="kimi_k3",
enable_auto_tools=True,
)
assert parser_cls is not None
assert issubclass(parser_cls, KimiK3Parser)
assert parser_cls.reasoning_parser_cls is KimiK3ReasoningParser
assert parser_cls.tool_parser_cls is KimiK3ToolParser
def _request() -> ChatCompletionRequest:
return ChatCompletionRequest(
model="test-model",
messages=[],
tools=[
{
"type": "function",
"function": {
"name": "calc",
"parameters": {"type": "object", "properties": {}},
},
}
],
tool_choice="auto",
)
def _named_request() -> ChatCompletionRequest:
return ChatCompletionRequest(
model="test-model",
messages=[],
tools=[
{
"type": "function",
"function": {
"name": "calc",
"parameters": {"type": "object", "properties": {}},
},
}
],
tool_choice={"type": "function", "function": {"name": "calc"}},
)
def _responses_request(*, tool_choice="auto") -> ResponsesRequest:
return ResponsesRequest.model_validate(
{
"model": "test-model",
"input": "Call the calc tool.",
"tools": [
{
"type": "function",
"name": "calc",
"parameters": {"type": "object", "properties": {}},
}
],
"tool_choice": tool_choice,
}
)
def _arg(key: str, typ: str, value: str) -> str:
return f'{OPEN}argument key="{key}" type="{typ}"{SEP}{value}{CLOSE}argument{SEP}'
def _call(tool: str, index: int, *args: str) -> str:
body = "".join(args)
return f'{OPEN}call tool="{tool}" index="{index}"{SEP}{body}{CLOSE}call{SEP}'
def _response(content: str) -> str:
return f"{OPEN}response{SEP}{content}{RESPONSE_CLOSE}"
def _tools(*calls: str) -> str:
return f"{OPEN}tools{SEP}{''.join(calls)}{CLOSE}tools{SEP}"
def test_extract_tool_calls_with_response_and_typed_arguments():
parser = KimiK3ToolParser(DummyTokenizer())
output = _response("answer") + _tools(
_call(
"calc",
1,
_arg("x", "number", "1"),
_arg("flag", "boolean", "true"),
_arg("text", "string", "raw"),
)
)
extracted = parser.extract_tool_calls(output, _request())
assert extracted.tools_called is True
assert extracted.content == "answer"
assert len(extracted.tool_calls) == 1
tool_call = extracted.tool_calls[0]
assert tool_call.id == "calc:0"
assert tool_call.function.name == "calc"
assert json.loads(tool_call.function.arguments) == {
"x": 1,
"flag": True,
"text": "raw",
}
def test_delegating_parser_preserves_tool_calls_after_reasoning():
parser = KimiK3DelegatingParser(DummyTokenizer())
output = (
f"{THINK_OPEN}step{THINK_CLOSE}"
+ _response("answer")
+ _tools(_call("calc", 1, _arg("x", "number", "1")))
)
reasoning, content, tool_calls = parser.parse(
output,
_request(),
enable_auto_tools=True,
)
assert reasoning == "step"
assert content == "answer"
assert tool_calls is not None
assert len(tool_calls) == 1
assert tool_calls[0].id == "calc:0"
assert tool_calls[0].name == "calc"
assert json.loads(tool_calls[0].arguments) == {"x": 1}
def test_delegating_parser_required_tool_choice_uses_xtml_parser():
parser = KimiK3DelegatingParser(DummyTokenizer())
request = _request().model_copy(update={"tool_choice": "required"})
output = (
f"{THINK_OPEN}step{THINK_CLOSE}"
+ _response("")
+ _tools(_call("calc", 1, _arg("x", "number", "1")))
)
reasoning, content, tool_calls = parser.parse(
output,
request,
enable_auto_tools=True,
)
assert reasoning == "step"
assert content is None
assert tool_calls is not None
assert len(tool_calls) == 1
assert tool_calls[0].name == "calc"
assert json.loads(tool_calls[0].arguments) == {"x": 1}
def test_delegating_parser_named_tool_choice_uses_xtml_parser():
parser = KimiK3DelegatingParser(DummyTokenizer())
output = (
f"{THINK_OPEN}step{THINK_CLOSE}"
+ _response("")
+ _tools(_call("calc", 1, _arg("x", "number", "1")))
)
reasoning, content, tool_calls = parser.parse(
output,
_named_request(),
enable_auto_tools=True,
)
assert reasoning == "step"
assert content is None
assert tool_calls is not None
assert len(tool_calls) == 1
assert tool_calls[0].name == "calc"
assert json.loads(tool_calls[0].arguments) == {"x": 1}
def test_delegating_parser_auto_no_call_strips_consumed_response_prefix():
parser = KimiK3DelegatingParser(
DummyTokenizer(), chat_template_kwargs={"thinking": False}
)
request = _request().model_copy(
update={"chat_template_kwargs": {"thinking": False}}
)
reasoning, content, tool_calls = parser.parse(
f"answer{RESPONSE_CLOSE}",
request,
enable_auto_tools=True,
)
assert reasoning is None
assert content == "answer"
assert tool_calls is None
def test_delegating_parser_required_call_strips_consumed_response_prefix():
parser = KimiK3DelegatingParser(
DummyTokenizer(), chat_template_kwargs={"thinking": False}
)
request = _request().model_copy(
update={
"tool_choice": "required",
"chat_template_kwargs": {"thinking": False},
}
)
output = RESPONSE_CLOSE + _tools(_call("calc", 1, _arg("x", "number", "1")))
reasoning, content, tool_calls = parser.parse(
output,
request,
enable_auto_tools=True,
)
assert reasoning is None
assert content is None
assert tool_calls is not None
assert len(tool_calls) == 1
assert tool_calls[0].name == "calc"
assert json.loads(tool_calls[0].arguments) == {"x": 1}
def test_delegating_parser_truncated_tools_do_not_leak_xtml():
parser = KimiK3DelegatingParser(
DummyTokenizer(), chat_template_kwargs={"thinking": False}
)
request = _request().model_copy(
update={
"tool_choice": "required",
"chat_template_kwargs": {"thinking": False},
}
)
reasoning, content, tool_calls = parser.parse(
(f'{RESPONSE_CLOSE}{OPEN}tools{SEP}{OPEN}call tool="calc" index="1"'),
request,
enable_auto_tools=True,
)
assert reasoning is None
assert content is None
assert tool_calls is None
def test_extract_tool_calls_unescapes_attributes():
parser = KimiK3ToolParser(DummyTokenizer())
output = _tools(_call("a&amp;b&quot;c", 1, _arg("k&amp;q", "string", "v")))
extracted = parser.extract_tool_calls(output, _request())
assert extracted.tools_called is True
assert extracted.tool_calls[0].function.name == 'a&b"c'
assert json.loads(extracted.tool_calls[0].function.arguments) == {"k&q": "v"}
def test_extract_tool_calls_allows_less_than_in_attributes():
parser = KimiK3ToolParser(DummyTokenizer())
output = _tools(_call("calc<beta", 1, _arg("foo<bar", "string", "raw")))
extracted = parser.extract_tool_calls(output, _request())
assert extracted.tools_called is True
assert extracted.tool_calls[0].function.name == "calc<beta"
assert json.loads(extracted.tool_calls[0].function.arguments) == {"foo<bar": "raw"}
def test_extract_content_from_whitespace_degraded_markers():
parser = KimiK3ToolParser(DummyTokenizer())
extracted = parser.extract_tool_calls(
f"{OPEN} response {SEP}answer{CLOSE} response {SEP}",
_request(),
)
assert extracted.tools_called is False
assert extracted.content == "answer"
def test_streaming_split_markers_do_not_leak():
parser = KimiK3ToolParser(DummyTokenizer())
request = _request()
previous_text = ""
previous_ids: list[int] = []
messages: list[DeltaMessage] = []
chunks = [
OPEN,
"response",
f"{SEP}Hi",
OPEN,
"tools",
SEP,
f'{OPEN}call tool="calc" index="1"{SEP}',
_arg("x", "number", "1"),
f"{CLOSE}call",
SEP,
]
for i, chunk in enumerate(chunks, start=1):
current_text = previous_text + chunk
current_ids = previous_ids + [i]
delta = parser.extract_tool_calls_streaming(
previous_text=previous_text,
current_text=current_text,
delta_text=chunk,
previous_token_ids=previous_ids,
current_token_ids=current_ids,
delta_token_ids=[i],
request=request,
)
if delta is not None:
messages.append(delta)
previous_text = current_text
previous_ids = current_ids
content = "".join(message.content or "" for message in messages)
tool_deltas = [
tool_call for message in messages for tool_call in (message.tool_calls or [])
]
assert content == "Hi"
assert OPEN not in content
assert SEP not in content
assert len(tool_deltas) == 1
assert tool_deltas[0].id == "calc:0"
assert tool_deltas[0].function.name == "calc"
assert json.loads(tool_deltas[0].function.arguments) == {"x": 1}
def test_streaming_consumed_response_prefix_no_call_keeps_content():
parser = KimiK3ToolParser(DummyTokenizer())
request = _request()
previous_text = ""
previous_ids: list[int] = []
messages: list[DeltaMessage] = []
chunks = ["O", "K", CLOSE, f"response{SEP}"]
for i, chunk in enumerate(chunks, start=1):
current_text = previous_text + chunk
current_ids = previous_ids + [i]
delta = parser.extract_tool_calls_streaming(
previous_text=previous_text,
current_text=current_text,
delta_text=chunk,
previous_token_ids=previous_ids,
current_token_ids=current_ids,
delta_token_ids=[i],
request=request,
)
if delta is not None:
messages.append(delta)
previous_text = current_text
previous_ids = current_ids
assert "".join(message.content or "" for message in messages) == "OK"
assert all(CLOSE not in (message.content or "") for message in messages)
def test_delegating_parser_tool_choice_none_strips_xtml_and_suppresses_calls():
parser = KimiK3DelegatingParser(
DummyTokenizer(), chat_template_kwargs={"thinking": False}
)
request = _request().model_copy(
update={
"tool_choice": "none",
"chat_template_kwargs": {"thinking": False},
}
)
messages: list[DeltaMessage] = []
chunks = [
OPEN,
"response",
f"{SEP}answer",
RESPONSE_CLOSE,
_tools(_call("calc", 1, _arg("x", "number", "1"))),
]
for index, chunk in enumerate(chunks, start=1):
delta = parser.parse_delta(
delta_text=chunk,
delta_token_ids=[index],
request=request,
prompt_token_ids=[1],
finished=index == len(chunks),
)
if delta is not None:
messages.append(delta)
content = "".join(message.content or "" for message in messages)
assert content == "answer"
assert OPEN not in content
assert CLOSE not in content
assert SEP not in content
assert all(not message.tool_calls for message in messages)
def test_adjust_request_keeps_xtml_markers_contiguous():
parser = KimiK3ToolParser(DummyTokenizer())
request = _request()
adjusted = parser.adjust_request(request)
assert adjusted.skip_special_tokens is False
if hasattr(adjusted, "spaces_between_special_tokens"):
assert adjusted.spaces_between_special_tokens is False
assert KimiK3ToolParser.supports_required_and_named is False
def test_adjust_request_required_uses_xtml_parser_not_json_guidance():
parser = KimiK3ToolParser(DummyTokenizer())
request = _request().model_copy(update={"tool_choice": "required"})
adjusted = parser.adjust_request(request)
assert adjusted.structured_outputs is None
assert adjusted.skip_special_tokens is False
if hasattr(adjusted, "spaces_between_special_tokens"):
assert adjusted.spaces_between_special_tokens is False
@pytest.mark.parametrize(
"tool_request",
[
_named_request(),
_responses_request(
tool_choice={"type": "function", "name": "calc"},
),
],
)
def test_adjust_request_rejects_named_tool_choice(tool_request):
parser = KimiK3ToolParser(DummyTokenizer())
with pytest.raises(VLLMValidationError) as exc_info:
parser.adjust_request(tool_request)
assert exc_info.value.parameter == "tool_choice"
assert "requires strict tool calling" in str(exc_info.value)
def test_responses_chat_params_carries_tool_choice_metadata():
request = _responses_request(tool_choice="required")
chat_params = request.build_chat_params(
default_template=None,
default_template_content_format="auto",
)
assert chat_params.tool_choice == "required"
def test_responses_chat_params_keeps_template_tool_choice_when_api_auto():
request = _responses_request().model_copy(
update={"chat_template_kwargs": {"tool_choice": "required"}}
)
chat_params = request.build_chat_params(
default_template=None,
default_template_content_format="auto",
)
assert chat_params.chat_template_kwargs["tool_choice"] == "required"
assert chat_params.tool_choice == "auto"
def test_responses_required_tool_choice_uses_xtml_parser():
parser = KimiK3DelegatingParser(
DummyTokenizer(), chat_template_kwargs={"thinking": False}
)
request = _responses_request(tool_choice="required").model_copy(
update={"chat_template_kwargs": {"thinking": False}}
)
output = RESPONSE_CLOSE + _tools(_call("calc", 1, _arg("x", "number", "1")))
reasoning, content, tool_calls = parser.parse(
output, request, enable_auto_tools=True, model_output_token_ids=[]
)
response_outputs = build_response_output_items(
reasoning=reasoning,
content=content,
tool_calls=tool_calls,
tools=request.tools,
)
assert len(response_outputs) == 1
tool_call = response_outputs[0]
assert tool_call.type == "function_call"
assert tool_call.name == "calc"
assert json.loads(tool_call.arguments) == {"x": 1}
def test_responses_named_tool_choice_uses_xtml_parser():
parser = KimiK3DelegatingParser(
DummyTokenizer(), chat_template_kwargs={"thinking": False}
)
request = _responses_request(
tool_choice={"type": "function", "name": "calc"}
).model_copy(update={"chat_template_kwargs": {"thinking": False}})
output = RESPONSE_CLOSE + _tools(_call("calc", 1, _arg("x", "number", "1")))
reasoning, content, tool_calls = parser.parse(
output, request, enable_auto_tools=True, model_output_token_ids=[]
)
response_outputs = build_response_output_items(
reasoning=reasoning,
content=content,
tool_calls=tool_calls,
tools=request.tools,
)
assert len(response_outputs) == 1
tool_call = response_outputs[0]
assert tool_call.type == "function_call"
assert tool_call.name == "calc"
assert json.loads(tool_call.arguments) == {"x": 1}
def test_chat_params_carries_tool_choice_metadata():
request = _request().model_copy(update={"tool_choice": "required"})
chat_params = request.build_chat_params(
default_template=None,
default_template_content_format="auto",
)
assert chat_params.tool_choice == "required"
def test_chat_params_carries_response_format_metadata():
request = ChatCompletionRequest(
model="test-model",
messages=[],
response_format={"type": "json_object"},
)
chat_params = request.build_chat_params(
default_template=None,
default_template_content_format="auto",
)
assert chat_params.response_format is request.response_format
assert chat_params.tool_choice is None
def test_chat_params_keeps_template_tool_choice_when_api_auto():
request = _request().model_copy(
update={
"tool_choice": "auto",
"chat_template_kwargs": {"tool_choice": "required"},
}
)
chat_params = request.build_chat_params(
default_template=None,
default_template_content_format="auto",
)
assert chat_params.chat_template_kwargs["tool_choice"] == "required"
assert chat_params.tool_choice == "auto"