[Model] Add Kimi K3 support: Python frontend [2/2] (#50093)
Signed-off-by: Bugen Zhao <i@bugenzhao.com>
This commit is contained in:
@@ -0,0 +1,250 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm.entrypoints.openai.chat_completion.protocol import (
|
||||
ChatCompletionRequest,
|
||||
)
|
||||
from vllm.entrypoints.openai.engine.protocol import DeltaMessage
|
||||
from vllm.parser.kimi_k3 import KimiK3Parser
|
||||
from vllm.parser.parser_manager import ParserManager
|
||||
from vllm.reasoning.kimi_k3_reasoning_parser import KimiK3ReasoningParser
|
||||
|
||||
pytestmark = pytest.mark.skip_global_cleanup
|
||||
|
||||
OPEN = "<|open|>"
|
||||
CLOSE = "<|close|>"
|
||||
SEP = "<|sep|>"
|
||||
THINK_OPEN = f"{OPEN}think{SEP}"
|
||||
THINK_CLOSE = f"{CLOSE}think{SEP}"
|
||||
RESPONSE_OPEN = f"{OPEN}response{SEP}"
|
||||
|
||||
|
||||
class DummyTokenizer:
|
||||
def get_vocab(self) -> dict[str, int]:
|
||||
return {}
|
||||
|
||||
def encode(self, text: str, add_special_tokens: bool = False) -> list[int]:
|
||||
if text == THINK_OPEN:
|
||||
return [1, 2, 3]
|
||||
if text == THINK_CLOSE:
|
||||
return [4, 2, 3]
|
||||
return [ord(ch) for ch in text]
|
||||
|
||||
|
||||
class ReasoningOnlyParser(KimiK3Parser):
|
||||
reasoning_parser_cls = KimiK3ReasoningParser
|
||||
|
||||
|
||||
def test_parser_manager_selects_kimi_k3_parser_for_reasoning_only():
|
||||
parser_cls = ParserManager.get_parser(reasoning_parser_name="kimi_k3")
|
||||
|
||||
assert parser_cls is not None
|
||||
assert issubclass(parser_cls, KimiK3Parser)
|
||||
assert parser_cls.reasoning_parser_cls is KimiK3ReasoningParser
|
||||
assert parser_cls.tool_parser_cls is None
|
||||
|
||||
|
||||
def test_parser_selection_thinking_disabled():
|
||||
parser = KimiK3ReasoningParser(
|
||||
DummyTokenizer(), chat_template_kwargs={"thinking": False}
|
||||
)
|
||||
|
||||
assert parser._thinking_enabled is False
|
||||
|
||||
|
||||
def test_extract_reasoning_with_xtml_tags():
|
||||
parser = KimiK3ReasoningParser(DummyTokenizer())
|
||||
request = ChatCompletionRequest(model="test-model", messages=[])
|
||||
|
||||
reasoning, content = parser.extract_reasoning_content(
|
||||
f"{THINK_OPEN}step{THINK_CLOSE}{RESPONSE_OPEN}answer",
|
||||
request,
|
||||
)
|
||||
|
||||
assert reasoning == "step"
|
||||
assert content == "answer"
|
||||
|
||||
|
||||
def test_extract_reasoning_with_generation_prefix_consumed():
|
||||
parser = KimiK3ReasoningParser(DummyTokenizer())
|
||||
request = ChatCompletionRequest(model="test-model", messages=[])
|
||||
|
||||
reasoning, content = parser.extract_reasoning_content(
|
||||
f"step{THINK_CLOSE}{RESPONSE_OPEN}answer",
|
||||
request,
|
||||
)
|
||||
|
||||
assert reasoning == "step"
|
||||
assert content == "answer"
|
||||
|
||||
|
||||
def test_delegating_parser_strips_response_wrapper_without_tool_parser():
|
||||
parser = ReasoningOnlyParser(DummyTokenizer())
|
||||
request = ChatCompletionRequest(model="test-model", messages=[])
|
||||
|
||||
reasoning, content, tool_calls = parser.parse(
|
||||
f"{THINK_OPEN}step{THINK_CLOSE}{RESPONSE_OPEN}answer",
|
||||
request,
|
||||
)
|
||||
|
||||
assert reasoning == "step"
|
||||
assert content == "answer"
|
||||
assert tool_calls == []
|
||||
|
||||
|
||||
def test_is_reasoning_end_uses_full_input_ids():
|
||||
parser = KimiK3ReasoningParser(DummyTokenizer())
|
||||
|
||||
assert not parser.is_reasoning_end([4, 2])
|
||||
assert parser.is_reasoning_end([4, 2, 3])
|
||||
|
||||
|
||||
def test_is_reasoning_end_ignores_stale_close_from_prior_turn():
|
||||
# DummyTokenizer: THINK_OPEN -> [1, 2, 3], THINK_CLOSE -> [4, 2, 3].
|
||||
# Multi-turn / agent continuation: a prior turn's think channel (its close
|
||||
# marker) is kept in the prompt, then the current turn opens a new think
|
||||
# block that has not closed yet. Reasoning must read as NOT ended, otherwise
|
||||
# the structured-output gate constrains the current turn's reasoning.
|
||||
parser = KimiK3ReasoningParser(DummyTokenizer())
|
||||
|
||||
stale_close = [4, 2, 3]
|
||||
new_open = [1, 2, 3]
|
||||
# prior close, then current-turn open still unclosed -> not ended
|
||||
assert not parser.is_reasoning_end([*stale_close, *new_open])
|
||||
# ...then the current turn emits its own close -> ended
|
||||
assert parser.is_reasoning_end([*stale_close, *new_open, *stale_close])
|
||||
# open with no close yet -> not ended
|
||||
assert not parser.is_reasoning_end([*new_open])
|
||||
|
||||
|
||||
def test_streaming_split_open_marker_is_held_back():
|
||||
parser = KimiK3ReasoningParser(DummyTokenizer())
|
||||
|
||||
first = parser.extract_reasoning_content_streaming(
|
||||
previous_text="",
|
||||
current_text=OPEN,
|
||||
delta_text=OPEN,
|
||||
previous_token_ids=[],
|
||||
current_token_ids=[1],
|
||||
delta_token_ids=[1],
|
||||
)
|
||||
second = parser.extract_reasoning_content_streaming(
|
||||
previous_text=OPEN,
|
||||
current_text=f"{OPEN}think",
|
||||
delta_text="think",
|
||||
previous_token_ids=[1],
|
||||
current_token_ids=[1, 2],
|
||||
delta_token_ids=[2],
|
||||
)
|
||||
third = parser.extract_reasoning_content_streaming(
|
||||
previous_text=f"{OPEN}think",
|
||||
current_text=THINK_OPEN + "step",
|
||||
delta_text=f"{SEP}step",
|
||||
previous_token_ids=[1, 2],
|
||||
current_token_ids=[1, 2, 3, 9],
|
||||
delta_token_ids=[3, 9],
|
||||
)
|
||||
|
||||
assert first is None
|
||||
assert second is None
|
||||
assert isinstance(third, DeltaMessage)
|
||||
assert third.reasoning == "step"
|
||||
|
||||
|
||||
def test_streaming_split_close_marker_hands_content_downstream():
|
||||
parser = KimiK3ReasoningParser(DummyTokenizer())
|
||||
|
||||
previous_text = f"{THINK_OPEN}step"
|
||||
partial_close = parser.extract_reasoning_content_streaming(
|
||||
previous_text=previous_text,
|
||||
current_text=previous_text + CLOSE,
|
||||
delta_text=CLOSE,
|
||||
previous_token_ids=[1, 2, 3, 9],
|
||||
current_token_ids=[1, 2, 3, 9, 4],
|
||||
delta_token_ids=[4],
|
||||
)
|
||||
closed = parser.extract_reasoning_content_streaming(
|
||||
previous_text=previous_text + CLOSE,
|
||||
current_text=previous_text + f"{THINK_CLOSE}{RESPONSE_OPEN}answer",
|
||||
delta_text=f"think{SEP}{RESPONSE_OPEN}answer",
|
||||
previous_token_ids=[1, 2, 3, 9, 4],
|
||||
current_token_ids=[1, 2, 3, 9, 4, 2, 3, 10],
|
||||
delta_token_ids=[2, 3, 10],
|
||||
)
|
||||
|
||||
assert partial_close is None
|
||||
assert isinstance(closed, DeltaMessage)
|
||||
assert closed.reasoning is None
|
||||
assert closed.content == f"{RESPONSE_OPEN}answer"
|
||||
assert parser.extract_content_ids([2, 3, 10]) == [10]
|
||||
|
||||
|
||||
def test_thinking_disabled_streams_content():
|
||||
parser = KimiK3ReasoningParser(
|
||||
DummyTokenizer(), chat_template_kwargs={"enable_thinking": False}
|
||||
)
|
||||
|
||||
delta = parser.extract_reasoning_content_streaming(
|
||||
previous_text="",
|
||||
current_text=f"{RESPONSE_OPEN}answer",
|
||||
delta_text=f"{RESPONSE_OPEN}answer",
|
||||
previous_token_ids=[],
|
||||
current_token_ids=[1],
|
||||
delta_token_ids=[1],
|
||||
)
|
||||
|
||||
assert isinstance(delta, DeltaMessage)
|
||||
assert delta.content == f"{RESPONSE_OPEN}answer"
|
||||
assert delta.reasoning is None
|
||||
|
||||
|
||||
def test_delegating_parser_thinking_false_streams_response_content():
|
||||
parser = ReasoningOnlyParser(
|
||||
DummyTokenizer(), chat_template_kwargs={"thinking": False}
|
||||
)
|
||||
request = ChatCompletionRequest(
|
||||
model="test-model",
|
||||
messages=[],
|
||||
chat_template_kwargs={"thinking": False},
|
||||
)
|
||||
|
||||
first = parser.parse_delta(
|
||||
delta_text="OK",
|
||||
delta_token_ids=[10],
|
||||
request=request,
|
||||
prompt_token_ids=[1],
|
||||
finished=False,
|
||||
)
|
||||
partial_close = parser.parse_delta(
|
||||
delta_text=CLOSE,
|
||||
delta_token_ids=[2],
|
||||
request=request,
|
||||
prompt_token_ids=[1],
|
||||
finished=False,
|
||||
)
|
||||
closed = parser.parse_delta(
|
||||
delta_text=f"response{SEP}",
|
||||
delta_token_ids=[3, 4],
|
||||
request=request,
|
||||
prompt_token_ids=[1],
|
||||
finished=False,
|
||||
)
|
||||
|
||||
assert first is not None
|
||||
assert first.content == "OK"
|
||||
assert first.reasoning is None
|
||||
assert partial_close is None
|
||||
assert closed is None
|
||||
|
||||
|
||||
def test_adjust_request_keeps_xtml_markers_contiguous():
|
||||
parser = KimiK3ReasoningParser(DummyTokenizer())
|
||||
request = ChatCompletionRequest(model="test-model", messages=[])
|
||||
|
||||
adjusted = parser.adjust_request(request)
|
||||
|
||||
assert adjusted.skip_special_tokens is False
|
||||
if hasattr(adjusted, "spaces_between_special_tokens"):
|
||||
assert adjusted.spaces_between_special_tokens is False
|
||||
@@ -0,0 +1,353 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm.exceptions import VLLMValidationError
|
||||
from vllm.renderers import ChatParams
|
||||
from vllm.renderers.kimi_k3 import KimiK3Renderer, _merge_k3_media_io_kwargs
|
||||
from vllm.renderers.registry import RENDERER_REGISTRY
|
||||
from vllm.tokenizers.registry import TokenizerRegistry
|
||||
|
||||
|
||||
class StubTokenizer:
|
||||
"""Stands in for the model's TikTokenTokenizer.
|
||||
|
||||
Records the kwargs it is called with and returns fixed token ids, so tests
|
||||
can assert how the renderer drives ``apply_chat_template`` without the real
|
||||
(git-LFS) tiktoken vocabulary.
|
||||
"""
|
||||
|
||||
def __init__(self, token_ids: list[int]) -> None:
|
||||
self.token_ids = token_ids
|
||||
self.calls: list[dict[str, Any]] = []
|
||||
self.conversations: list[list[dict[str, Any]]] = []
|
||||
|
||||
def apply_chat_template(self, conversation, **kwargs) -> list[int]:
|
||||
self.conversations.append(conversation)
|
||||
self.calls.append(kwargs)
|
||||
return list(self.token_ids)
|
||||
|
||||
|
||||
@dataclass
|
||||
class MockHFConfig:
|
||||
model_type: str = "kimi_k3"
|
||||
|
||||
|
||||
@dataclass
|
||||
class MockModelConfig:
|
||||
runner_type: str = "generate"
|
||||
is_multimodal_model: bool = False
|
||||
multimodal_config: Any = None
|
||||
hf_config: MockHFConfig = field(default_factory=MockHFConfig)
|
||||
allowed_local_media_path: str = ""
|
||||
allowed_media_domains: Any = None
|
||||
enable_prompt_embeds: bool = False
|
||||
renderer_num_workers: int = 1
|
||||
|
||||
|
||||
@dataclass
|
||||
class MockParallelConfig:
|
||||
_api_process_rank: int = 0
|
||||
|
||||
|
||||
@dataclass
|
||||
class MockVllmConfig:
|
||||
model_config: MockModelConfig
|
||||
parallel_config: MockParallelConfig
|
||||
|
||||
|
||||
def _make_renderer(tokenizer: StubTokenizer) -> KimiK3Renderer:
|
||||
config = MockVllmConfig(MockModelConfig(), MockParallelConfig())
|
||||
return KimiK3Renderer(config, tokenizer)
|
||||
|
||||
|
||||
def test_kimi_k3_registered():
|
||||
assert RENDERER_REGISTRY.load_renderer_cls("kimi_k3").__name__ == "KimiK3Renderer"
|
||||
assert (
|
||||
TokenizerRegistry.load_tokenizer_cls("kimi_k3").__name__ == "CachedHfTokenizer"
|
||||
)
|
||||
|
||||
|
||||
def test_k3_media_io_defaults_preserve_original_mode():
|
||||
# Default: K3 keeps the original image mode (no background flattening).
|
||||
assert _merge_k3_media_io_kwargs(None) == {"image": {"image_mode": None}}
|
||||
|
||||
# Server-/request-level values take precedence over the K3 default.
|
||||
assert _merge_k3_media_io_kwargs({"image": {"image_mode": "RGB"}}) == {
|
||||
"image": {"image_mode": "RGB"}
|
||||
}
|
||||
|
||||
# Unrelated image kwargs are merged with the default.
|
||||
assert _merge_k3_media_io_kwargs(
|
||||
{"image": {"rgba_background_color": (0, 0, 0)}}
|
||||
) == {"image": {"image_mode": None, "rgba_background_color": (0, 0, 0)}}
|
||||
|
||||
|
||||
def test_apply_chat_template_forces_tokenize_and_pins_return_dict():
|
||||
tokenizer = StubTokenizer([7, 8, 9])
|
||||
renderer = _make_renderer(tokenizer)
|
||||
tools = [{"type": "function", "function": {"name": "search"}}]
|
||||
params = ChatParams(
|
||||
chat_template_kwargs={"tools": tools, "tokenize": False, "thinking": True}
|
||||
)
|
||||
|
||||
token_ids = renderer._apply_chat_template(
|
||||
[{"role": "user", "content": "hi"}], params
|
||||
)
|
||||
|
||||
assert token_ids == [7, 8, 9]
|
||||
kwargs = tokenizer.calls[-1]
|
||||
# tokenize is forced on even though the request asked for False, so K3 keeps
|
||||
# the special-vs-ordinary token distinction instead of re-tokenizing a string.
|
||||
assert kwargs["tokenize"] is True
|
||||
# return_dict is pinned False so we always get a flat list of ids.
|
||||
assert kwargs["return_dict"] is False
|
||||
assert kwargs["tools"] == tools
|
||||
assert kwargs["thinking"] is True
|
||||
|
||||
|
||||
def test_apply_chat_template_translates_standard_thinking_kwargs():
|
||||
# Standard enable_thinking/reasoning_effort kwargs must be translated
|
||||
# to K3's native thinking/thinking_effort.
|
||||
tokenizer = StubTokenizer([7, 8, 9])
|
||||
renderer = _make_renderer(tokenizer)
|
||||
params = ChatParams(
|
||||
chat_template_kwargs={"enable_thinking": False, "reasoning_effort": "none"}
|
||||
)
|
||||
|
||||
renderer._apply_chat_template([{"role": "user", "content": "hi"}], params)
|
||||
|
||||
kwargs = tokenizer.calls[-1]
|
||||
assert kwargs["thinking"] is False
|
||||
assert "thinking_effort" not in kwargs
|
||||
assert "enable_thinking" not in kwargs
|
||||
assert "reasoning_effort" not in kwargs
|
||||
|
||||
|
||||
@pytest.mark.parametrize("reasoning_effort", ["low", "high", "max"])
|
||||
def test_apply_chat_template_translates_supported_reasoning_effort(
|
||||
reasoning_effort: str,
|
||||
):
|
||||
tokenizer = StubTokenizer([7, 8, 9])
|
||||
renderer = _make_renderer(tokenizer)
|
||||
params = ChatParams(chat_template_kwargs={"reasoning_effort": reasoning_effort})
|
||||
|
||||
renderer._apply_chat_template([{"role": "user", "content": "hi"}], params)
|
||||
|
||||
kwargs = tokenizer.calls[-1]
|
||||
assert kwargs["thinking_effort"] == reasoning_effort
|
||||
assert "reasoning_effort" not in kwargs
|
||||
|
||||
|
||||
@pytest.mark.parametrize("reasoning_effort", ["minimal", "medium", "xhigh"])
|
||||
def test_apply_chat_template_rejects_unsupported_reasoning_effort(
|
||||
reasoning_effort: str,
|
||||
):
|
||||
tokenizer = StubTokenizer([7, 8, 9])
|
||||
renderer = _make_renderer(tokenizer)
|
||||
params = ChatParams(chat_template_kwargs={"reasoning_effort": reasoning_effort})
|
||||
|
||||
with pytest.raises(VLLMValidationError, match="thinking_effort") as exc_info:
|
||||
renderer._apply_chat_template([{"role": "user", "content": "hi"}], params)
|
||||
|
||||
assert exc_info.value.parameter == "thinking_effort"
|
||||
assert exc_info.value.value == reasoning_effort
|
||||
assert tokenizer.calls == []
|
||||
|
||||
|
||||
def test_apply_chat_template_validates_canonical_native_thinking_effort():
|
||||
tokenizer = StubTokenizer([7, 8, 9])
|
||||
renderer = _make_renderer(tokenizer)
|
||||
params = ChatParams(
|
||||
chat_template_kwargs={
|
||||
"thinking_effort": "low",
|
||||
"reasoning_effort": "medium",
|
||||
}
|
||||
)
|
||||
|
||||
renderer._apply_chat_template([{"role": "user", "content": "hi"}], params)
|
||||
|
||||
assert tokenizer.calls[-1]["thinking_effort"] == "low"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("thinking_effort", ["none", "minimal", "medium", "xhigh"])
|
||||
def test_apply_chat_template_rejects_unsupported_native_thinking_effort(
|
||||
thinking_effort: str,
|
||||
):
|
||||
tokenizer = StubTokenizer([7, 8, 9])
|
||||
renderer = _make_renderer(tokenizer)
|
||||
params = ChatParams(chat_template_kwargs={"thinking_effort": thinking_effort})
|
||||
|
||||
with pytest.raises(VLLMValidationError, match="thinking_effort") as exc_info:
|
||||
renderer._apply_chat_template([{"role": "user", "content": "hi"}], params)
|
||||
|
||||
assert exc_info.value.parameter == "thinking_effort"
|
||||
assert exc_info.value.value == thinking_effort
|
||||
assert tokenizer.calls == []
|
||||
|
||||
|
||||
def test_apply_chat_template_native_k3_kwargs_take_precedence():
|
||||
tokenizer = StubTokenizer([7, 8, 9])
|
||||
renderer = _make_renderer(tokenizer)
|
||||
params = ChatParams(
|
||||
chat_template_kwargs={
|
||||
"thinking": True,
|
||||
"enable_thinking": False,
|
||||
"thinking_effort": "low",
|
||||
"reasoning_effort": "high",
|
||||
}
|
||||
)
|
||||
|
||||
renderer._apply_chat_template([{"role": "user", "content": "hi"}], params)
|
||||
|
||||
kwargs = tokenizer.calls[-1]
|
||||
assert kwargs["thinking"] is True
|
||||
assert kwargs["thinking_effort"] == "low"
|
||||
|
||||
|
||||
def test_apply_chat_template_adds_k3_api_metadata():
|
||||
tokenizer = StubTokenizer([7, 8, 9])
|
||||
renderer = _make_renderer(tokenizer)
|
||||
response_format = {"type": "json_object"}
|
||||
params = ChatParams(
|
||||
tool_choice="required",
|
||||
response_format=response_format,
|
||||
)
|
||||
|
||||
renderer._apply_chat_template([{"role": "user", "content": "hi"}], params)
|
||||
|
||||
kwargs = tokenizer.calls[-1]
|
||||
assert kwargs["tool_choice"] == "required"
|
||||
assert kwargs["response_format"] == response_format
|
||||
|
||||
|
||||
def test_apply_chat_template_auto_tool_choice_keeps_template_kwarg():
|
||||
tokenizer = StubTokenizer([7, 8, 9])
|
||||
renderer = _make_renderer(tokenizer)
|
||||
params = ChatParams(
|
||||
chat_template_kwargs={"tool_choice": "required"},
|
||||
tool_choice="auto",
|
||||
)
|
||||
|
||||
renderer._apply_chat_template([{"role": "user", "content": "hi"}], params)
|
||||
|
||||
assert tokenizer.calls[-1]["tool_choice"] == "required"
|
||||
|
||||
|
||||
def test_apply_chat_template_omits_tool_choice_without_tools():
|
||||
tokenizer = StubTokenizer([7, 8, 9])
|
||||
renderer = _make_renderer(tokenizer)
|
||||
|
||||
renderer._apply_chat_template(
|
||||
[{"role": "user", "content": "hi"}], ChatParams(tool_choice=None)
|
||||
)
|
||||
|
||||
assert "tool_choice" not in tokenizer.calls[-1]
|
||||
|
||||
|
||||
def test_render_messages_returns_token_prompt():
|
||||
renderer = _make_renderer(StubTokenizer([1, 2, 3]))
|
||||
|
||||
conversation, prompt = renderer.render_messages(
|
||||
[{"role": "user", "content": "hi"}], ChatParams()
|
||||
)
|
||||
|
||||
assert prompt == {"prompt_token_ids": [1, 2, 3]}
|
||||
assert "multi_modal_data" not in prompt
|
||||
assert conversation[0]["role"] == "user"
|
||||
|
||||
|
||||
def test_render_messages_derives_private_xtml_tool_attrs():
|
||||
tokenizer = StubTokenizer([1, 2, 3])
|
||||
renderer = _make_renderer(tokenizer)
|
||||
|
||||
conversation, _ = renderer.render_messages(
|
||||
[
|
||||
{
|
||||
"role": "assistant",
|
||||
"content": "",
|
||||
"tool_calls": [
|
||||
{
|
||||
"id": "lookup:0",
|
||||
"type": "function",
|
||||
"function": {"name": "lookup", "arguments": "{}"},
|
||||
},
|
||||
{
|
||||
"id": "lookup:1",
|
||||
"type": "function",
|
||||
"function": {"name": "lookup", "arguments": "{}"},
|
||||
},
|
||||
],
|
||||
},
|
||||
{
|
||||
"role": "tool",
|
||||
"tool_call_id": "lookup:1",
|
||||
"tool": "client-supplied-name",
|
||||
"index": 99,
|
||||
"content": "second",
|
||||
},
|
||||
{
|
||||
"role": "tool",
|
||||
"tool_call_id": "lookup:0",
|
||||
"content": "first",
|
||||
},
|
||||
],
|
||||
ChatParams(),
|
||||
)
|
||||
|
||||
assert [message["content"] for message in conversation[1:]] == [
|
||||
"first",
|
||||
"second",
|
||||
]
|
||||
assert conversation[1]["tool"] == "lookup"
|
||||
assert conversation[1]["index"] == 1
|
||||
assert conversation[2]["tool"] == "lookup"
|
||||
assert conversation[2]["index"] == 2
|
||||
assert tokenizer.conversations[-1] == conversation
|
||||
|
||||
|
||||
def test_render_messages_ignores_client_supplied_xtml_tool_attrs():
|
||||
tokenizer = StubTokenizer([1, 2, 3])
|
||||
renderer = _make_renderer(tokenizer)
|
||||
|
||||
conversation, _ = renderer.render_messages(
|
||||
[
|
||||
{
|
||||
"role": "assistant",
|
||||
"content": "",
|
||||
"tool_calls": [
|
||||
{
|
||||
"id": "lookup:0",
|
||||
"type": "function",
|
||||
"function": {"name": "lookup", "arguments": "{}"},
|
||||
}
|
||||
],
|
||||
},
|
||||
{
|
||||
"role": "tool",
|
||||
"tool_call_id": "unknown",
|
||||
"tool": "lookup",
|
||||
"index": 1,
|
||||
"content": "result",
|
||||
},
|
||||
],
|
||||
ChatParams(),
|
||||
)
|
||||
|
||||
assert "tool" not in conversation[1]
|
||||
assert "index" not in conversation[1]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_render_messages_async_returns_token_prompt():
|
||||
renderer = _make_renderer(StubTokenizer([4, 5]))
|
||||
|
||||
conversation, prompt = await renderer.render_messages_async(
|
||||
[{"role": "user", "content": "hi"}], ChatParams()
|
||||
)
|
||||
|
||||
assert prompt == {"prompt_token_ids": [4, 5]}
|
||||
assert conversation[0]["role"] == "user"
|
||||
@@ -0,0 +1,62 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Named tool choice for Kimi K3: allowed when the XTML structural tag is
|
||||
attached (strict tool calling), rejected otherwise."""
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm.entrypoints.openai.chat_completion.protocol import (
|
||||
ChatCompletionRequest,
|
||||
)
|
||||
from vllm.exceptions import VLLMValidationError
|
||||
from vllm.sampling_params import StructuredOutputsParams
|
||||
|
||||
|
||||
class _DummyTokenizer:
|
||||
def get_vocab(self):
|
||||
return {}
|
||||
|
||||
def encode(self, text, add_special_tokens=False):
|
||||
return [ord(c) for c in text]
|
||||
|
||||
|
||||
def _request(with_tag: bool) -> ChatCompletionRequest:
|
||||
req = ChatCompletionRequest(
|
||||
model="k3",
|
||||
messages=[{"role": "user", "content": "hi"}],
|
||||
tools=[
|
||||
{
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "get_weather",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {"city": {"type": "string"}},
|
||||
"required": ["city"],
|
||||
},
|
||||
},
|
||||
}
|
||||
],
|
||||
tool_choice={"type": "function", "function": {"name": "get_weather"}},
|
||||
)
|
||||
if with_tag:
|
||||
req.structured_outputs = StructuredOutputsParams(
|
||||
structural_tag='{"type": "structural_tag", "format": {}}'
|
||||
)
|
||||
return req
|
||||
|
||||
|
||||
def _parser():
|
||||
from vllm.tool_parsers.kimi_k3_tool_parser import KimiK3ToolParser
|
||||
|
||||
return KimiK3ToolParser(_DummyTokenizer())
|
||||
|
||||
|
||||
def test_named_choice_allowed_with_structural_tag():
|
||||
req = _parser().adjust_request(_request(with_tag=True))
|
||||
assert req.skip_special_tokens is False
|
||||
|
||||
|
||||
def test_named_choice_rejected_without_structural_tag():
|
||||
with pytest.raises(VLLMValidationError):
|
||||
_parser().adjust_request(_request(with_tag=False))
|
||||
@@ -5,7 +5,8 @@ from types import SimpleNamespace
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import pytest
|
||||
from xgrammar import StructuralTag
|
||||
from xgrammar import Grammar, StructuralTag
|
||||
from xgrammar.testing import _is_grammar_accept_string
|
||||
|
||||
from vllm.entrypoints.openai.chat_completion.protocol import (
|
||||
ChatCompletionNamedFunction,
|
||||
@@ -24,6 +25,7 @@ from vllm.tool_parsers.deepseekv32_engine_tool_parser import (
|
||||
from vllm.tool_parsers.glm47_moe_tool_parser import Glm47MoeModelToolParser
|
||||
from vllm.tool_parsers.hermes_tool_parser import Hermes2ProToolParser
|
||||
from vllm.tool_parsers.kimi_k2_tool_parser import KimiK2ToolParser
|
||||
from vllm.tool_parsers.kimi_k3_tool_parser import KimiK3ToolParser
|
||||
from vllm.tool_parsers.llama_tool_parser import Llama3JsonToolParser
|
||||
from vllm.tool_parsers.minimax_m2_tool_parser import MinimaxM2ToolParser
|
||||
from vllm.tool_parsers.qwen3_engine_tool_parser import Qwen3EngineToolParser
|
||||
@@ -165,6 +167,196 @@ def test_hermes_required_tool_calls_use_empty_separator():
|
||||
assert tag.format.separator == ""
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Kimi K3 (XTML channel format) structural tag
|
||||
# ---------------------------------------------------------------------------
|
||||
_K3_RESPONSE_OPEN = "<|open|>response<|sep|>"
|
||||
_K3_RESPONSE_CLOSE = "<|close|>response<|sep|>"
|
||||
_K3_TOOLS_OPEN = "<|open|>tools<|sep|>"
|
||||
_K3_TOOLS_CLOSE = "<|close|>tools<|sep|>"
|
||||
_K3_CALL_CLOSE = "<|close|>call<|sep|>"
|
||||
_K3_ARG_CLOSE = "<|close|>argument<|sep|>"
|
||||
_K3_MESSAGE_CLOSE = "<|close|>message<|sep|>"
|
||||
|
||||
|
||||
def _k3_tools_by_name() -> list[ChatCompletionToolsParam]:
|
||||
return [
|
||||
ChatCompletionToolsParam(
|
||||
type="function",
|
||||
function={
|
||||
"name": "get_weather",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"city": {"type": "string"},
|
||||
"days": {"type": "integer"},
|
||||
},
|
||||
"required": ["city"],
|
||||
},
|
||||
},
|
||||
),
|
||||
ChatCompletionToolsParam(
|
||||
type="function",
|
||||
function={
|
||||
"name": "run_command",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {"command": {"type": "string"}},
|
||||
"required": ["command"],
|
||||
},
|
||||
},
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
def _k3_arg(key: str, typ: str, val: str) -> str:
|
||||
return f'<|open|>argument key="{key}" type="{typ}"<|sep|>{val}{_K3_ARG_CLOSE}'
|
||||
|
||||
|
||||
def _k3_call(name: str, args: str, idx: int = 1) -> str:
|
||||
return f'<|open|>call tool="{name}" index="{idx}"<|sep|>{args}{_K3_CALL_CLOSE}'
|
||||
|
||||
|
||||
def _k3_response(content: str = "") -> str:
|
||||
return f"{_K3_RESPONSE_OPEN}{content}{_K3_RESPONSE_CLOSE}"
|
||||
|
||||
|
||||
def _k3_tools(*calls: str) -> str:
|
||||
return f"{_K3_TOOLS_OPEN}{''.join(calls)}{_K3_TOOLS_CLOSE}"
|
||||
|
||||
|
||||
def _k3_grammar(tool_choice, tools=None):
|
||||
tag = get_model_structural_tag(
|
||||
model="kimi_k3",
|
||||
tools=tools if tools is not None else _k3_tools_by_name(),
|
||||
tool_choice=tool_choice,
|
||||
reasoning=False,
|
||||
)
|
||||
assert isinstance(tag, StructuralTag)
|
||||
return Grammar.from_structural_tag(tag)
|
||||
|
||||
|
||||
def test_kimi_k3_registered_as_vllm_builtin():
|
||||
assert "kimi_k3" in VLLM_BUILTIN_STRUCTURAL_TAG_MODELS
|
||||
assert KimiK3ToolParser.structural_tag_model == "kimi_k3"
|
||||
|
||||
|
||||
def test_kimi_k3_auto_without_strict_is_unconstrained():
|
||||
# auto + no strict tool => no structural tag (matches the strict gate).
|
||||
tag = get_model_structural_tag(
|
||||
model="kimi_k3",
|
||||
tools=_k3_tools_by_name(),
|
||||
tool_choice="auto",
|
||||
reasoning=False,
|
||||
)
|
||||
assert tag is None
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"body",
|
||||
[
|
||||
# single required arg
|
||||
_k3_response()
|
||||
+ _k3_tools(_k3_call("get_weather", _k3_arg("city", "string", "Paris"))),
|
||||
# response content + two args (string + number)
|
||||
_k3_response("Checking.")
|
||||
+ _k3_tools(
|
||||
_k3_call(
|
||||
"get_weather",
|
||||
_k3_arg("city", "string", "Paris") + _k3_arg("days", "number", "3"),
|
||||
)
|
||||
),
|
||||
# args in reverse order (parser is order-agnostic)
|
||||
_k3_response()
|
||||
+ _k3_tools(
|
||||
_k3_call(
|
||||
"get_weather",
|
||||
_k3_arg("days", "number", "3") + _k3_arg("city", "string", "Paris"),
|
||||
)
|
||||
),
|
||||
# two calls, second tool
|
||||
_k3_response()
|
||||
+ _k3_tools(
|
||||
_k3_call("get_weather", _k3_arg("city", "string", "Paris"), 1),
|
||||
_k3_call("run_command", _k3_arg("command", "string", "ls -la"), 2),
|
||||
),
|
||||
# string value with regex metacharacters / spaces
|
||||
_k3_response()
|
||||
+ _k3_tools(
|
||||
_k3_call(
|
||||
"run_command", _k3_arg("command", "string", "grep -E 'a|b{2,}' x.py")
|
||||
)
|
||||
),
|
||||
# trailing message-close marker (model's natural turn terminator)
|
||||
_k3_response()
|
||||
+ _k3_tools(_k3_call("get_weather", _k3_arg("city", "string", "Paris")))
|
||||
+ _K3_MESSAGE_CLOSE,
|
||||
# non-thinking mode: response-open is the prompt prefix, so it is absent
|
||||
_K3_RESPONSE_CLOSE
|
||||
+ _k3_tools(_k3_call("get_weather", _k3_arg("city", "string", "Paris"))),
|
||||
],
|
||||
)
|
||||
def test_kimi_k3_required_accepts_valid_tool_calls(body: str):
|
||||
assert _is_grammar_accept_string(_k3_grammar("required"), body)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"body",
|
||||
[
|
||||
# unknown tool name
|
||||
_k3_response()
|
||||
+ _k3_tools(_k3_call("get_temperature", _k3_arg("city", "string", "x"))),
|
||||
# number arg given a non-numeric JSON value
|
||||
_k3_response()
|
||||
+ _k3_tools(_k3_call("get_weather", _k3_arg("days", "number", "abc"))),
|
||||
# undeclared argument key
|
||||
_k3_response()
|
||||
+ _k3_tools(
|
||||
_k3_call(
|
||||
"get_weather",
|
||||
_k3_arg("city", "string", "Paris") + _k3_arg("zzz", "string", "x"),
|
||||
)
|
||||
),
|
||||
# required schema but no argument tags
|
||||
_k3_response() + _k3_tools(_k3_call("get_weather", "")),
|
||||
# missing tools close marker
|
||||
_k3_response()
|
||||
+ _K3_TOOLS_OPEN
|
||||
+ _k3_call("get_weather", _k3_arg("city", "string", "Paris")),
|
||||
# required but no tool call
|
||||
_k3_response("hello"),
|
||||
],
|
||||
)
|
||||
def test_kimi_k3_required_rejects_invalid(body: str):
|
||||
assert not _is_grammar_accept_string(_k3_grammar("required"), body)
|
||||
|
||||
|
||||
def test_kimi_k3_schema_without_required_accepts_empty_call():
|
||||
tools = [
|
||||
ChatCompletionToolsParam(
|
||||
type="function",
|
||||
function={
|
||||
"name": "get_weather",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {"city": {"type": "string"}},
|
||||
},
|
||||
},
|
||||
)
|
||||
]
|
||||
grammar = _k3_grammar("required", tools=tools)
|
||||
body = _k3_response() + _k3_tools(_k3_call("get_weather", ""))
|
||||
|
||||
assert _is_grammar_accept_string(grammar, body)
|
||||
|
||||
|
||||
def test_kimi_k3_auto_strict_allows_response_only(sample_tools_strict):
|
||||
# With a strict tool the tag is built; the tools channel is optional so a
|
||||
# plain response (no tool call) is still valid.
|
||||
grammar = _k3_grammar("auto", tools=sample_tools_strict)
|
||||
assert _is_grammar_accept_string(grammar, _k3_response("Just answering."))
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model", sorted(XGRAMMAR_BUILTIN_STRUCTURAL_TAG_MODELS))
|
||||
def test_get_model_structural_tag_supports_named_tool_choice(
|
||||
model: str,
|
||||
@@ -339,3 +531,151 @@ def test_get_function_parameters_relaxes_function_strict_false():
|
||||
)
|
||||
|
||||
assert _get_function_parameters(function) is True
|
||||
|
||||
|
||||
def _k3_tools_with_root_defs() -> list[ChatCompletionToolsParam]:
|
||||
return [
|
||||
ChatCompletionToolsParam(
|
||||
type="function",
|
||||
function={
|
||||
"name": "make_config",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"config": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"build": {"$ref": "#/$defs/build"},
|
||||
"index": {"type": "string"},
|
||||
},
|
||||
"required": ["index"],
|
||||
"additionalProperties": False,
|
||||
},
|
||||
},
|
||||
"required": ["config"],
|
||||
"$defs": {
|
||||
"build": {
|
||||
"type": "object",
|
||||
"properties": {"outDir": {"type": "string"}},
|
||||
"additionalProperties": False,
|
||||
}
|
||||
},
|
||||
},
|
||||
},
|
||||
)
|
||||
]
|
||||
|
||||
|
||||
def test_kimi_k3_property_ref_to_root_defs_compiles_and_accepts():
|
||||
# Root-level $defs referenced from inside a property schema (the walle
|
||||
# TestReferences shape). Slicing the property out of the parameters
|
||||
# document orphans "#/$defs/..." unless the builder re-attaches $defs;
|
||||
# before the fix Grammar.from_structural_tag raised on the dangling ref.
|
||||
grammar = _k3_grammar("required", tools=_k3_tools_with_root_defs())
|
||||
|
||||
body = _k3_response() + _k3_tools(
|
||||
_k3_call(
|
||||
"make_config",
|
||||
_k3_arg(
|
||||
"config",
|
||||
"object",
|
||||
'{"build": {"outDir": "dist"}, "index": "a.html"}',
|
||||
),
|
||||
)
|
||||
)
|
||||
assert _is_grammar_accept_string(grammar, body)
|
||||
|
||||
|
||||
def _k3_tools_with_string_enum() -> list[ChatCompletionToolsParam]:
|
||||
return [
|
||||
ChatCompletionToolsParam(
|
||||
type="function",
|
||||
function={
|
||||
"name": "set_unit",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"unit": {
|
||||
"type": "string",
|
||||
"enum": ["celsius", " fahrenheit", "\tkelvin"],
|
||||
},
|
||||
},
|
||||
"required": ["unit"],
|
||||
},
|
||||
},
|
||||
)
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("value", ["celsius", " fahrenheit", "\tkelvin"])
|
||||
def test_kimi_k3_string_enum_accepts_exact_values(value: str):
|
||||
# Raw string channel with enum: constrained to the exact enum values,
|
||||
# including leading-whitespace variants the model otherwise flubs.
|
||||
grammar = _k3_grammar("required", tools=_k3_tools_with_string_enum())
|
||||
body = _k3_response() + _k3_tools(
|
||||
_k3_call("set_unit", _k3_arg("unit", "string", value))
|
||||
)
|
||||
assert _is_grammar_accept_string(grammar, body)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("value", ["kelvin", "Celsius", "celsius ", ""])
|
||||
def test_kimi_k3_string_enum_rejects_non_members(value: str):
|
||||
grammar = _k3_grammar("required", tools=_k3_tools_with_string_enum())
|
||||
body = _k3_response() + _k3_tools(
|
||||
_k3_call("set_unit", _k3_arg("unit", "string", value))
|
||||
)
|
||||
assert not _is_grammar_accept_string(grammar, body)
|
||||
|
||||
|
||||
def _k3_tools_with_maxlen() -> list[ChatCompletionToolsParam]:
|
||||
return [
|
||||
ChatCompletionToolsParam(
|
||||
type="function",
|
||||
function={
|
||||
"name": "set_note",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"note": {"type": "string", "maxLength": 8, "minLength": 2},
|
||||
},
|
||||
"required": ["note"],
|
||||
},
|
||||
},
|
||||
)
|
||||
]
|
||||
|
||||
|
||||
def test_kimi_k3_string_maxlength_bounds_raw_channel():
|
||||
# Raw string channel with maxLength/minLength: enforced via a bounded
|
||||
# regex that keeps the "<|" marker prefix unambiguous but still allows
|
||||
# a bare '<' inside values.
|
||||
grammar = _k3_grammar("required", tools=_k3_tools_with_maxlen())
|
||||
|
||||
def body(val: str) -> str:
|
||||
return _k3_response() + _k3_tools(
|
||||
_k3_call("set_note", _k3_arg("note", "string", val))
|
||||
)
|
||||
|
||||
assert _is_grammar_accept_string(grammar, body("ab"))
|
||||
assert _is_grammar_accept_string(grammar, body("a<b then"))
|
||||
assert not _is_grammar_accept_string(grammar, body("way too long note"))
|
||||
assert not _is_grammar_accept_string(grammar, body("a")) # under minLength
|
||||
|
||||
|
||||
def test_kimi_k3_forced_tool_choice_builds_single_mandatory_call():
|
||||
# Named tool choice normalizes to "forced": the tag must require exactly
|
||||
# the named tool's call (no response-only escape).
|
||||
grammar = _k3_grammar(
|
||||
ChatCompletionNamedToolChoiceParam(
|
||||
type="function",
|
||||
function=ChatCompletionNamedFunction(name="get_weather"),
|
||||
),
|
||||
tools=_k3_tools_by_name(),
|
||||
)
|
||||
|
||||
ok = _k3_response() + _k3_tools(
|
||||
_k3_call("get_weather", _k3_arg("city", "string", "Paris"))
|
||||
)
|
||||
response_only = _k3_response("no call here")
|
||||
assert _is_grammar_accept_string(grammar, ok)
|
||||
assert not _is_grammar_accept_string(grammar, response_only)
|
||||
|
||||
@@ -0,0 +1,600 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import json
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm.entrypoints.openai.chat_completion.protocol import (
|
||||
ChatCompletionRequest,
|
||||
)
|
||||
from vllm.entrypoints.openai.engine.protocol import DeltaMessage
|
||||
from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
|
||||
from vllm.entrypoints.openai.responses.utils import build_response_output_items
|
||||
from vllm.exceptions import VLLMValidationError
|
||||
from vllm.parser.kimi_k3 import KimiK3Parser
|
||||
from vllm.parser.parser_manager import ParserManager
|
||||
from vllm.reasoning.kimi_k3_reasoning_parser import KimiK3ReasoningParser
|
||||
from vllm.tool_parsers.kimi_k3_tool_parser import KimiK3ToolParser
|
||||
|
||||
OPEN = "<|open|>"
|
||||
CLOSE = "<|close|>"
|
||||
SEP = "<|sep|>"
|
||||
THINK_OPEN = f"{OPEN}think{SEP}"
|
||||
THINK_CLOSE = f"{CLOSE}think{SEP}"
|
||||
RESPONSE_CLOSE = f"{CLOSE}response{SEP}"
|
||||
|
||||
|
||||
class DummyTokenizer:
|
||||
def get_vocab(self) -> dict[str, int]:
|
||||
return {}
|
||||
|
||||
def encode(self, text: str, add_special_tokens: bool = False) -> list[int]:
|
||||
if text == THINK_OPEN:
|
||||
return [1, 2, 3]
|
||||
if text == THINK_CLOSE:
|
||||
return [4, 2, 3]
|
||||
return [ord(ch) for ch in text]
|
||||
|
||||
|
||||
class KimiK3DelegatingParser(KimiK3Parser):
|
||||
reasoning_parser_cls = KimiK3ReasoningParser
|
||||
tool_parser_cls = KimiK3ToolParser
|
||||
|
||||
|
||||
def test_parser_manager_selects_kimi_k3_parser():
|
||||
parser_cls = ParserManager.get_parser(
|
||||
tool_parser_name="kimi_k3",
|
||||
reasoning_parser_name="kimi_k3",
|
||||
enable_auto_tools=True,
|
||||
)
|
||||
|
||||
assert parser_cls is not None
|
||||
assert issubclass(parser_cls, KimiK3Parser)
|
||||
assert parser_cls.reasoning_parser_cls is KimiK3ReasoningParser
|
||||
assert parser_cls.tool_parser_cls is KimiK3ToolParser
|
||||
|
||||
|
||||
def _request() -> ChatCompletionRequest:
|
||||
return ChatCompletionRequest(
|
||||
model="test-model",
|
||||
messages=[],
|
||||
tools=[
|
||||
{
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "calc",
|
||||
"parameters": {"type": "object", "properties": {}},
|
||||
},
|
||||
}
|
||||
],
|
||||
tool_choice="auto",
|
||||
)
|
||||
|
||||
|
||||
def _named_request() -> ChatCompletionRequest:
|
||||
return ChatCompletionRequest(
|
||||
model="test-model",
|
||||
messages=[],
|
||||
tools=[
|
||||
{
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "calc",
|
||||
"parameters": {"type": "object", "properties": {}},
|
||||
},
|
||||
}
|
||||
],
|
||||
tool_choice={"type": "function", "function": {"name": "calc"}},
|
||||
)
|
||||
|
||||
|
||||
def _responses_request(*, tool_choice="auto") -> ResponsesRequest:
|
||||
return ResponsesRequest.model_validate(
|
||||
{
|
||||
"model": "test-model",
|
||||
"input": "Call the calc tool.",
|
||||
"tools": [
|
||||
{
|
||||
"type": "function",
|
||||
"name": "calc",
|
||||
"parameters": {"type": "object", "properties": {}},
|
||||
}
|
||||
],
|
||||
"tool_choice": tool_choice,
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def _arg(key: str, typ: str, value: str) -> str:
|
||||
return f'{OPEN}argument key="{key}" type="{typ}"{SEP}{value}{CLOSE}argument{SEP}'
|
||||
|
||||
|
||||
def _call(tool: str, index: int, *args: str) -> str:
|
||||
body = "".join(args)
|
||||
return f'{OPEN}call tool="{tool}" index="{index}"{SEP}{body}{CLOSE}call{SEP}'
|
||||
|
||||
|
||||
def _response(content: str) -> str:
|
||||
return f"{OPEN}response{SEP}{content}{RESPONSE_CLOSE}"
|
||||
|
||||
|
||||
def _tools(*calls: str) -> str:
|
||||
return f"{OPEN}tools{SEP}{''.join(calls)}{CLOSE}tools{SEP}"
|
||||
|
||||
|
||||
def test_extract_tool_calls_with_response_and_typed_arguments():
|
||||
parser = KimiK3ToolParser(DummyTokenizer())
|
||||
|
||||
output = _response("answer") + _tools(
|
||||
_call(
|
||||
"calc",
|
||||
1,
|
||||
_arg("x", "number", "1"),
|
||||
_arg("flag", "boolean", "true"),
|
||||
_arg("text", "string", "raw"),
|
||||
)
|
||||
)
|
||||
extracted = parser.extract_tool_calls(output, _request())
|
||||
|
||||
assert extracted.tools_called is True
|
||||
assert extracted.content == "answer"
|
||||
assert len(extracted.tool_calls) == 1
|
||||
tool_call = extracted.tool_calls[0]
|
||||
assert tool_call.id == "calc:0"
|
||||
assert tool_call.function.name == "calc"
|
||||
assert json.loads(tool_call.function.arguments) == {
|
||||
"x": 1,
|
||||
"flag": True,
|
||||
"text": "raw",
|
||||
}
|
||||
|
||||
|
||||
def test_delegating_parser_preserves_tool_calls_after_reasoning():
|
||||
parser = KimiK3DelegatingParser(DummyTokenizer())
|
||||
output = (
|
||||
f"{THINK_OPEN}step{THINK_CLOSE}"
|
||||
+ _response("answer")
|
||||
+ _tools(_call("calc", 1, _arg("x", "number", "1")))
|
||||
)
|
||||
|
||||
reasoning, content, tool_calls = parser.parse(
|
||||
output,
|
||||
_request(),
|
||||
enable_auto_tools=True,
|
||||
)
|
||||
|
||||
assert reasoning == "step"
|
||||
assert content == "answer"
|
||||
assert tool_calls is not None
|
||||
assert len(tool_calls) == 1
|
||||
assert tool_calls[0].id == "calc:0"
|
||||
assert tool_calls[0].name == "calc"
|
||||
assert json.loads(tool_calls[0].arguments) == {"x": 1}
|
||||
|
||||
|
||||
def test_delegating_parser_required_tool_choice_uses_xtml_parser():
|
||||
parser = KimiK3DelegatingParser(DummyTokenizer())
|
||||
request = _request().model_copy(update={"tool_choice": "required"})
|
||||
output = (
|
||||
f"{THINK_OPEN}step{THINK_CLOSE}"
|
||||
+ _response("")
|
||||
+ _tools(_call("calc", 1, _arg("x", "number", "1")))
|
||||
)
|
||||
|
||||
reasoning, content, tool_calls = parser.parse(
|
||||
output,
|
||||
request,
|
||||
enable_auto_tools=True,
|
||||
)
|
||||
|
||||
assert reasoning == "step"
|
||||
assert content is None
|
||||
assert tool_calls is not None
|
||||
assert len(tool_calls) == 1
|
||||
assert tool_calls[0].name == "calc"
|
||||
assert json.loads(tool_calls[0].arguments) == {"x": 1}
|
||||
|
||||
|
||||
def test_delegating_parser_named_tool_choice_uses_xtml_parser():
|
||||
parser = KimiK3DelegatingParser(DummyTokenizer())
|
||||
output = (
|
||||
f"{THINK_OPEN}step{THINK_CLOSE}"
|
||||
+ _response("")
|
||||
+ _tools(_call("calc", 1, _arg("x", "number", "1")))
|
||||
)
|
||||
|
||||
reasoning, content, tool_calls = parser.parse(
|
||||
output,
|
||||
_named_request(),
|
||||
enable_auto_tools=True,
|
||||
)
|
||||
|
||||
assert reasoning == "step"
|
||||
assert content is None
|
||||
assert tool_calls is not None
|
||||
assert len(tool_calls) == 1
|
||||
assert tool_calls[0].name == "calc"
|
||||
assert json.loads(tool_calls[0].arguments) == {"x": 1}
|
||||
|
||||
|
||||
def test_delegating_parser_auto_no_call_strips_consumed_response_prefix():
|
||||
parser = KimiK3DelegatingParser(
|
||||
DummyTokenizer(), chat_template_kwargs={"thinking": False}
|
||||
)
|
||||
request = _request().model_copy(
|
||||
update={"chat_template_kwargs": {"thinking": False}}
|
||||
)
|
||||
|
||||
reasoning, content, tool_calls = parser.parse(
|
||||
f"answer{RESPONSE_CLOSE}",
|
||||
request,
|
||||
enable_auto_tools=True,
|
||||
)
|
||||
|
||||
assert reasoning is None
|
||||
assert content == "answer"
|
||||
assert tool_calls is None
|
||||
|
||||
|
||||
def test_delegating_parser_required_call_strips_consumed_response_prefix():
|
||||
parser = KimiK3DelegatingParser(
|
||||
DummyTokenizer(), chat_template_kwargs={"thinking": False}
|
||||
)
|
||||
request = _request().model_copy(
|
||||
update={
|
||||
"tool_choice": "required",
|
||||
"chat_template_kwargs": {"thinking": False},
|
||||
}
|
||||
)
|
||||
output = RESPONSE_CLOSE + _tools(_call("calc", 1, _arg("x", "number", "1")))
|
||||
|
||||
reasoning, content, tool_calls = parser.parse(
|
||||
output,
|
||||
request,
|
||||
enable_auto_tools=True,
|
||||
)
|
||||
|
||||
assert reasoning is None
|
||||
assert content is None
|
||||
assert tool_calls is not None
|
||||
assert len(tool_calls) == 1
|
||||
assert tool_calls[0].name == "calc"
|
||||
assert json.loads(tool_calls[0].arguments) == {"x": 1}
|
||||
|
||||
|
||||
def test_delegating_parser_truncated_tools_do_not_leak_xtml():
|
||||
parser = KimiK3DelegatingParser(
|
||||
DummyTokenizer(), chat_template_kwargs={"thinking": False}
|
||||
)
|
||||
request = _request().model_copy(
|
||||
update={
|
||||
"tool_choice": "required",
|
||||
"chat_template_kwargs": {"thinking": False},
|
||||
}
|
||||
)
|
||||
|
||||
reasoning, content, tool_calls = parser.parse(
|
||||
(f'{RESPONSE_CLOSE}{OPEN}tools{SEP}{OPEN}call tool="calc" index="1"'),
|
||||
request,
|
||||
enable_auto_tools=True,
|
||||
)
|
||||
|
||||
assert reasoning is None
|
||||
assert content is None
|
||||
assert tool_calls is None
|
||||
|
||||
|
||||
def test_extract_tool_calls_unescapes_attributes():
|
||||
parser = KimiK3ToolParser(DummyTokenizer())
|
||||
|
||||
output = _tools(_call("a&b"c", 1, _arg("k&q", "string", "v")))
|
||||
extracted = parser.extract_tool_calls(output, _request())
|
||||
|
||||
assert extracted.tools_called is True
|
||||
assert extracted.tool_calls[0].function.name == 'a&b"c'
|
||||
assert json.loads(extracted.tool_calls[0].function.arguments) == {"k&q": "v"}
|
||||
|
||||
|
||||
def test_extract_tool_calls_allows_less_than_in_attributes():
|
||||
parser = KimiK3ToolParser(DummyTokenizer())
|
||||
|
||||
output = _tools(_call("calc<beta", 1, _arg("foo<bar", "string", "raw")))
|
||||
extracted = parser.extract_tool_calls(output, _request())
|
||||
|
||||
assert extracted.tools_called is True
|
||||
assert extracted.tool_calls[0].function.name == "calc<beta"
|
||||
assert json.loads(extracted.tool_calls[0].function.arguments) == {"foo<bar": "raw"}
|
||||
|
||||
|
||||
def test_extract_content_from_whitespace_degraded_markers():
|
||||
parser = KimiK3ToolParser(DummyTokenizer())
|
||||
|
||||
extracted = parser.extract_tool_calls(
|
||||
f"{OPEN} response {SEP}answer{CLOSE} response {SEP}",
|
||||
_request(),
|
||||
)
|
||||
|
||||
assert extracted.tools_called is False
|
||||
assert extracted.content == "answer"
|
||||
|
||||
|
||||
def test_streaming_split_markers_do_not_leak():
|
||||
parser = KimiK3ToolParser(DummyTokenizer())
|
||||
request = _request()
|
||||
previous_text = ""
|
||||
previous_ids: list[int] = []
|
||||
messages: list[DeltaMessage] = []
|
||||
chunks = [
|
||||
OPEN,
|
||||
"response",
|
||||
f"{SEP}Hi",
|
||||
OPEN,
|
||||
"tools",
|
||||
SEP,
|
||||
f'{OPEN}call tool="calc" index="1"{SEP}',
|
||||
_arg("x", "number", "1"),
|
||||
f"{CLOSE}call",
|
||||
SEP,
|
||||
]
|
||||
|
||||
for i, chunk in enumerate(chunks, start=1):
|
||||
current_text = previous_text + chunk
|
||||
current_ids = previous_ids + [i]
|
||||
delta = parser.extract_tool_calls_streaming(
|
||||
previous_text=previous_text,
|
||||
current_text=current_text,
|
||||
delta_text=chunk,
|
||||
previous_token_ids=previous_ids,
|
||||
current_token_ids=current_ids,
|
||||
delta_token_ids=[i],
|
||||
request=request,
|
||||
)
|
||||
if delta is not None:
|
||||
messages.append(delta)
|
||||
previous_text = current_text
|
||||
previous_ids = current_ids
|
||||
|
||||
content = "".join(message.content or "" for message in messages)
|
||||
tool_deltas = [
|
||||
tool_call for message in messages for tool_call in (message.tool_calls or [])
|
||||
]
|
||||
|
||||
assert content == "Hi"
|
||||
assert OPEN not in content
|
||||
assert SEP not in content
|
||||
assert len(tool_deltas) == 1
|
||||
assert tool_deltas[0].id == "calc:0"
|
||||
assert tool_deltas[0].function.name == "calc"
|
||||
assert json.loads(tool_deltas[0].function.arguments) == {"x": 1}
|
||||
|
||||
|
||||
def test_streaming_consumed_response_prefix_no_call_keeps_content():
|
||||
parser = KimiK3ToolParser(DummyTokenizer())
|
||||
request = _request()
|
||||
previous_text = ""
|
||||
previous_ids: list[int] = []
|
||||
messages: list[DeltaMessage] = []
|
||||
chunks = ["O", "K", CLOSE, f"response{SEP}"]
|
||||
|
||||
for i, chunk in enumerate(chunks, start=1):
|
||||
current_text = previous_text + chunk
|
||||
current_ids = previous_ids + [i]
|
||||
delta = parser.extract_tool_calls_streaming(
|
||||
previous_text=previous_text,
|
||||
current_text=current_text,
|
||||
delta_text=chunk,
|
||||
previous_token_ids=previous_ids,
|
||||
current_token_ids=current_ids,
|
||||
delta_token_ids=[i],
|
||||
request=request,
|
||||
)
|
||||
if delta is not None:
|
||||
messages.append(delta)
|
||||
previous_text = current_text
|
||||
previous_ids = current_ids
|
||||
|
||||
assert "".join(message.content or "" for message in messages) == "OK"
|
||||
assert all(CLOSE not in (message.content or "") for message in messages)
|
||||
|
||||
|
||||
def test_delegating_parser_tool_choice_none_strips_xtml_and_suppresses_calls():
|
||||
parser = KimiK3DelegatingParser(
|
||||
DummyTokenizer(), chat_template_kwargs={"thinking": False}
|
||||
)
|
||||
request = _request().model_copy(
|
||||
update={
|
||||
"tool_choice": "none",
|
||||
"chat_template_kwargs": {"thinking": False},
|
||||
}
|
||||
)
|
||||
messages: list[DeltaMessage] = []
|
||||
chunks = [
|
||||
OPEN,
|
||||
"response",
|
||||
f"{SEP}answer",
|
||||
RESPONSE_CLOSE,
|
||||
_tools(_call("calc", 1, _arg("x", "number", "1"))),
|
||||
]
|
||||
|
||||
for index, chunk in enumerate(chunks, start=1):
|
||||
delta = parser.parse_delta(
|
||||
delta_text=chunk,
|
||||
delta_token_ids=[index],
|
||||
request=request,
|
||||
prompt_token_ids=[1],
|
||||
finished=index == len(chunks),
|
||||
)
|
||||
if delta is not None:
|
||||
messages.append(delta)
|
||||
|
||||
content = "".join(message.content or "" for message in messages)
|
||||
assert content == "answer"
|
||||
assert OPEN not in content
|
||||
assert CLOSE not in content
|
||||
assert SEP not in content
|
||||
assert all(not message.tool_calls for message in messages)
|
||||
|
||||
|
||||
def test_adjust_request_keeps_xtml_markers_contiguous():
|
||||
parser = KimiK3ToolParser(DummyTokenizer())
|
||||
request = _request()
|
||||
|
||||
adjusted = parser.adjust_request(request)
|
||||
|
||||
assert adjusted.skip_special_tokens is False
|
||||
if hasattr(adjusted, "spaces_between_special_tokens"):
|
||||
assert adjusted.spaces_between_special_tokens is False
|
||||
assert KimiK3ToolParser.supports_required_and_named is False
|
||||
|
||||
|
||||
def test_adjust_request_required_uses_xtml_parser_not_json_guidance():
|
||||
parser = KimiK3ToolParser(DummyTokenizer())
|
||||
request = _request().model_copy(update={"tool_choice": "required"})
|
||||
|
||||
adjusted = parser.adjust_request(request)
|
||||
|
||||
assert adjusted.structured_outputs is None
|
||||
assert adjusted.skip_special_tokens is False
|
||||
if hasattr(adjusted, "spaces_between_special_tokens"):
|
||||
assert adjusted.spaces_between_special_tokens is False
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"tool_request",
|
||||
[
|
||||
_named_request(),
|
||||
_responses_request(
|
||||
tool_choice={"type": "function", "name": "calc"},
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_adjust_request_rejects_named_tool_choice(tool_request):
|
||||
parser = KimiK3ToolParser(DummyTokenizer())
|
||||
|
||||
with pytest.raises(VLLMValidationError) as exc_info:
|
||||
parser.adjust_request(tool_request)
|
||||
|
||||
assert exc_info.value.parameter == "tool_choice"
|
||||
assert "requires strict tool calling" in str(exc_info.value)
|
||||
|
||||
|
||||
def test_responses_chat_params_carries_tool_choice_metadata():
|
||||
request = _responses_request(tool_choice="required")
|
||||
|
||||
chat_params = request.build_chat_params(
|
||||
default_template=None,
|
||||
default_template_content_format="auto",
|
||||
)
|
||||
|
||||
assert chat_params.tool_choice == "required"
|
||||
|
||||
|
||||
def test_responses_chat_params_keeps_template_tool_choice_when_api_auto():
|
||||
request = _responses_request().model_copy(
|
||||
update={"chat_template_kwargs": {"tool_choice": "required"}}
|
||||
)
|
||||
|
||||
chat_params = request.build_chat_params(
|
||||
default_template=None,
|
||||
default_template_content_format="auto",
|
||||
)
|
||||
|
||||
assert chat_params.chat_template_kwargs["tool_choice"] == "required"
|
||||
assert chat_params.tool_choice == "auto"
|
||||
|
||||
|
||||
def test_responses_required_tool_choice_uses_xtml_parser():
|
||||
parser = KimiK3DelegatingParser(
|
||||
DummyTokenizer(), chat_template_kwargs={"thinking": False}
|
||||
)
|
||||
request = _responses_request(tool_choice="required").model_copy(
|
||||
update={"chat_template_kwargs": {"thinking": False}}
|
||||
)
|
||||
output = RESPONSE_CLOSE + _tools(_call("calc", 1, _arg("x", "number", "1")))
|
||||
|
||||
reasoning, content, tool_calls = parser.parse(
|
||||
output, request, enable_auto_tools=True, model_output_token_ids=[]
|
||||
)
|
||||
response_outputs = build_response_output_items(
|
||||
reasoning=reasoning,
|
||||
content=content,
|
||||
tool_calls=tool_calls,
|
||||
tools=request.tools,
|
||||
)
|
||||
|
||||
assert len(response_outputs) == 1
|
||||
tool_call = response_outputs[0]
|
||||
assert tool_call.type == "function_call"
|
||||
assert tool_call.name == "calc"
|
||||
assert json.loads(tool_call.arguments) == {"x": 1}
|
||||
|
||||
|
||||
def test_responses_named_tool_choice_uses_xtml_parser():
|
||||
parser = KimiK3DelegatingParser(
|
||||
DummyTokenizer(), chat_template_kwargs={"thinking": False}
|
||||
)
|
||||
request = _responses_request(
|
||||
tool_choice={"type": "function", "name": "calc"}
|
||||
).model_copy(update={"chat_template_kwargs": {"thinking": False}})
|
||||
output = RESPONSE_CLOSE + _tools(_call("calc", 1, _arg("x", "number", "1")))
|
||||
|
||||
reasoning, content, tool_calls = parser.parse(
|
||||
output, request, enable_auto_tools=True, model_output_token_ids=[]
|
||||
)
|
||||
response_outputs = build_response_output_items(
|
||||
reasoning=reasoning,
|
||||
content=content,
|
||||
tool_calls=tool_calls,
|
||||
tools=request.tools,
|
||||
)
|
||||
|
||||
assert len(response_outputs) == 1
|
||||
tool_call = response_outputs[0]
|
||||
assert tool_call.type == "function_call"
|
||||
assert tool_call.name == "calc"
|
||||
assert json.loads(tool_call.arguments) == {"x": 1}
|
||||
|
||||
|
||||
def test_chat_params_carries_tool_choice_metadata():
|
||||
request = _request().model_copy(update={"tool_choice": "required"})
|
||||
|
||||
chat_params = request.build_chat_params(
|
||||
default_template=None,
|
||||
default_template_content_format="auto",
|
||||
)
|
||||
|
||||
assert chat_params.tool_choice == "required"
|
||||
|
||||
|
||||
def test_chat_params_carries_response_format_metadata():
|
||||
request = ChatCompletionRequest(
|
||||
model="test-model",
|
||||
messages=[],
|
||||
response_format={"type": "json_object"},
|
||||
)
|
||||
|
||||
chat_params = request.build_chat_params(
|
||||
default_template=None,
|
||||
default_template_content_format="auto",
|
||||
)
|
||||
|
||||
assert chat_params.response_format is request.response_format
|
||||
assert chat_params.tool_choice is None
|
||||
|
||||
|
||||
def test_chat_params_keeps_template_tool_choice_when_api_auto():
|
||||
request = _request().model_copy(
|
||||
update={
|
||||
"tool_choice": "auto",
|
||||
"chat_template_kwargs": {"tool_choice": "required"},
|
||||
}
|
||||
)
|
||||
|
||||
chat_params = request.build_chat_params(
|
||||
default_template=None,
|
||||
default_template_content_format="auto",
|
||||
)
|
||||
|
||||
assert chat_params.chat_template_kwargs["tool_choice"] == "required"
|
||||
assert chat_params.tool_choice == "auto"
|
||||
Reference in New Issue
Block a user