[Mini-vLLM] Cut dead subsystems: models, entrypoints, backends, utils, tests

- Drop dead model modules, keeping files live imports require
- Remove anthropic, cohere, codegen entrypoints and batch STT modes
- Delete unused attention backends, MoE experts, and NPU/XPU util leaves
- Strip dead third_party trees and ROCm leaves with no CUDA path
- Remove mooncake EC connectors: dependency missing since branch base
- Re-audit verified by full import sweep and Qwen3.5 smoke test

Co-authored-by: Hinata <hinata@idihore.id>
Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
2026-10-19 00:20:48 +07:00
co-authored by Hinata
parent 6a95f62e2f
commit 81074e94a3
940 changed files with 11 additions and 380533 deletions
@@ -1,260 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Regression tests for Responses API tool-calling request adjustment.
Covers two bugs on the ``/v1/responses`` path that broke streaming tool
calling for parsers relying on special-token delimiters (Gemma4):
1. :class:`Gemma4ToolParser.adjust_request` used an
``isinstance(request, ChatCompletionRequest)`` guard, so a
:class:`ResponsesRequest` with tools never had
``skip_special_tokens`` flipped to ``False``. The default (``True``)
stripped ``<|tool_call>`` / ``<tool_call|>`` delimiters, causing
:meth:`Gemma4ToolParser.extract_tool_calls_streaming` to fall through
to the content branch and leak the raw ``call:fn{...}`` body via
``response.output_text.delta``.
2. :meth:`ToolParser.adjust_request` built
:class:`ResponseTextConfig` in two steps (bare constructor then
``.format = ...``). Under Pydantic v2 the later assignment is not
tracked in ``__fields_set__``, which can drop the nested config from
``model_dump``. It also passed a ``description`` kwarg carrying the
wrong-purpose string ``"Response format for tool calling"``.
3. :class:`Gemma4EngineToolParser` (the engine-based parser, #45588) sets
``supports_required_and_named=False`` but did not skip the forced
``structured_outputs`` JSON for ``required``/named tool choice. The model
was constrained to JSON the native parser cannot read, so the call leaked
as content with empty ``tool_calls``. ``adjust_request`` now skips that
constraint so Gemma4 emits its native ``<|tool_call>`` syntax.
"""
from __future__ import annotations
from typing import Any
from openai.types.responses.tool_param import FunctionToolParam
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
from vllm.tool_parsers.abstract_tool_parser import ToolParser
from vllm.tool_parsers.gemma4_engine_tool_parser import (
Gemma4EngineToolParser as Gemma4ToolParser,
)
def _get_weather_tool() -> FunctionToolParam:
return FunctionToolParam(
type="function",
name="get_weather",
description="Get current weather for a city",
parameters={
"type": "object",
"properties": {"city": {"type": "string"}},
"required": ["city"],
},
strict=True,
)
def _build_responses_request(*, tool_choice: str | dict[str, Any]) -> ResponsesRequest:
return ResponsesRequest(
model="gemma4-test",
input=[{"role": "user", "content": "What is the weather in Hanoi?"}],
tools=[_get_weather_tool()],
tool_choice=tool_choice,
stream=True,
max_output_tokens=200,
)
def _build_chat_request(
*,
tool_choice: str | dict[str, Any],
chat_template_kwargs: dict[str, Any] | None = None,
) -> ChatCompletionRequest:
data: dict[str, Any] = {
"model": "gemma4-test",
"messages": [{"role": "user", "content": "What is the weather in Hanoi?"}],
"tools": [
{
"type": "function",
"function": {
"name": "get_weather",
"description": "Get current weather for a city",
"parameters": {
"type": "object",
"properties": {"city": {"type": "string"}},
"required": ["city"],
},
},
}
],
"tool_choice": tool_choice,
}
if chat_template_kwargs is not None:
data["chat_template_kwargs"] = chat_template_kwargs
return ChatCompletionRequest.model_validate(data)
class _StubTokenizer:
"""Minimal tokenizer stub to satisfy ``Gemma4EngineToolParser.__init__``."""
_VOCAB: dict[str, int] = {
"<|tool_call>": 256_000,
"<tool_call|>": 256_001,
'<|"|>': 52,
"<|channel>": 256_002,
"<channel|>": 256_003,
}
def get_vocab(self) -> dict[str, int]:
return dict(self._VOCAB)
@property
def all_special_tokens(self) -> list[str]:
return list(self._VOCAB.keys())
@property
def all_special_ids(self) -> list[int]:
return list(self._VOCAB.values())
def test_gemma4_adjust_request_sets_skip_special_tokens_on_responses() -> None:
"""``Gemma4ToolParser.adjust_request`` must flip
``skip_special_tokens=False`` for both ``ChatCompletionRequest`` and
``ResponsesRequest`` so that ``<|tool_call>`` delimiters reach the
streaming extractor. The previous
``isinstance(ChatCompletionRequest)`` guard omitted the Responses
path, causing raw ``call:fn{...}`` text to leak via
``response.output_text.delta``.
"""
parser = Gemma4ToolParser(_StubTokenizer())
request = _build_responses_request(tool_choice="auto")
assert request.skip_special_tokens is True, (
"Precondition: ResponsesRequest.skip_special_tokens default is True"
)
parser.adjust_request(request)
assert request.skip_special_tokens is False
def test_tool_parser_adjust_request_builds_valid_response_text_config() -> None:
"""``ToolParser.adjust_request`` must produce a ``ResponseTextConfig``
whose dumped form contains the JSON schema under the ``schema`` alias
and does not leak the unrelated ``"Response format for tool calling"``
description string that the previous two-step construction injected.
"""
parser = ToolParser.__new__(ToolParser)
parser.model_tokenizer = None
request = _build_responses_request(tool_choice="required")
ToolParser.adjust_request(parser, request)
assert request.text is not None
assert request.text.format is not None
assert request.text.format.type == "json_schema"
dump: dict[str, Any] = request.text.model_dump(mode="json", by_alias=True)
fmt = dump.get("format") or {}
assert fmt.get("type") == "json_schema"
assert fmt.get("name") == "tool_calling_response"
assert fmt.get("strict") is True
# Nested config must be present under the alias. Two-step Pydantic v2
# construction could drop it from __fields_set__.
assert "schema" in fmt and isinstance(fmt["schema"], dict)
# The old code passed a wrong-purpose string; valid field should now
# either be absent or None (the openai-python default).
assert fmt.get("description") in (None, "")
def test_gemma4_required_skips_structured_outputs_chatcompletion() -> None:
"""Required + ChatCompletion: ``Gemma4EngineToolParser`` must skip the
forced JSON ``structured_outputs`` so the model emits its native
``<|tool_call>`` syntax. The base parser constrained output to JSON the
native parser cannot read, leaking it as content with empty
``tool_calls`` (regression after #45588).
"""
parser = Gemma4ToolParser(_StubTokenizer())
request = _build_chat_request(tool_choice="required")
parser.adjust_request(request)
assert request.structured_outputs is None
assert request.skip_special_tokens is False
def test_gemma4_named_skips_structured_outputs_chatcompletion() -> None:
"""Named + ChatCompletion: the forced single-function JSON schema must be
skipped, same as ``required``.
"""
parser = Gemma4ToolParser(_StubTokenizer())
request = _build_chat_request(
tool_choice={"type": "function", "function": {"name": "get_weather"}}
)
parser.adjust_request(request)
assert request.structured_outputs is None
assert request.skip_special_tokens is False
def test_gemma4_required_skips_structured_outputs_responses() -> None:
"""Required + Responses: the forced JSON schema (``request.text``) must be
skipped so the native delimiters reach the extractor.
"""
parser = Gemma4ToolParser(_StubTokenizer())
request = _build_responses_request(tool_choice="required")
parser.adjust_request(request)
assert request.text is None
assert request.skip_special_tokens is False
def test_gemma4_named_skips_structured_outputs_responses() -> None:
"""Named (``ToolChoiceFunction``) + Responses: the forced single-function
JSON schema must be skipped.
"""
parser = Gemma4ToolParser(_StubTokenizer())
request = _build_responses_request(
tool_choice={"type": "function", "name": "get_weather"}
)
parser.adjust_request(request)
assert request.text is None
assert request.skip_special_tokens is False
def test_gemma4_keeps_special_tokens_with_tools_thinking_disabled() -> None:
"""Tools active + thinking disabled: ``skip_special_tokens`` must stay
False so ``<|tool_call>`` delimiters reach the extractor. The merged
enable_thinking early-return stripped them, breaking tool calling when
thinking is off.
"""
parser = Gemma4ToolParser(_StubTokenizer())
request = _build_chat_request(
tool_choice="auto", chat_template_kwargs={"enable_thinking": False}
)
parser.adjust_request(request)
assert request.skip_special_tokens is False
def test_gemma4_keeps_skip_special_tokens_false_when_nothing_to_preserve() -> None:
"""No active tools + thinking disabled: ``skip_special_tokens`` stays
``False`` because the parser engine's ``__DROP__`` terminal mechanism
strips unconfigured special tokens automatically.
"""
parser = Gemma4ToolParser(_StubTokenizer())
request = _build_chat_request(
tool_choice="none", chat_template_kwargs={"enable_thinking": False}
)
parser.adjust_request(request)
assert request.skip_special_tokens is False
-613
View File
@@ -1,613 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import json
from typing import cast
import pytest
from vllm.entrypoints.generate.base.protocol import DeltaMessage
from vllm.entrypoints.openai.chat_completion.protocol import (
ChatCompletionRequest,
)
from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
from vllm.entrypoints.openai.responses.utils import build_response_output_items
from vllm.exceptions import VLLMValidationError
from vllm.parser.kimi_k3 import KimiK3Parser
from vllm.parser.parser_manager import ParserManager
from vllm.reasoning.kimi_k3_reasoning_parser import KimiK3ReasoningParser
from vllm.tokenizers.protocol import TokenizerLike
from vllm.tool_parsers.kimi_k3_tool_parser import KimiK3ToolParser
OPEN = "<|open|>"
CLOSE = "<|close|>"
SEP = "<|sep|>"
THINK_OPEN = f"{OPEN}think{SEP}"
THINK_CLOSE = f"{CLOSE}think{SEP}"
RESPONSE_CLOSE = f"{CLOSE}response{SEP}"
class DummyTokenizer:
def get_vocab(self) -> dict[str, int]:
return {}
def encode(self, text: str, add_special_tokens: bool = False) -> list[int]:
if text == THINK_OPEN:
return [1, 2, 3]
if text == THINK_CLOSE:
return [4, 2, 3]
return [ord(ch) for ch in text]
def _dummy_tokenizer() -> TokenizerLike:
return cast(TokenizerLike, DummyTokenizer())
class KimiK3DelegatingParser(KimiK3Parser):
reasoning_parser_cls = KimiK3ReasoningParser
tool_parser_cls = KimiK3ToolParser
def test_parser_manager_selects_kimi_k3_parser():
parser_cls = ParserManager.get_parser(
tool_parser_name="kimi_k3",
reasoning_parser_name="kimi_k3",
enable_auto_tools=True,
)
assert parser_cls is not None
assert issubclass(parser_cls, KimiK3Parser)
assert parser_cls.reasoning_parser_cls is KimiK3ReasoningParser
assert parser_cls.tool_parser_cls is KimiK3ToolParser
def _request() -> ChatCompletionRequest:
return ChatCompletionRequest(
model="test-model",
messages=[],
tools=[
{
"type": "function",
"function": {
"name": "calc",
"parameters": {"type": "object", "properties": {}},
},
}
],
tool_choice="auto",
)
def _named_request() -> ChatCompletionRequest:
return ChatCompletionRequest(
model="test-model",
messages=[],
tools=[
{
"type": "function",
"function": {
"name": "calc",
"parameters": {"type": "object", "properties": {}},
},
}
],
tool_choice={"type": "function", "function": {"name": "calc"}},
)
def _responses_request(*, tool_choice="auto") -> ResponsesRequest:
return ResponsesRequest.model_validate(
{
"model": "test-model",
"input": "Call the calc tool.",
"tools": [
{
"type": "function",
"name": "calc",
"parameters": {"type": "object", "properties": {}},
}
],
"tool_choice": tool_choice,
}
)
def _arg(key: str, typ: str, value: str) -> str:
return f'{OPEN}argument key="{key}" type="{typ}"{SEP}{value}{CLOSE}argument{SEP}'
def _call(tool: str, index: int, *args: str) -> str:
body = "".join(args)
return f'{OPEN}call tool="{tool}" index="{index}"{SEP}{body}{CLOSE}call{SEP}'
def _response(content: str) -> str:
return f"{OPEN}response{SEP}{content}{RESPONSE_CLOSE}"
def _tools(*calls: str) -> str:
return f"{OPEN}tools{SEP}{''.join(calls)}{CLOSE}tools{SEP}"
def test_extract_tool_calls_with_response_and_typed_arguments():
parser = KimiK3ToolParser(_dummy_tokenizer())
output = _response("answer") + _tools(
_call(
"calc",
1,
_arg("x", "number", "1"),
_arg("flag", "boolean", "true"),
_arg("text", "string", "raw"),
)
)
extracted = parser.extract_tool_calls(output, _request())
assert extracted.tools_called is True
assert extracted.content == "answer"
assert len(extracted.tool_calls) == 1
tool_call = extracted.tool_calls[0]
assert tool_call.function.name == "calc"
assert json.loads(tool_call.function.arguments) == {
"x": 1,
"flag": True,
"text": "raw",
}
def test_delegating_parser_preserves_tool_calls_after_reasoning():
parser = KimiK3DelegatingParser(_dummy_tokenizer())
output = (
f"{THINK_OPEN}step{THINK_CLOSE}"
+ _response("answer")
+ _tools(_call("calc", 1, _arg("x", "number", "1")))
)
reasoning, content, tool_calls = parser.parse(
output,
_request(),
enable_auto_tools=True,
)
assert reasoning == "step"
assert content == "answer"
assert tool_calls is not None
assert len(tool_calls) == 1
assert tool_calls[0].name == "calc"
assert json.loads(tool_calls[0].arguments) == {"x": 1}
def test_delegating_parser_required_tool_choice_uses_xtml_parser():
parser = KimiK3DelegatingParser(_dummy_tokenizer())
request = _request().model_copy(update={"tool_choice": "required"})
output = (
f"{THINK_OPEN}step{THINK_CLOSE}"
+ _response("")
+ _tools(_call("calc", 1, _arg("x", "number", "1")))
)
reasoning, content, tool_calls = parser.parse(
output,
request,
enable_auto_tools=True,
)
assert reasoning == "step"
assert content is None
assert tool_calls is not None
assert len(tool_calls) == 1
assert tool_calls[0].name == "calc"
assert json.loads(tool_calls[0].arguments) == {"x": 1}
def test_delegating_parser_named_tool_choice_uses_xtml_parser():
parser = KimiK3DelegatingParser(_dummy_tokenizer())
output = (
f"{THINK_OPEN}step{THINK_CLOSE}"
+ _response("")
+ _tools(_call("calc", 1, _arg("x", "number", "1")))
)
reasoning, content, tool_calls = parser.parse(
output,
_named_request(),
enable_auto_tools=True,
)
assert reasoning == "step"
assert content is None
assert tool_calls is not None
assert len(tool_calls) == 1
assert tool_calls[0].name == "calc"
assert json.loads(tool_calls[0].arguments) == {"x": 1}
def test_delegating_parser_auto_no_call_strips_consumed_response_prefix():
parser = KimiK3DelegatingParser(
_dummy_tokenizer(), chat_template_kwargs={"thinking": False}
)
request = _request().model_copy(
update={"chat_template_kwargs": {"thinking": False}}
)
reasoning, content, tool_calls = parser.parse(
f"answer{RESPONSE_CLOSE}",
request,
enable_auto_tools=True,
)
assert reasoning is None
assert content == "answer"
assert tool_calls is None
def test_delegating_parser_required_call_strips_consumed_response_prefix():
parser = KimiK3DelegatingParser(
_dummy_tokenizer(), chat_template_kwargs={"thinking": False}
)
request = _request().model_copy(
update={
"tool_choice": "required",
"chat_template_kwargs": {"thinking": False},
}
)
output = RESPONSE_CLOSE + _tools(_call("calc", 1, _arg("x", "number", "1")))
reasoning, content, tool_calls = parser.parse(
output,
request,
enable_auto_tools=True,
)
assert reasoning is None
assert content is None
assert tool_calls is not None
assert len(tool_calls) == 1
assert tool_calls[0].name == "calc"
assert json.loads(tool_calls[0].arguments) == {"x": 1}
def test_delegating_parser_truncated_tools_do_not_leak_xtml():
parser = KimiK3DelegatingParser(
_dummy_tokenizer(), chat_template_kwargs={"thinking": False}
)
request = _request().model_copy(
update={
"tool_choice": "required",
"chat_template_kwargs": {"thinking": False},
}
)
reasoning, content, tool_calls = parser.parse(
(f'{RESPONSE_CLOSE}{OPEN}tools{SEP}{OPEN}call tool="calc" index="1"'),
request,
enable_auto_tools=True,
)
assert reasoning is None
assert content is None
assert tool_calls is None
def test_extract_tool_calls_unescapes_attributes():
parser = KimiK3ToolParser(_dummy_tokenizer())
output = _tools(_call("a&amp;b&quot;c", 1, _arg("k&amp;q", "string", "v")))
extracted = parser.extract_tool_calls(output, _request())
assert extracted.tools_called is True
assert extracted.tool_calls[0].function.name == 'a&b"c'
assert json.loads(extracted.tool_calls[0].function.arguments) == {"k&q": "v"}
def test_extract_tool_calls_allows_less_than_in_attributes():
parser = KimiK3ToolParser(_dummy_tokenizer())
output = _tools(_call("calc<beta", 1, _arg("foo<bar", "string", "raw")))
extracted = parser.extract_tool_calls(output, _request())
assert extracted.tools_called is True
assert extracted.tool_calls[0].function.name == "calc<beta"
assert json.loads(extracted.tool_calls[0].function.arguments) == {"foo<bar": "raw"}
def test_extract_content_from_whitespace_degraded_markers():
parser = KimiK3ToolParser(_dummy_tokenizer())
extracted = parser.extract_tool_calls(
f"{OPEN} response {SEP}answer{CLOSE} response {SEP}",
_request(),
)
assert extracted.tools_called is False
assert extracted.content == "answer"
def test_streaming_split_markers_do_not_leak():
parser = KimiK3ToolParser(_dummy_tokenizer())
request = _request()
previous_text = ""
previous_ids: list[int] = []
messages: list[DeltaMessage] = []
chunks = [
OPEN,
"response",
f"{SEP}Hi",
OPEN,
"tools",
SEP,
f'{OPEN}call tool="calc" index="1"{SEP}',
_arg("x", "number", "1"),
f"{CLOSE}call",
SEP,
]
for i, chunk in enumerate(chunks, start=1):
current_text = previous_text + chunk
current_ids = previous_ids + [i]
delta = parser.extract_tool_calls_streaming(
previous_text=previous_text,
current_text=current_text,
delta_text=chunk,
previous_token_ids=previous_ids,
current_token_ids=current_ids,
delta_token_ids=[i],
request=request,
)
if delta is not None:
messages.append(delta)
previous_text = current_text
previous_ids = current_ids
content = "".join(message.content or "" for message in messages)
tool_deltas = [
tool_call for message in messages for tool_call in (message.tool_calls or [])
]
assert content == "Hi"
assert OPEN not in content
assert SEP not in content
assert len(tool_deltas) == 1
assert tool_deltas[0].function is not None
assert tool_deltas[0].function.name == "calc"
assert json.loads(tool_deltas[0].function.arguments) == {"x": 1}
def test_tool_call_ids_are_unique_across_messages():
output = _tools(_call("calc", 1))
first = KimiK3ToolParser(_dummy_tokenizer()).extract_tool_calls(output, _request())
second = KimiK3ToolParser(_dummy_tokenizer()).extract_tool_calls(output, _request())
assert first.tool_calls[0].id != second.tool_calls[0].id
def test_streaming_consumed_response_prefix_no_call_keeps_content():
parser = KimiK3ToolParser(_dummy_tokenizer())
request = _request()
previous_text = ""
previous_ids: list[int] = []
messages: list[DeltaMessage] = []
chunks = ["O", "K", CLOSE, f"response{SEP}"]
for i, chunk in enumerate(chunks, start=1):
current_text = previous_text + chunk
current_ids = previous_ids + [i]
delta = parser.extract_tool_calls_streaming(
previous_text=previous_text,
current_text=current_text,
delta_text=chunk,
previous_token_ids=previous_ids,
current_token_ids=current_ids,
delta_token_ids=[i],
request=request,
)
if delta is not None:
messages.append(delta)
previous_text = current_text
previous_ids = current_ids
assert "".join(message.content or "" for message in messages) == "OK"
assert all(CLOSE not in (message.content or "") for message in messages)
def test_delegating_parser_tool_choice_none_strips_xtml_and_suppresses_calls():
parser = KimiK3DelegatingParser(
_dummy_tokenizer(), chat_template_kwargs={"thinking": False}
)
request = _request().model_copy(
update={
"tool_choice": "none",
"chat_template_kwargs": {"thinking": False},
}
)
messages: list[DeltaMessage] = []
chunks = [
OPEN,
"response",
f"{SEP}answer",
RESPONSE_CLOSE,
_tools(_call("calc", 1, _arg("x", "number", "1"))),
]
for index, chunk in enumerate(chunks, start=1):
delta = parser.parse_delta(
delta_text=chunk,
delta_token_ids=[index],
request=request,
prompt_token_ids=[1],
finished=index == len(chunks),
)
if delta is not None:
messages.append(delta)
content = "".join(message.content or "" for message in messages)
assert content == "answer"
assert OPEN not in content
assert CLOSE not in content
assert SEP not in content
assert all(not message.tool_calls for message in messages)
def test_adjust_request_keeps_xtml_markers_contiguous():
parser = KimiK3ToolParser(_dummy_tokenizer())
request = _request()
adjusted = parser.adjust_request(request)
assert adjusted.skip_special_tokens is False
if hasattr(adjusted, "spaces_between_special_tokens"):
assert adjusted.spaces_between_special_tokens is False
assert KimiK3ToolParser.supports_required_and_named is False
def test_adjust_request_required_uses_xtml_parser_not_json_guidance():
parser = KimiK3ToolParser(_dummy_tokenizer())
request = _request().model_copy(update={"tool_choice": "required"})
adjusted = parser.adjust_request(request)
assert adjusted.structured_outputs is None
assert adjusted.skip_special_tokens is False
if hasattr(adjusted, "spaces_between_special_tokens"):
assert adjusted.spaces_between_special_tokens is False
@pytest.mark.parametrize(
"tool_request",
[
_named_request(),
_responses_request(
tool_choice={"type": "function", "name": "calc"},
),
],
)
def test_adjust_request_rejects_named_tool_choice(tool_request):
parser = KimiK3ToolParser(_dummy_tokenizer())
with pytest.raises(VLLMValidationError) as exc_info:
parser.adjust_request(tool_request)
assert exc_info.value.parameter == "tool_choice"
assert "requires strict tool calling" in str(exc_info.value)
def test_responses_chat_params_carries_tool_choice_metadata():
request = _responses_request(tool_choice="required")
chat_params = request.build_chat_params(
default_template=None,
default_template_content_format="auto",
)
assert chat_params.tool_choice == "required"
def test_responses_chat_params_keeps_template_tool_choice_when_api_auto():
request = _responses_request().model_copy(
update={"chat_template_kwargs": {"tool_choice": "required"}}
)
chat_params = request.build_chat_params(
default_template=None,
default_template_content_format="auto",
)
assert chat_params.chat_template_kwargs["tool_choice"] == "required"
assert chat_params.tool_choice == "auto"
def test_responses_required_tool_choice_uses_xtml_parser():
parser = KimiK3DelegatingParser(
_dummy_tokenizer(), chat_template_kwargs={"thinking": False}
)
request = _responses_request(tool_choice="required").model_copy(
update={"chat_template_kwargs": {"thinking": False}}
)
output = RESPONSE_CLOSE + _tools(_call("calc", 1, _arg("x", "number", "1")))
reasoning, content, tool_calls = parser.parse(
output, request, enable_auto_tools=True, model_output_token_ids=[]
)
response_outputs = build_response_output_items(
reasoning=reasoning,
content=content,
tool_calls=tool_calls,
tools=request.tools,
)
assert len(response_outputs) == 1
tool_call = response_outputs[0]
assert tool_call.type == "function_call"
assert tool_call.name == "calc"
assert json.loads(tool_call.arguments) == {"x": 1}
def test_responses_named_tool_choice_uses_xtml_parser():
parser = KimiK3DelegatingParser(
_dummy_tokenizer(), chat_template_kwargs={"thinking": False}
)
request = _responses_request(
tool_choice={"type": "function", "name": "calc"}
).model_copy(update={"chat_template_kwargs": {"thinking": False}})
output = RESPONSE_CLOSE + _tools(_call("calc", 1, _arg("x", "number", "1")))
reasoning, content, tool_calls = parser.parse(
output, request, enable_auto_tools=True, model_output_token_ids=[]
)
response_outputs = build_response_output_items(
reasoning=reasoning,
content=content,
tool_calls=tool_calls,
tools=request.tools,
)
assert len(response_outputs) == 1
tool_call = response_outputs[0]
assert tool_call.type == "function_call"
assert tool_call.name == "calc"
assert json.loads(tool_call.arguments) == {"x": 1}
def test_chat_params_carries_tool_choice_metadata():
request = _request().model_copy(update={"tool_choice": "required"})
chat_params = request.build_chat_params(
default_template=None,
default_template_content_format="auto",
)
assert chat_params.tool_choice == "required"
def test_chat_params_carries_response_format_metadata():
request = ChatCompletionRequest(
model="test-model",
messages=[],
response_format={"type": "json_object"},
)
chat_params = request.build_chat_params(
default_template=None,
default_template_content_format="auto",
)
assert chat_params.response_format is request.response_format
assert chat_params.tool_choice is None
def test_chat_params_keeps_template_tool_choice_when_api_auto():
request = _request().model_copy(
update={
"tool_choice": "auto",
"chat_template_kwargs": {"tool_choice": "required"},
}
)
chat_params = request.build_chat_params(
default_template=None,
default_template_content_format="auto",
)
assert chat_params.chat_template_kwargs["tool_choice"] == "required"
assert chat_params.tool_choice == "auto"
-415
View File
@@ -1,415 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Unit tests for the MuseGlimmer ATEM tool parser and reasoning parser.
MuseGlimmer writes every turn as a sequence of channel-scoped messages rather
than JSON, so the two parsers are tested together: the reasoning parser strips
the reasoning span and forwards the remaining channels as content, and the tool
parser reads ATEM markup out of those channels.
Four areas, in order:
1. non-streaming tool-call extraction, including channel scoping (an
``<atem:invoke>`` echoed inside reasoning must never become a call);
2. the reasoning -> tool-parser handoff, which regressed once by returning
``content=None`` and starving the tool parser;
3. streaming, where markers routinely straddle chunk boundaries, plus
truncation isolation for an unterminated ``to=self`` block;
4. tool-name normalization against the tools registered on the request.
These drive the parsers directly and need no checkpoint. The tests that require
a real tokenizer live in ``test_muse_glimmer_parse_delta.py``.
"""
import json
from types import SimpleNamespace
from typing import cast
import pytest
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
from vllm.reasoning.muse_glimmer_reasoning_parser import MuseGlimmerReasoningParser
from vllm.tool_parsers.muse_glimmer_tool_parser import MuseGlimmerToolParser
R: MuseGlimmerReasoningParser
T: MuseGlimmerToolParser
@pytest.fixture(autouse=True)
def _fresh_parsers():
"""Give each test request-scoped parser state through the real constructors."""
global R, T
R = MuseGlimmerReasoningParser(object())
T = MuseGlimmerToolParser(object())
# Any framing token that must NEVER appear in surfaced reasoning/content.
_FRAMING = [
"<|start|>",
"<|message|>",
"<|eom|>",
"<|eot|>",
"to=self",
"to=user",
"to=read.read",
"<atem:",
]
class _FakeReq:
"""Minimal ChatCompletionRequest stand-in (no registered tools)."""
tools = None
def _fake_req() -> ChatCompletionRequest:
return cast(ChatCompletionRequest, _FakeReq())
# The parsers accept a missing request at runtime.
_NO_REQ = cast(ChatCompletionRequest, None)
def _req(*names: str) -> ChatCompletionRequest:
"""A request with ``names`` registered as tools."""
return cast(
ChatCompletionRequest,
SimpleNamespace(
tools=[SimpleNamespace(function=SimpleNamespace(name=n)) for n in names]
),
)
def _call(name):
"""One tool-call turn invoking ``name``."""
return (
f"<|start|>assistant to={name}<|message|>"
f'<atem:function_calls>\n<atem:invoke name="{name}">\n'
f'<atem:parameter name="city">Paris</atem:parameter>\n'
f"</atem:invoke>\n</atem:function_calls>"
)
# ---------------------------------------------------------------- tool calls
def test_single_tool_call_after_reasoning():
raw = (
"to=self<|message|>Let me check the weather.<|eom|>"
"<|start|>assistant to=weather.get<|message|>"
'<atem:function_calls>\n<atem:invoke name="weather.get">\n'
'<atem:parameter name="city">Paris</atem:parameter>\n'
'<atem:parameter name="units">celsius</atem:parameter>\n'
"</atem:invoke>\n</atem:function_calls><|eot|>"
)
out = MuseGlimmerToolParser.extract_tool_calls(T, raw, _NO_REQ)
assert out.tools_called and len(out.tool_calls) == 1
assert out.tool_calls[0].function.name == "weather.get"
assert json.loads(out.tool_calls[0].function.arguments) == {
"city": "Paris",
"units": "celsius",
}
def test_parallel_calls_across_eom_boundaries():
raw = (
"<|start|>assistant to=math.add<|message|>"
'<atem:function_calls>\n<atem:invoke name="math.add">\n'
'<atem:parameter name="a">1</atem:parameter>\n'
'<atem:parameter name="b">2</atem:parameter>\n'
"</atem:invoke>\n</atem:function_calls><|eom|>"
"<|start|>assistant to=math.mul<|message|>"
'<atem:function_calls>\n<atem:invoke name="math.mul">\n'
'<atem:parameter name="a">3</atem:parameter>\n'
'<atem:parameter name="b">4</atem:parameter>\n'
"</atem:invoke>\n</atem:function_calls><|eot|>"
)
out = MuseGlimmerToolParser.extract_tool_calls(T, raw, _NO_REQ)
assert out.tools_called and len(out.tool_calls) == 2, len(out.tool_calls)
assert [t.function.name for t in out.tool_calls] == ["math.add", "math.mul"]
# JSON-typed values decode to ints
assert json.loads(out.tool_calls[0].function.arguments) == {"a": 1, "b": 2}
def test_echoed_invoke_in_reasoning_is_not_parsed():
"""Channel scoping: an invoke quoted inside reasoning is not a call."""
raw = (
'to=self<|message|>I could call <atem:invoke name="evil.fn">'
'<atem:parameter name="x">1</atem:parameter></atem:invoke> '
"but I will not.<|eom|>"
"<|start|>assistant to=user<|message|>The answer is 42.<|eot|>"
)
out = MuseGlimmerToolParser.extract_tool_calls(T, raw, _NO_REQ)
assert not out.tools_called, "channel scoping failed -- echoed invoke parsed!"
assert out.content == "The answer is 42.", repr(out.content)
def test_plain_answer_yields_no_tool_calls():
out = MuseGlimmerToolParser.extract_tool_calls(
T, "to=user<|message|>Just a plain answer.<|eot|>", _NO_REQ
)
assert not out.tools_called
def test_json_object_array_and_bool_params_decode():
raw = (
"<|start|>assistant to=api.call<|message|>"
'<atem:function_calls>\n<atem:invoke name="api.call">\n'
'<atem:parameter name="payload">{"nested": [1, 2, 3]}</atem:parameter>\n'
'<atem:parameter name="flag">true</atem:parameter>\n'
"</atem:invoke>\n</atem:function_calls><|eot|>"
)
out = MuseGlimmerToolParser.extract_tool_calls(T, raw, _NO_REQ)
assert json.loads(out.tool_calls[0].function.arguments) == {
"payload": {"nested": [1, 2, 3]},
"flag": True,
}
# ------------------------------------------------- reasoning -> tool handoff
def test_reasoning_to_toolcall_handoff():
"""The regression: content=None here starved the tool parser."""
raw = (
" to=self<|message|>Let me call the tool.<|eom|>"
"<|start|>assistant to=weather.get<|message|>"
'<atem:function_calls>\n<atem:invoke name="weather.get">\n'
'<atem:parameter name="city">Paris</atem:parameter>\n'
"</atem:invoke>\n</atem:function_calls>"
)
reasoning, content = MuseGlimmerReasoningParser.extract_reasoning(R, raw, _NO_REQ)
assert reasoning == "Let me call the tool.", repr(reasoning)
assert content is not None and "<atem:invoke" in content, repr(content)
out = MuseGlimmerToolParser.extract_tool_calls(T, content, _NO_REQ)
assert out.tools_called and len(out.tool_calls) == 1
assert out.tool_calls[0].function.name == "weather.get"
def test_reasoning_then_user_answer():
raw = (
" to=self<|message|>thinking<|eom|>"
"<|start|>assistant to=user<|message|>The answer is 42.<|eot|>"
)
reasoning, content = MuseGlimmerReasoningParser.extract_reasoning(R, raw, _NO_REQ)
assert reasoning == "thinking", repr(reasoning)
assert content == "The answer is 42.", repr(content)
assert not MuseGlimmerToolParser.extract_tool_calls(
T, content, _NO_REQ
).tools_called
def test_plain_content_without_framing_passes_through():
reasoning, content = MuseGlimmerReasoningParser.extract_reasoning(
R, "Just a direct answer.", _NO_REQ
)
assert reasoning is None and content == "Just a direct answer.", (
reasoning,
content,
)
def test_reasoning_then_parallel_calls():
raw = (
" to=self<|message|>need two calls<|eom|>"
"<|start|>assistant to=math.add<|message|>"
'<atem:function_calls>\n<atem:invoke name="math.add">\n'
'<atem:parameter name="a">1</atem:parameter>\n</atem:invoke>\n'
"</atem:function_calls><|eom|>"
"<|start|>assistant to=math.mul<|message|>"
'<atem:function_calls>\n<atem:invoke name="math.mul">\n'
'<atem:parameter name="a">3</atem:parameter>\n</atem:invoke>\n'
"</atem:function_calls><|eot|>"
)
reasoning, content = MuseGlimmerReasoningParser.extract_reasoning(R, raw, _NO_REQ)
assert reasoning == "need two calls", repr(reasoning)
assert content is not None
out = MuseGlimmerToolParser.extract_tool_calls(T, content, _NO_REQ)
assert [t.function.name for t in out.tool_calls] == ["math.add", "math.mul"], (
out.tool_calls
)
# ----------------------------------------------------------------- streaming
def _stream(raw: str, chunk: int):
"""Feed ``raw`` incrementally in ``chunk``-char steps through BOTH streaming
parsers; return (reasoning, content, tool_calls)."""
reasoning, content, toolcalls = [], [], []
prev = ""
i = 0
while i < len(raw):
cur = raw[: i + chunk]
delta = cur[len(prev) :]
dm = MuseGlimmerReasoningParser.extract_reasoning_streaming(
R, prev, cur, delta, [], [], []
)
if dm is not None:
if dm.reasoning:
reasoning.append(dm.reasoning)
content_delta = getattr(dm, "content", None)
# Tool-channel content is an internal handoff to the tool parser,
# not client-visible content from the unified parser.
if content_delta and "<atem:function_calls>" not in content_delta:
content.append(content_delta)
dt = MuseGlimmerToolParser.extract_tool_calls_streaming(
T, prev, cur, delta, [], [], [], _fake_req()
)
if dt is not None and dt.tool_calls:
toolcalls.extend(dt.tool_calls)
prev = cur
i += chunk
return "".join(reasoning), "".join(content), toolcalls
def _fn_of(tc):
fn = tc.function
if isinstance(fn, dict):
return fn.get("name"), fn.get("arguments")
return fn.name, fn.arguments
RAW_TOOLCALL = (
" to=self<|message|>I should read the hostname file to answer.<|eom|>"
"<|start|>assistant to=read.read<|message|>"
'<atem:function_calls>\n<atem:invoke name="read.read">\n'
'<atem:parameter name="path">/etc/hostname</atem:parameter>\n'
"</atem:invoke>\n</atem:function_calls>"
)
RAW_ANSWER = (
" to=self<|message|>Think about it.<|eom|>"
"<|start|>assistant to=user<|message|>The answer is 42.<|eot|>"
)
# NO closing <|eom|> -> truncated CoT
RAW_TRUNCATED = (
" to=self<|message|>Maybe I should call "
'<atem:function_calls>\n<atem:invoke name="read.read">\n'
'<atem:parameter name="path">/etc/hostname</atem:parameter>\n'
"</atem:invoke>\n</atem:function_calls> but wait"
)
def _check_toolcall_stream(chunk):
reasoning, content, tcs = _stream(RAW_TOOLCALL, chunk)
# (a) no framing token leaks into reasoning or content
for f in _FRAMING:
assert f not in reasoning, (
f"framing {f!r} leaked into reasoning (chunk={chunk})"
)
assert f not in content, f"framing {f!r} leaked into content (chunk={chunk})"
# (b) exactly one tool_call with correct name + args
assert len(tcs) == 1, f"expected 1 tool_call, got {len(tcs)} (chunk={chunk})"
name, args = _fn_of(tcs[0])
assert name == "read.read", name
assert json.loads(args) == {"path": "/etc/hostname"}, args
assert tcs[0].index == 0 and tcs[0].type == "function" and tcs[0].id
# (c) reasoning captured separately and clean
assert reasoning == "I should read the hostname file to answer.", repr(reasoning)
assert content == "", repr(content)
def test_streaming_toolcall_chunk3():
_check_toolcall_stream(3)
def test_streaming_toolcall_charwise():
# worst case: markers arrive one char at a time (mid-marker deltas)
_check_toolcall_stream(1)
def test_streaming_toolcall_bigchunks():
_check_toolcall_stream(17)
def test_streaming_reasoning_then_content():
reasoning, content, tcs = _stream(RAW_ANSWER, 3)
for f in _FRAMING:
assert f not in reasoning and f not in content, f
assert reasoning == "Think about it.", repr(reasoning)
assert content == "The answer is 42.", repr(content)
assert tcs == []
def test_truncated_cot_no_toolcall_nonstreaming():
out = MuseGlimmerToolParser.extract_tool_calls(T, RAW_TRUNCATED, _fake_req())
assert not out.tools_called and out.tool_calls == []
# partial reasoning must still be recovered by the reasoning parser
reasoning, _ = MuseGlimmerReasoningParser.extract_reasoning(
R, RAW_TRUNCATED, _fake_req()
)
assert reasoning and "Maybe I should call" in reasoning, repr(reasoning)
def test_truncated_cot_no_toolcall_streaming():
_, _, tcs = _stream(RAW_TRUNCATED, 3)
assert tcs == [], f"truncated CoT invoke leaked as streaming tool call: {tcs}"
# ------------------------------------------------------- name normalization
#
# MuseGlimmer emits `get_weather.get_weather` for a bare-registered
# `get_weather`, and `weather.get` verbatim for a namespaced one. The parser
# normalizes against the tools actually registered on the request.
def test_doubled_bare_name_collapses():
out = MuseGlimmerToolParser.extract_tool_calls(
T, _call("get_weather.get_weather"), _req("get_weather")
)
assert out.tools_called and out.tool_calls[0].function.name == "get_weather", (
out.tool_calls[0].function.name
)
def test_namespaced_name_preserved():
out = MuseGlimmerToolParser.extract_tool_calls(
T, _call("weather.get"), _req("weather.get")
)
assert out.tool_calls[0].function.name == "weather.get"
def test_unregistered_namespace_is_preserved():
# Suffix-only matching can silently dispatch a tool from the wrong namespace.
out = MuseGlimmerToolParser.extract_tool_calls(
T, _call("foo.get_weather"), _req("get_weather")
)
assert out.tool_calls[0].function.name == "foo.get_weather"
def test_trailing_segment_ambiguous_left_alone():
# two registered tools share leaf 'get' -> ambiguous -> do NOT rewrite
out = MuseGlimmerToolParser.extract_tool_calls(
T, _call("x.get"), _req("weather.get", "time.get")
)
assert out.tool_calls[0].function.name == "x.get"
def test_no_registered_tools_passthrough():
out = MuseGlimmerToolParser.extract_tool_calls(
T, _call("get_weather.get_weather"), _NO_REQ
)
assert out.tool_calls[0].function.name == "get_weather.get_weather"
def test_exact_match_kept():
out = MuseGlimmerToolParser.extract_tool_calls(
T, _call("get_weather"), _req("get_weather")
)
assert out.tool_calls[0].function.name == "get_weather"
def test_named_custom_tool_choice_skips_structured_outputs():
# Custom tools ride the function shim; a named custom choice must skip
# the base adjust_request like a named function choice does.
request = ResponsesRequest(
input="hi",
tools=[{"type": "custom", "name": "emit_command"}],
tool_choice={"type": "custom", "name": "emit_command"},
)
T.adjust_request(request)
assert request.structured_outputs is None
@@ -1,187 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""parse_delta-level regression tests for the MuseGlimmer parsers.
These drive the UNIFIED parser-engine streaming API the serving layer actually
uses (``DelegatingParser.parse_delta``), NOT the ``extract_*_streaming`` methods
directly. This is the coverage that was missing when a live no-tools streaming
request leaked raw harmony framing into ``content``: the phase machine marked
``reasoning_ended=True`` at the prompt boundary (because ``is_reasoning_end`` on
a prompt with no ``to=self`` wrongly returned True), skipping the reasoning
phase for the whole generation.
Requires a real MuseGlimmer tokenizer; skipped if the checkpoint is unavailable.
"""
import json
import os
import pytest
CKPT = os.environ.get("MUSE_GLIMMER_CKPT", "")
pytestmark = pytest.mark.skipif(
not os.path.isdir(CKPT), reason=f"MuseGlimmer checkpoint not found at {CKPT}"
)
_FRAMING = [
"<|start|>",
"<|message|>",
"<|eom|>",
"<|eot|>",
"to=self",
"to=user",
"<atem:",
]
PROMPT = "<|start|>user<|message|>hi<|eom|><|start|>assistant"
class _Req:
tools = None
tool_choice = None
include_reasoning = True
@pytest.fixture(scope="module")
def tok():
from transformers import AutoTokenizer
return AutoTokenizer.from_pretrained(CKPT, trust_remote_code=True)
@pytest.fixture(scope="module")
def parser_cls():
from vllm.parser import ParserManager
return ParserManager.get_parser(
tool_parser_name="muse_glimmer",
reasoning_parser_name="muse_glimmer",
enable_auto_tools=True,
)
def _drive(parser_cls, tok, gen_text, req=None):
"""Feed gen_text token-by-token through parse_delta (as serving does)."""
from vllm.parser.abstract_parser import StreamState
if req is None:
req = _Req()
parser = parser_cls(tok)
parser._stream_state = StreamState()
prompt_ids = tok.encode(PROMPT, add_special_tokens=False)
gen_ids = tok.encode(gen_text, add_special_tokens=False)
reasoning, content, tools = [], [], []
for i, tid in enumerate(gen_ids):
dm = parser.parse_delta(
tok.decode([tid]),
[tid],
req,
prompt_token_ids=prompt_ids if i == 0 else None,
finished=(i == len(gen_ids) - 1),
)
if dm is None:
continue
if getattr(dm, "reasoning", None):
reasoning.append(dm.reasoning)
if getattr(dm, "content", None):
content.append(dm.content)
for tc in getattr(dm, "tool_calls", None) or []:
fn = tc.function
name = fn.get("name") if isinstance(fn, dict) else getattr(fn, "name", None)
args = (
fn.get("arguments")
if isinstance(fn, dict)
else getattr(fn, "arguments", None)
)
tools.append((tc.index, name, args))
return "".join(reasoning), "".join(content), tools
def _assert_no_framing(text):
for f in _FRAMING:
assert f not in text, f"framing {f!r} leaked: {text!r}"
def test_parse_delta_no_tools_reasoning_then_answer(parser_cls, tok):
# The core regression: reasoning phase must stay active for a no-tools turn.
reasoning, content, tools = _drive(
parser_cls,
tok,
" to=self<|message|>Let me think step by step about the sum.<|eom|>"
"<|start|>assistant to=user<|message|>The answer is 42.<|eot|>",
)
_assert_no_framing(content)
assert reasoning == "Let me think step by step about the sum.", repr(reasoning)
assert content == "The answer is 42.", repr(content)
assert tools == []
def test_parse_delta_content_only(parser_cls, tok):
reasoning, content, tools = _drive(
parser_cls,
tok,
" to=user<|message|>Just a direct answer.<|eot|>",
)
_assert_no_framing(content)
assert content == "Just a direct answer.", repr(content)
assert tools == []
def test_parse_delta_tool_call(parser_cls, tok):
reasoning, content, tools = _drive(
parser_cls,
tok,
" to=self<|message|>I should read the hostname.<|eom|>"
"<|start|>assistant to=read.read<|message|>"
'<atem:function_calls>\n<atem:invoke name="read.read">\n'
'<atem:parameter name="path">/etc/hostname</atem:parameter>\n'
"</atem:invoke>\n</atem:function_calls>",
)
_assert_no_framing(content)
assert reasoning == "I should read the hostname.", repr(reasoning)
assert len(tools) == 1, tools
idx, name, args = tools[0]
assert idx == 0 and name == "read.read"
assert json.loads(args) == {"path": "/etc/hostname"}
def test_parse_delta_truncated_cot_no_toolcall(parser_cls, tok):
# Contemplated invoke inside an unterminated to=self block: no tool call,
# no framing leak into content, partial reasoning recovered.
reasoning, content, tools = _drive(
parser_cls,
tok,
" to=self<|message|>Maybe I should call "
'<atem:function_calls>\n<atem:invoke name="read.read">\n'
'<atem:parameter name="path">/etc/hostname</atem:parameter>\n'
"</atem:invoke>\n</atem:function_calls> but wait",
)
_assert_no_framing(content)
assert tools == [], f"contemplated invoke leaked as tool call: {tools}"
assert "Maybe I should call" in reasoning, repr(reasoning)
def test_parse_delta_reasoning_suppressed_when_not_requested(parser_cls, tok):
class _NoReasonReq:
tools = None
tool_choice = None
include_reasoning = False
reasoning, content, tools = _drive(
parser_cls,
tok,
" to=self<|message|>secret thoughts<|eom|>"
"<|start|>assistant to=user<|message|>Public answer.<|eot|>",
req=_NoReasonReq(),
)
assert reasoning == "", repr(reasoning) # suppressed
assert content == "Public answer.", repr(content)
_assert_no_framing(content)
if __name__ == "__main__":
import sys
sys.exit(pytest.main([__file__, "-v"]))