[Mini-vLLM] Cut dead subsystems: models, entrypoints, backends, utils, tests
- Drop dead model modules, keeping files live imports require - Remove anthropic, cohere, codegen entrypoints and batch STT modes - Delete unused attention backends, MoE experts, and NPU/XPU util leaves - Strip dead third_party trees and ROCm leaves with no CUDA path - Remove mooncake EC connectors: dependency missing since branch base - Re-audit verified by full import sweep and Qwen3.5 smoke test Co-authored-by: Hinata <hinata@idihore.id> Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
@@ -1,260 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Regression tests for Responses API tool-calling request adjustment.
|
||||
|
||||
Covers two bugs on the ``/v1/responses`` path that broke streaming tool
|
||||
calling for parsers relying on special-token delimiters (Gemma4):
|
||||
|
||||
1. :class:`Gemma4ToolParser.adjust_request` used an
|
||||
``isinstance(request, ChatCompletionRequest)`` guard, so a
|
||||
:class:`ResponsesRequest` with tools never had
|
||||
``skip_special_tokens`` flipped to ``False``. The default (``True``)
|
||||
stripped ``<|tool_call>`` / ``<tool_call|>`` delimiters, causing
|
||||
:meth:`Gemma4ToolParser.extract_tool_calls_streaming` to fall through
|
||||
to the content branch and leak the raw ``call:fn{...}`` body via
|
||||
``response.output_text.delta``.
|
||||
|
||||
2. :meth:`ToolParser.adjust_request` built
|
||||
:class:`ResponseTextConfig` in two steps (bare constructor then
|
||||
``.format = ...``). Under Pydantic v2 the later assignment is not
|
||||
tracked in ``__fields_set__``, which can drop the nested config from
|
||||
``model_dump``. It also passed a ``description`` kwarg carrying the
|
||||
wrong-purpose string ``"Response format for tool calling"``.
|
||||
|
||||
3. :class:`Gemma4EngineToolParser` (the engine-based parser, #45588) sets
|
||||
``supports_required_and_named=False`` but did not skip the forced
|
||||
``structured_outputs`` JSON for ``required``/named tool choice. The model
|
||||
was constrained to JSON the native parser cannot read, so the call leaked
|
||||
as content with empty ``tool_calls``. ``adjust_request`` now skips that
|
||||
constraint so Gemma4 emits its native ``<|tool_call>`` syntax.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
from openai.types.responses.tool_param import FunctionToolParam
|
||||
|
||||
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
|
||||
from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
|
||||
from vllm.tool_parsers.abstract_tool_parser import ToolParser
|
||||
from vllm.tool_parsers.gemma4_engine_tool_parser import (
|
||||
Gemma4EngineToolParser as Gemma4ToolParser,
|
||||
)
|
||||
|
||||
|
||||
def _get_weather_tool() -> FunctionToolParam:
|
||||
return FunctionToolParam(
|
||||
type="function",
|
||||
name="get_weather",
|
||||
description="Get current weather for a city",
|
||||
parameters={
|
||||
"type": "object",
|
||||
"properties": {"city": {"type": "string"}},
|
||||
"required": ["city"],
|
||||
},
|
||||
strict=True,
|
||||
)
|
||||
|
||||
|
||||
def _build_responses_request(*, tool_choice: str | dict[str, Any]) -> ResponsesRequest:
|
||||
return ResponsesRequest(
|
||||
model="gemma4-test",
|
||||
input=[{"role": "user", "content": "What is the weather in Hanoi?"}],
|
||||
tools=[_get_weather_tool()],
|
||||
tool_choice=tool_choice,
|
||||
stream=True,
|
||||
max_output_tokens=200,
|
||||
)
|
||||
|
||||
|
||||
def _build_chat_request(
|
||||
*,
|
||||
tool_choice: str | dict[str, Any],
|
||||
chat_template_kwargs: dict[str, Any] | None = None,
|
||||
) -> ChatCompletionRequest:
|
||||
data: dict[str, Any] = {
|
||||
"model": "gemma4-test",
|
||||
"messages": [{"role": "user", "content": "What is the weather in Hanoi?"}],
|
||||
"tools": [
|
||||
{
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "get_weather",
|
||||
"description": "Get current weather for a city",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {"city": {"type": "string"}},
|
||||
"required": ["city"],
|
||||
},
|
||||
},
|
||||
}
|
||||
],
|
||||
"tool_choice": tool_choice,
|
||||
}
|
||||
if chat_template_kwargs is not None:
|
||||
data["chat_template_kwargs"] = chat_template_kwargs
|
||||
return ChatCompletionRequest.model_validate(data)
|
||||
|
||||
|
||||
class _StubTokenizer:
|
||||
"""Minimal tokenizer stub to satisfy ``Gemma4EngineToolParser.__init__``."""
|
||||
|
||||
_VOCAB: dict[str, int] = {
|
||||
"<|tool_call>": 256_000,
|
||||
"<tool_call|>": 256_001,
|
||||
'<|"|>': 52,
|
||||
"<|channel>": 256_002,
|
||||
"<channel|>": 256_003,
|
||||
}
|
||||
|
||||
def get_vocab(self) -> dict[str, int]:
|
||||
return dict(self._VOCAB)
|
||||
|
||||
@property
|
||||
def all_special_tokens(self) -> list[str]:
|
||||
return list(self._VOCAB.keys())
|
||||
|
||||
@property
|
||||
def all_special_ids(self) -> list[int]:
|
||||
return list(self._VOCAB.values())
|
||||
|
||||
|
||||
def test_gemma4_adjust_request_sets_skip_special_tokens_on_responses() -> None:
|
||||
"""``Gemma4ToolParser.adjust_request`` must flip
|
||||
``skip_special_tokens=False`` for both ``ChatCompletionRequest`` and
|
||||
``ResponsesRequest`` so that ``<|tool_call>`` delimiters reach the
|
||||
streaming extractor. The previous
|
||||
``isinstance(ChatCompletionRequest)`` guard omitted the Responses
|
||||
path, causing raw ``call:fn{...}`` text to leak via
|
||||
``response.output_text.delta``.
|
||||
"""
|
||||
parser = Gemma4ToolParser(_StubTokenizer())
|
||||
|
||||
request = _build_responses_request(tool_choice="auto")
|
||||
assert request.skip_special_tokens is True, (
|
||||
"Precondition: ResponsesRequest.skip_special_tokens default is True"
|
||||
)
|
||||
|
||||
parser.adjust_request(request)
|
||||
|
||||
assert request.skip_special_tokens is False
|
||||
|
||||
|
||||
def test_tool_parser_adjust_request_builds_valid_response_text_config() -> None:
|
||||
"""``ToolParser.adjust_request`` must produce a ``ResponseTextConfig``
|
||||
whose dumped form contains the JSON schema under the ``schema`` alias
|
||||
and does not leak the unrelated ``"Response format for tool calling"``
|
||||
description string that the previous two-step construction injected.
|
||||
"""
|
||||
parser = ToolParser.__new__(ToolParser)
|
||||
parser.model_tokenizer = None
|
||||
|
||||
request = _build_responses_request(tool_choice="required")
|
||||
ToolParser.adjust_request(parser, request)
|
||||
|
||||
assert request.text is not None
|
||||
assert request.text.format is not None
|
||||
assert request.text.format.type == "json_schema"
|
||||
|
||||
dump: dict[str, Any] = request.text.model_dump(mode="json", by_alias=True)
|
||||
fmt = dump.get("format") or {}
|
||||
assert fmt.get("type") == "json_schema"
|
||||
assert fmt.get("name") == "tool_calling_response"
|
||||
assert fmt.get("strict") is True
|
||||
# Nested config must be present under the alias. Two-step Pydantic v2
|
||||
# construction could drop it from __fields_set__.
|
||||
assert "schema" in fmt and isinstance(fmt["schema"], dict)
|
||||
# The old code passed a wrong-purpose string; valid field should now
|
||||
# either be absent or None (the openai-python default).
|
||||
assert fmt.get("description") in (None, "")
|
||||
|
||||
|
||||
def test_gemma4_required_skips_structured_outputs_chatcompletion() -> None:
|
||||
"""Required + ChatCompletion: ``Gemma4EngineToolParser`` must skip the
|
||||
forced JSON ``structured_outputs`` so the model emits its native
|
||||
``<|tool_call>`` syntax. The base parser constrained output to JSON the
|
||||
native parser cannot read, leaking it as content with empty
|
||||
``tool_calls`` (regression after #45588).
|
||||
"""
|
||||
parser = Gemma4ToolParser(_StubTokenizer())
|
||||
request = _build_chat_request(tool_choice="required")
|
||||
|
||||
parser.adjust_request(request)
|
||||
|
||||
assert request.structured_outputs is None
|
||||
assert request.skip_special_tokens is False
|
||||
|
||||
|
||||
def test_gemma4_named_skips_structured_outputs_chatcompletion() -> None:
|
||||
"""Named + ChatCompletion: the forced single-function JSON schema must be
|
||||
skipped, same as ``required``.
|
||||
"""
|
||||
parser = Gemma4ToolParser(_StubTokenizer())
|
||||
request = _build_chat_request(
|
||||
tool_choice={"type": "function", "function": {"name": "get_weather"}}
|
||||
)
|
||||
|
||||
parser.adjust_request(request)
|
||||
|
||||
assert request.structured_outputs is None
|
||||
assert request.skip_special_tokens is False
|
||||
|
||||
|
||||
def test_gemma4_required_skips_structured_outputs_responses() -> None:
|
||||
"""Required + Responses: the forced JSON schema (``request.text``) must be
|
||||
skipped so the native delimiters reach the extractor.
|
||||
"""
|
||||
parser = Gemma4ToolParser(_StubTokenizer())
|
||||
request = _build_responses_request(tool_choice="required")
|
||||
|
||||
parser.adjust_request(request)
|
||||
|
||||
assert request.text is None
|
||||
assert request.skip_special_tokens is False
|
||||
|
||||
|
||||
def test_gemma4_named_skips_structured_outputs_responses() -> None:
|
||||
"""Named (``ToolChoiceFunction``) + Responses: the forced single-function
|
||||
JSON schema must be skipped.
|
||||
"""
|
||||
parser = Gemma4ToolParser(_StubTokenizer())
|
||||
request = _build_responses_request(
|
||||
tool_choice={"type": "function", "name": "get_weather"}
|
||||
)
|
||||
|
||||
parser.adjust_request(request)
|
||||
|
||||
assert request.text is None
|
||||
assert request.skip_special_tokens is False
|
||||
|
||||
|
||||
def test_gemma4_keeps_special_tokens_with_tools_thinking_disabled() -> None:
|
||||
"""Tools active + thinking disabled: ``skip_special_tokens`` must stay
|
||||
False so ``<|tool_call>`` delimiters reach the extractor. The merged
|
||||
enable_thinking early-return stripped them, breaking tool calling when
|
||||
thinking is off.
|
||||
"""
|
||||
parser = Gemma4ToolParser(_StubTokenizer())
|
||||
request = _build_chat_request(
|
||||
tool_choice="auto", chat_template_kwargs={"enable_thinking": False}
|
||||
)
|
||||
|
||||
parser.adjust_request(request)
|
||||
|
||||
assert request.skip_special_tokens is False
|
||||
|
||||
|
||||
def test_gemma4_keeps_skip_special_tokens_false_when_nothing_to_preserve() -> None:
|
||||
"""No active tools + thinking disabled: ``skip_special_tokens`` stays
|
||||
``False`` because the parser engine's ``__DROP__`` terminal mechanism
|
||||
strips unconfigured special tokens automatically.
|
||||
"""
|
||||
parser = Gemma4ToolParser(_StubTokenizer())
|
||||
request = _build_chat_request(
|
||||
tool_choice="none", chat_template_kwargs={"enable_thinking": False}
|
||||
)
|
||||
|
||||
parser.adjust_request(request)
|
||||
|
||||
assert request.skip_special_tokens is False
|
||||
@@ -1,613 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import json
|
||||
from typing import cast
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm.entrypoints.generate.base.protocol import DeltaMessage
|
||||
from vllm.entrypoints.openai.chat_completion.protocol import (
|
||||
ChatCompletionRequest,
|
||||
)
|
||||
from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
|
||||
from vllm.entrypoints.openai.responses.utils import build_response_output_items
|
||||
from vllm.exceptions import VLLMValidationError
|
||||
from vllm.parser.kimi_k3 import KimiK3Parser
|
||||
from vllm.parser.parser_manager import ParserManager
|
||||
from vllm.reasoning.kimi_k3_reasoning_parser import KimiK3ReasoningParser
|
||||
from vllm.tokenizers.protocol import TokenizerLike
|
||||
from vllm.tool_parsers.kimi_k3_tool_parser import KimiK3ToolParser
|
||||
|
||||
OPEN = "<|open|>"
|
||||
CLOSE = "<|close|>"
|
||||
SEP = "<|sep|>"
|
||||
THINK_OPEN = f"{OPEN}think{SEP}"
|
||||
THINK_CLOSE = f"{CLOSE}think{SEP}"
|
||||
RESPONSE_CLOSE = f"{CLOSE}response{SEP}"
|
||||
|
||||
|
||||
class DummyTokenizer:
|
||||
def get_vocab(self) -> dict[str, int]:
|
||||
return {}
|
||||
|
||||
def encode(self, text: str, add_special_tokens: bool = False) -> list[int]:
|
||||
if text == THINK_OPEN:
|
||||
return [1, 2, 3]
|
||||
if text == THINK_CLOSE:
|
||||
return [4, 2, 3]
|
||||
return [ord(ch) for ch in text]
|
||||
|
||||
|
||||
def _dummy_tokenizer() -> TokenizerLike:
|
||||
return cast(TokenizerLike, DummyTokenizer())
|
||||
|
||||
|
||||
class KimiK3DelegatingParser(KimiK3Parser):
|
||||
reasoning_parser_cls = KimiK3ReasoningParser
|
||||
tool_parser_cls = KimiK3ToolParser
|
||||
|
||||
|
||||
def test_parser_manager_selects_kimi_k3_parser():
|
||||
parser_cls = ParserManager.get_parser(
|
||||
tool_parser_name="kimi_k3",
|
||||
reasoning_parser_name="kimi_k3",
|
||||
enable_auto_tools=True,
|
||||
)
|
||||
|
||||
assert parser_cls is not None
|
||||
assert issubclass(parser_cls, KimiK3Parser)
|
||||
assert parser_cls.reasoning_parser_cls is KimiK3ReasoningParser
|
||||
assert parser_cls.tool_parser_cls is KimiK3ToolParser
|
||||
|
||||
|
||||
def _request() -> ChatCompletionRequest:
|
||||
return ChatCompletionRequest(
|
||||
model="test-model",
|
||||
messages=[],
|
||||
tools=[
|
||||
{
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "calc",
|
||||
"parameters": {"type": "object", "properties": {}},
|
||||
},
|
||||
}
|
||||
],
|
||||
tool_choice="auto",
|
||||
)
|
||||
|
||||
|
||||
def _named_request() -> ChatCompletionRequest:
|
||||
return ChatCompletionRequest(
|
||||
model="test-model",
|
||||
messages=[],
|
||||
tools=[
|
||||
{
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "calc",
|
||||
"parameters": {"type": "object", "properties": {}},
|
||||
},
|
||||
}
|
||||
],
|
||||
tool_choice={"type": "function", "function": {"name": "calc"}},
|
||||
)
|
||||
|
||||
|
||||
def _responses_request(*, tool_choice="auto") -> ResponsesRequest:
|
||||
return ResponsesRequest.model_validate(
|
||||
{
|
||||
"model": "test-model",
|
||||
"input": "Call the calc tool.",
|
||||
"tools": [
|
||||
{
|
||||
"type": "function",
|
||||
"name": "calc",
|
||||
"parameters": {"type": "object", "properties": {}},
|
||||
}
|
||||
],
|
||||
"tool_choice": tool_choice,
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def _arg(key: str, typ: str, value: str) -> str:
|
||||
return f'{OPEN}argument key="{key}" type="{typ}"{SEP}{value}{CLOSE}argument{SEP}'
|
||||
|
||||
|
||||
def _call(tool: str, index: int, *args: str) -> str:
|
||||
body = "".join(args)
|
||||
return f'{OPEN}call tool="{tool}" index="{index}"{SEP}{body}{CLOSE}call{SEP}'
|
||||
|
||||
|
||||
def _response(content: str) -> str:
|
||||
return f"{OPEN}response{SEP}{content}{RESPONSE_CLOSE}"
|
||||
|
||||
|
||||
def _tools(*calls: str) -> str:
|
||||
return f"{OPEN}tools{SEP}{''.join(calls)}{CLOSE}tools{SEP}"
|
||||
|
||||
|
||||
def test_extract_tool_calls_with_response_and_typed_arguments():
|
||||
parser = KimiK3ToolParser(_dummy_tokenizer())
|
||||
|
||||
output = _response("answer") + _tools(
|
||||
_call(
|
||||
"calc",
|
||||
1,
|
||||
_arg("x", "number", "1"),
|
||||
_arg("flag", "boolean", "true"),
|
||||
_arg("text", "string", "raw"),
|
||||
)
|
||||
)
|
||||
extracted = parser.extract_tool_calls(output, _request())
|
||||
|
||||
assert extracted.tools_called is True
|
||||
assert extracted.content == "answer"
|
||||
assert len(extracted.tool_calls) == 1
|
||||
tool_call = extracted.tool_calls[0]
|
||||
assert tool_call.function.name == "calc"
|
||||
assert json.loads(tool_call.function.arguments) == {
|
||||
"x": 1,
|
||||
"flag": True,
|
||||
"text": "raw",
|
||||
}
|
||||
|
||||
|
||||
def test_delegating_parser_preserves_tool_calls_after_reasoning():
|
||||
parser = KimiK3DelegatingParser(_dummy_tokenizer())
|
||||
output = (
|
||||
f"{THINK_OPEN}step{THINK_CLOSE}"
|
||||
+ _response("answer")
|
||||
+ _tools(_call("calc", 1, _arg("x", "number", "1")))
|
||||
)
|
||||
|
||||
reasoning, content, tool_calls = parser.parse(
|
||||
output,
|
||||
_request(),
|
||||
enable_auto_tools=True,
|
||||
)
|
||||
|
||||
assert reasoning == "step"
|
||||
assert content == "answer"
|
||||
assert tool_calls is not None
|
||||
assert len(tool_calls) == 1
|
||||
assert tool_calls[0].name == "calc"
|
||||
assert json.loads(tool_calls[0].arguments) == {"x": 1}
|
||||
|
||||
|
||||
def test_delegating_parser_required_tool_choice_uses_xtml_parser():
|
||||
parser = KimiK3DelegatingParser(_dummy_tokenizer())
|
||||
request = _request().model_copy(update={"tool_choice": "required"})
|
||||
output = (
|
||||
f"{THINK_OPEN}step{THINK_CLOSE}"
|
||||
+ _response("")
|
||||
+ _tools(_call("calc", 1, _arg("x", "number", "1")))
|
||||
)
|
||||
|
||||
reasoning, content, tool_calls = parser.parse(
|
||||
output,
|
||||
request,
|
||||
enable_auto_tools=True,
|
||||
)
|
||||
|
||||
assert reasoning == "step"
|
||||
assert content is None
|
||||
assert tool_calls is not None
|
||||
assert len(tool_calls) == 1
|
||||
assert tool_calls[0].name == "calc"
|
||||
assert json.loads(tool_calls[0].arguments) == {"x": 1}
|
||||
|
||||
|
||||
def test_delegating_parser_named_tool_choice_uses_xtml_parser():
|
||||
parser = KimiK3DelegatingParser(_dummy_tokenizer())
|
||||
output = (
|
||||
f"{THINK_OPEN}step{THINK_CLOSE}"
|
||||
+ _response("")
|
||||
+ _tools(_call("calc", 1, _arg("x", "number", "1")))
|
||||
)
|
||||
|
||||
reasoning, content, tool_calls = parser.parse(
|
||||
output,
|
||||
_named_request(),
|
||||
enable_auto_tools=True,
|
||||
)
|
||||
|
||||
assert reasoning == "step"
|
||||
assert content is None
|
||||
assert tool_calls is not None
|
||||
assert len(tool_calls) == 1
|
||||
assert tool_calls[0].name == "calc"
|
||||
assert json.loads(tool_calls[0].arguments) == {"x": 1}
|
||||
|
||||
|
||||
def test_delegating_parser_auto_no_call_strips_consumed_response_prefix():
|
||||
parser = KimiK3DelegatingParser(
|
||||
_dummy_tokenizer(), chat_template_kwargs={"thinking": False}
|
||||
)
|
||||
request = _request().model_copy(
|
||||
update={"chat_template_kwargs": {"thinking": False}}
|
||||
)
|
||||
|
||||
reasoning, content, tool_calls = parser.parse(
|
||||
f"answer{RESPONSE_CLOSE}",
|
||||
request,
|
||||
enable_auto_tools=True,
|
||||
)
|
||||
|
||||
assert reasoning is None
|
||||
assert content == "answer"
|
||||
assert tool_calls is None
|
||||
|
||||
|
||||
def test_delegating_parser_required_call_strips_consumed_response_prefix():
|
||||
parser = KimiK3DelegatingParser(
|
||||
_dummy_tokenizer(), chat_template_kwargs={"thinking": False}
|
||||
)
|
||||
request = _request().model_copy(
|
||||
update={
|
||||
"tool_choice": "required",
|
||||
"chat_template_kwargs": {"thinking": False},
|
||||
}
|
||||
)
|
||||
output = RESPONSE_CLOSE + _tools(_call("calc", 1, _arg("x", "number", "1")))
|
||||
|
||||
reasoning, content, tool_calls = parser.parse(
|
||||
output,
|
||||
request,
|
||||
enable_auto_tools=True,
|
||||
)
|
||||
|
||||
assert reasoning is None
|
||||
assert content is None
|
||||
assert tool_calls is not None
|
||||
assert len(tool_calls) == 1
|
||||
assert tool_calls[0].name == "calc"
|
||||
assert json.loads(tool_calls[0].arguments) == {"x": 1}
|
||||
|
||||
|
||||
def test_delegating_parser_truncated_tools_do_not_leak_xtml():
|
||||
parser = KimiK3DelegatingParser(
|
||||
_dummy_tokenizer(), chat_template_kwargs={"thinking": False}
|
||||
)
|
||||
request = _request().model_copy(
|
||||
update={
|
||||
"tool_choice": "required",
|
||||
"chat_template_kwargs": {"thinking": False},
|
||||
}
|
||||
)
|
||||
|
||||
reasoning, content, tool_calls = parser.parse(
|
||||
(f'{RESPONSE_CLOSE}{OPEN}tools{SEP}{OPEN}call tool="calc" index="1"'),
|
||||
request,
|
||||
enable_auto_tools=True,
|
||||
)
|
||||
|
||||
assert reasoning is None
|
||||
assert content is None
|
||||
assert tool_calls is None
|
||||
|
||||
|
||||
def test_extract_tool_calls_unescapes_attributes():
|
||||
parser = KimiK3ToolParser(_dummy_tokenizer())
|
||||
|
||||
output = _tools(_call("a&b"c", 1, _arg("k&q", "string", "v")))
|
||||
extracted = parser.extract_tool_calls(output, _request())
|
||||
|
||||
assert extracted.tools_called is True
|
||||
assert extracted.tool_calls[0].function.name == 'a&b"c'
|
||||
assert json.loads(extracted.tool_calls[0].function.arguments) == {"k&q": "v"}
|
||||
|
||||
|
||||
def test_extract_tool_calls_allows_less_than_in_attributes():
|
||||
parser = KimiK3ToolParser(_dummy_tokenizer())
|
||||
|
||||
output = _tools(_call("calc<beta", 1, _arg("foo<bar", "string", "raw")))
|
||||
extracted = parser.extract_tool_calls(output, _request())
|
||||
|
||||
assert extracted.tools_called is True
|
||||
assert extracted.tool_calls[0].function.name == "calc<beta"
|
||||
assert json.loads(extracted.tool_calls[0].function.arguments) == {"foo<bar": "raw"}
|
||||
|
||||
|
||||
def test_extract_content_from_whitespace_degraded_markers():
|
||||
parser = KimiK3ToolParser(_dummy_tokenizer())
|
||||
|
||||
extracted = parser.extract_tool_calls(
|
||||
f"{OPEN} response {SEP}answer{CLOSE} response {SEP}",
|
||||
_request(),
|
||||
)
|
||||
|
||||
assert extracted.tools_called is False
|
||||
assert extracted.content == "answer"
|
||||
|
||||
|
||||
def test_streaming_split_markers_do_not_leak():
|
||||
parser = KimiK3ToolParser(_dummy_tokenizer())
|
||||
request = _request()
|
||||
previous_text = ""
|
||||
previous_ids: list[int] = []
|
||||
messages: list[DeltaMessage] = []
|
||||
chunks = [
|
||||
OPEN,
|
||||
"response",
|
||||
f"{SEP}Hi",
|
||||
OPEN,
|
||||
"tools",
|
||||
SEP,
|
||||
f'{OPEN}call tool="calc" index="1"{SEP}',
|
||||
_arg("x", "number", "1"),
|
||||
f"{CLOSE}call",
|
||||
SEP,
|
||||
]
|
||||
|
||||
for i, chunk in enumerate(chunks, start=1):
|
||||
current_text = previous_text + chunk
|
||||
current_ids = previous_ids + [i]
|
||||
delta = parser.extract_tool_calls_streaming(
|
||||
previous_text=previous_text,
|
||||
current_text=current_text,
|
||||
delta_text=chunk,
|
||||
previous_token_ids=previous_ids,
|
||||
current_token_ids=current_ids,
|
||||
delta_token_ids=[i],
|
||||
request=request,
|
||||
)
|
||||
if delta is not None:
|
||||
messages.append(delta)
|
||||
previous_text = current_text
|
||||
previous_ids = current_ids
|
||||
|
||||
content = "".join(message.content or "" for message in messages)
|
||||
tool_deltas = [
|
||||
tool_call for message in messages for tool_call in (message.tool_calls or [])
|
||||
]
|
||||
|
||||
assert content == "Hi"
|
||||
assert OPEN not in content
|
||||
assert SEP not in content
|
||||
assert len(tool_deltas) == 1
|
||||
assert tool_deltas[0].function is not None
|
||||
assert tool_deltas[0].function.name == "calc"
|
||||
assert json.loads(tool_deltas[0].function.arguments) == {"x": 1}
|
||||
|
||||
|
||||
def test_tool_call_ids_are_unique_across_messages():
|
||||
output = _tools(_call("calc", 1))
|
||||
|
||||
first = KimiK3ToolParser(_dummy_tokenizer()).extract_tool_calls(output, _request())
|
||||
second = KimiK3ToolParser(_dummy_tokenizer()).extract_tool_calls(output, _request())
|
||||
|
||||
assert first.tool_calls[0].id != second.tool_calls[0].id
|
||||
|
||||
|
||||
def test_streaming_consumed_response_prefix_no_call_keeps_content():
|
||||
parser = KimiK3ToolParser(_dummy_tokenizer())
|
||||
request = _request()
|
||||
previous_text = ""
|
||||
previous_ids: list[int] = []
|
||||
messages: list[DeltaMessage] = []
|
||||
chunks = ["O", "K", CLOSE, f"response{SEP}"]
|
||||
|
||||
for i, chunk in enumerate(chunks, start=1):
|
||||
current_text = previous_text + chunk
|
||||
current_ids = previous_ids + [i]
|
||||
delta = parser.extract_tool_calls_streaming(
|
||||
previous_text=previous_text,
|
||||
current_text=current_text,
|
||||
delta_text=chunk,
|
||||
previous_token_ids=previous_ids,
|
||||
current_token_ids=current_ids,
|
||||
delta_token_ids=[i],
|
||||
request=request,
|
||||
)
|
||||
if delta is not None:
|
||||
messages.append(delta)
|
||||
previous_text = current_text
|
||||
previous_ids = current_ids
|
||||
|
||||
assert "".join(message.content or "" for message in messages) == "OK"
|
||||
assert all(CLOSE not in (message.content or "") for message in messages)
|
||||
|
||||
|
||||
def test_delegating_parser_tool_choice_none_strips_xtml_and_suppresses_calls():
|
||||
parser = KimiK3DelegatingParser(
|
||||
_dummy_tokenizer(), chat_template_kwargs={"thinking": False}
|
||||
)
|
||||
request = _request().model_copy(
|
||||
update={
|
||||
"tool_choice": "none",
|
||||
"chat_template_kwargs": {"thinking": False},
|
||||
}
|
||||
)
|
||||
messages: list[DeltaMessage] = []
|
||||
chunks = [
|
||||
OPEN,
|
||||
"response",
|
||||
f"{SEP}answer",
|
||||
RESPONSE_CLOSE,
|
||||
_tools(_call("calc", 1, _arg("x", "number", "1"))),
|
||||
]
|
||||
|
||||
for index, chunk in enumerate(chunks, start=1):
|
||||
delta = parser.parse_delta(
|
||||
delta_text=chunk,
|
||||
delta_token_ids=[index],
|
||||
request=request,
|
||||
prompt_token_ids=[1],
|
||||
finished=index == len(chunks),
|
||||
)
|
||||
if delta is not None:
|
||||
messages.append(delta)
|
||||
|
||||
content = "".join(message.content or "" for message in messages)
|
||||
assert content == "answer"
|
||||
assert OPEN not in content
|
||||
assert CLOSE not in content
|
||||
assert SEP not in content
|
||||
assert all(not message.tool_calls for message in messages)
|
||||
|
||||
|
||||
def test_adjust_request_keeps_xtml_markers_contiguous():
|
||||
parser = KimiK3ToolParser(_dummy_tokenizer())
|
||||
request = _request()
|
||||
|
||||
adjusted = parser.adjust_request(request)
|
||||
|
||||
assert adjusted.skip_special_tokens is False
|
||||
if hasattr(adjusted, "spaces_between_special_tokens"):
|
||||
assert adjusted.spaces_between_special_tokens is False
|
||||
assert KimiK3ToolParser.supports_required_and_named is False
|
||||
|
||||
|
||||
def test_adjust_request_required_uses_xtml_parser_not_json_guidance():
|
||||
parser = KimiK3ToolParser(_dummy_tokenizer())
|
||||
request = _request().model_copy(update={"tool_choice": "required"})
|
||||
|
||||
adjusted = parser.adjust_request(request)
|
||||
|
||||
assert adjusted.structured_outputs is None
|
||||
assert adjusted.skip_special_tokens is False
|
||||
if hasattr(adjusted, "spaces_between_special_tokens"):
|
||||
assert adjusted.spaces_between_special_tokens is False
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"tool_request",
|
||||
[
|
||||
_named_request(),
|
||||
_responses_request(
|
||||
tool_choice={"type": "function", "name": "calc"},
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_adjust_request_rejects_named_tool_choice(tool_request):
|
||||
parser = KimiK3ToolParser(_dummy_tokenizer())
|
||||
|
||||
with pytest.raises(VLLMValidationError) as exc_info:
|
||||
parser.adjust_request(tool_request)
|
||||
|
||||
assert exc_info.value.parameter == "tool_choice"
|
||||
assert "requires strict tool calling" in str(exc_info.value)
|
||||
|
||||
|
||||
def test_responses_chat_params_carries_tool_choice_metadata():
|
||||
request = _responses_request(tool_choice="required")
|
||||
|
||||
chat_params = request.build_chat_params(
|
||||
default_template=None,
|
||||
default_template_content_format="auto",
|
||||
)
|
||||
|
||||
assert chat_params.tool_choice == "required"
|
||||
|
||||
|
||||
def test_responses_chat_params_keeps_template_tool_choice_when_api_auto():
|
||||
request = _responses_request().model_copy(
|
||||
update={"chat_template_kwargs": {"tool_choice": "required"}}
|
||||
)
|
||||
|
||||
chat_params = request.build_chat_params(
|
||||
default_template=None,
|
||||
default_template_content_format="auto",
|
||||
)
|
||||
|
||||
assert chat_params.chat_template_kwargs["tool_choice"] == "required"
|
||||
assert chat_params.tool_choice == "auto"
|
||||
|
||||
|
||||
def test_responses_required_tool_choice_uses_xtml_parser():
|
||||
parser = KimiK3DelegatingParser(
|
||||
_dummy_tokenizer(), chat_template_kwargs={"thinking": False}
|
||||
)
|
||||
request = _responses_request(tool_choice="required").model_copy(
|
||||
update={"chat_template_kwargs": {"thinking": False}}
|
||||
)
|
||||
output = RESPONSE_CLOSE + _tools(_call("calc", 1, _arg("x", "number", "1")))
|
||||
|
||||
reasoning, content, tool_calls = parser.parse(
|
||||
output, request, enable_auto_tools=True, model_output_token_ids=[]
|
||||
)
|
||||
response_outputs = build_response_output_items(
|
||||
reasoning=reasoning,
|
||||
content=content,
|
||||
tool_calls=tool_calls,
|
||||
tools=request.tools,
|
||||
)
|
||||
|
||||
assert len(response_outputs) == 1
|
||||
tool_call = response_outputs[0]
|
||||
assert tool_call.type == "function_call"
|
||||
assert tool_call.name == "calc"
|
||||
assert json.loads(tool_call.arguments) == {"x": 1}
|
||||
|
||||
|
||||
def test_responses_named_tool_choice_uses_xtml_parser():
|
||||
parser = KimiK3DelegatingParser(
|
||||
_dummy_tokenizer(), chat_template_kwargs={"thinking": False}
|
||||
)
|
||||
request = _responses_request(
|
||||
tool_choice={"type": "function", "name": "calc"}
|
||||
).model_copy(update={"chat_template_kwargs": {"thinking": False}})
|
||||
output = RESPONSE_CLOSE + _tools(_call("calc", 1, _arg("x", "number", "1")))
|
||||
|
||||
reasoning, content, tool_calls = parser.parse(
|
||||
output, request, enable_auto_tools=True, model_output_token_ids=[]
|
||||
)
|
||||
response_outputs = build_response_output_items(
|
||||
reasoning=reasoning,
|
||||
content=content,
|
||||
tool_calls=tool_calls,
|
||||
tools=request.tools,
|
||||
)
|
||||
|
||||
assert len(response_outputs) == 1
|
||||
tool_call = response_outputs[0]
|
||||
assert tool_call.type == "function_call"
|
||||
assert tool_call.name == "calc"
|
||||
assert json.loads(tool_call.arguments) == {"x": 1}
|
||||
|
||||
|
||||
def test_chat_params_carries_tool_choice_metadata():
|
||||
request = _request().model_copy(update={"tool_choice": "required"})
|
||||
|
||||
chat_params = request.build_chat_params(
|
||||
default_template=None,
|
||||
default_template_content_format="auto",
|
||||
)
|
||||
|
||||
assert chat_params.tool_choice == "required"
|
||||
|
||||
|
||||
def test_chat_params_carries_response_format_metadata():
|
||||
request = ChatCompletionRequest(
|
||||
model="test-model",
|
||||
messages=[],
|
||||
response_format={"type": "json_object"},
|
||||
)
|
||||
|
||||
chat_params = request.build_chat_params(
|
||||
default_template=None,
|
||||
default_template_content_format="auto",
|
||||
)
|
||||
|
||||
assert chat_params.response_format is request.response_format
|
||||
assert chat_params.tool_choice is None
|
||||
|
||||
|
||||
def test_chat_params_keeps_template_tool_choice_when_api_auto():
|
||||
request = _request().model_copy(
|
||||
update={
|
||||
"tool_choice": "auto",
|
||||
"chat_template_kwargs": {"tool_choice": "required"},
|
||||
}
|
||||
)
|
||||
|
||||
chat_params = request.build_chat_params(
|
||||
default_template=None,
|
||||
default_template_content_format="auto",
|
||||
)
|
||||
|
||||
assert chat_params.chat_template_kwargs["tool_choice"] == "required"
|
||||
assert chat_params.tool_choice == "auto"
|
||||
@@ -1,415 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
"""Unit tests for the MuseGlimmer ATEM tool parser and reasoning parser.
|
||||
|
||||
MuseGlimmer writes every turn as a sequence of channel-scoped messages rather
|
||||
than JSON, so the two parsers are tested together: the reasoning parser strips
|
||||
the reasoning span and forwards the remaining channels as content, and the tool
|
||||
parser reads ATEM markup out of those channels.
|
||||
|
||||
Four areas, in order:
|
||||
|
||||
1. non-streaming tool-call extraction, including channel scoping (an
|
||||
``<atem:invoke>`` echoed inside reasoning must never become a call);
|
||||
2. the reasoning -> tool-parser handoff, which regressed once by returning
|
||||
``content=None`` and starving the tool parser;
|
||||
3. streaming, where markers routinely straddle chunk boundaries, plus
|
||||
truncation isolation for an unterminated ``to=self`` block;
|
||||
4. tool-name normalization against the tools registered on the request.
|
||||
|
||||
These drive the parsers directly and need no checkpoint. The tests that require
|
||||
a real tokenizer live in ``test_muse_glimmer_parse_delta.py``.
|
||||
"""
|
||||
|
||||
import json
|
||||
from types import SimpleNamespace
|
||||
from typing import cast
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
|
||||
from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
|
||||
from vllm.reasoning.muse_glimmer_reasoning_parser import MuseGlimmerReasoningParser
|
||||
from vllm.tool_parsers.muse_glimmer_tool_parser import MuseGlimmerToolParser
|
||||
|
||||
R: MuseGlimmerReasoningParser
|
||||
T: MuseGlimmerToolParser
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _fresh_parsers():
|
||||
"""Give each test request-scoped parser state through the real constructors."""
|
||||
global R, T
|
||||
R = MuseGlimmerReasoningParser(object())
|
||||
T = MuseGlimmerToolParser(object())
|
||||
|
||||
|
||||
# Any framing token that must NEVER appear in surfaced reasoning/content.
|
||||
_FRAMING = [
|
||||
"<|start|>",
|
||||
"<|message|>",
|
||||
"<|eom|>",
|
||||
"<|eot|>",
|
||||
"to=self",
|
||||
"to=user",
|
||||
"to=read.read",
|
||||
"<atem:",
|
||||
]
|
||||
|
||||
|
||||
class _FakeReq:
|
||||
"""Minimal ChatCompletionRequest stand-in (no registered tools)."""
|
||||
|
||||
tools = None
|
||||
|
||||
|
||||
def _fake_req() -> ChatCompletionRequest:
|
||||
return cast(ChatCompletionRequest, _FakeReq())
|
||||
|
||||
|
||||
# The parsers accept a missing request at runtime.
|
||||
_NO_REQ = cast(ChatCompletionRequest, None)
|
||||
|
||||
|
||||
def _req(*names: str) -> ChatCompletionRequest:
|
||||
"""A request with ``names`` registered as tools."""
|
||||
return cast(
|
||||
ChatCompletionRequest,
|
||||
SimpleNamespace(
|
||||
tools=[SimpleNamespace(function=SimpleNamespace(name=n)) for n in names]
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def _call(name):
|
||||
"""One tool-call turn invoking ``name``."""
|
||||
return (
|
||||
f"<|start|>assistant to={name}<|message|>"
|
||||
f'<atem:function_calls>\n<atem:invoke name="{name}">\n'
|
||||
f'<atem:parameter name="city">Paris</atem:parameter>\n'
|
||||
f"</atem:invoke>\n</atem:function_calls>"
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- tool calls
|
||||
|
||||
|
||||
def test_single_tool_call_after_reasoning():
|
||||
raw = (
|
||||
"to=self<|message|>Let me check the weather.<|eom|>"
|
||||
"<|start|>assistant to=weather.get<|message|>"
|
||||
'<atem:function_calls>\n<atem:invoke name="weather.get">\n'
|
||||
'<atem:parameter name="city">Paris</atem:parameter>\n'
|
||||
'<atem:parameter name="units">celsius</atem:parameter>\n'
|
||||
"</atem:invoke>\n</atem:function_calls><|eot|>"
|
||||
)
|
||||
out = MuseGlimmerToolParser.extract_tool_calls(T, raw, _NO_REQ)
|
||||
assert out.tools_called and len(out.tool_calls) == 1
|
||||
assert out.tool_calls[0].function.name == "weather.get"
|
||||
assert json.loads(out.tool_calls[0].function.arguments) == {
|
||||
"city": "Paris",
|
||||
"units": "celsius",
|
||||
}
|
||||
|
||||
|
||||
def test_parallel_calls_across_eom_boundaries():
|
||||
raw = (
|
||||
"<|start|>assistant to=math.add<|message|>"
|
||||
'<atem:function_calls>\n<atem:invoke name="math.add">\n'
|
||||
'<atem:parameter name="a">1</atem:parameter>\n'
|
||||
'<atem:parameter name="b">2</atem:parameter>\n'
|
||||
"</atem:invoke>\n</atem:function_calls><|eom|>"
|
||||
"<|start|>assistant to=math.mul<|message|>"
|
||||
'<atem:function_calls>\n<atem:invoke name="math.mul">\n'
|
||||
'<atem:parameter name="a">3</atem:parameter>\n'
|
||||
'<atem:parameter name="b">4</atem:parameter>\n'
|
||||
"</atem:invoke>\n</atem:function_calls><|eot|>"
|
||||
)
|
||||
out = MuseGlimmerToolParser.extract_tool_calls(T, raw, _NO_REQ)
|
||||
assert out.tools_called and len(out.tool_calls) == 2, len(out.tool_calls)
|
||||
assert [t.function.name for t in out.tool_calls] == ["math.add", "math.mul"]
|
||||
# JSON-typed values decode to ints
|
||||
assert json.loads(out.tool_calls[0].function.arguments) == {"a": 1, "b": 2}
|
||||
|
||||
|
||||
def test_echoed_invoke_in_reasoning_is_not_parsed():
|
||||
"""Channel scoping: an invoke quoted inside reasoning is not a call."""
|
||||
raw = (
|
||||
'to=self<|message|>I could call <atem:invoke name="evil.fn">'
|
||||
'<atem:parameter name="x">1</atem:parameter></atem:invoke> '
|
||||
"but I will not.<|eom|>"
|
||||
"<|start|>assistant to=user<|message|>The answer is 42.<|eot|>"
|
||||
)
|
||||
out = MuseGlimmerToolParser.extract_tool_calls(T, raw, _NO_REQ)
|
||||
assert not out.tools_called, "channel scoping failed -- echoed invoke parsed!"
|
||||
assert out.content == "The answer is 42.", repr(out.content)
|
||||
|
||||
|
||||
def test_plain_answer_yields_no_tool_calls():
|
||||
out = MuseGlimmerToolParser.extract_tool_calls(
|
||||
T, "to=user<|message|>Just a plain answer.<|eot|>", _NO_REQ
|
||||
)
|
||||
assert not out.tools_called
|
||||
|
||||
|
||||
def test_json_object_array_and_bool_params_decode():
|
||||
raw = (
|
||||
"<|start|>assistant to=api.call<|message|>"
|
||||
'<atem:function_calls>\n<atem:invoke name="api.call">\n'
|
||||
'<atem:parameter name="payload">{"nested": [1, 2, 3]}</atem:parameter>\n'
|
||||
'<atem:parameter name="flag">true</atem:parameter>\n'
|
||||
"</atem:invoke>\n</atem:function_calls><|eot|>"
|
||||
)
|
||||
out = MuseGlimmerToolParser.extract_tool_calls(T, raw, _NO_REQ)
|
||||
assert json.loads(out.tool_calls[0].function.arguments) == {
|
||||
"payload": {"nested": [1, 2, 3]},
|
||||
"flag": True,
|
||||
}
|
||||
|
||||
|
||||
# ------------------------------------------------- reasoning -> tool handoff
|
||||
|
||||
|
||||
def test_reasoning_to_toolcall_handoff():
|
||||
"""The regression: content=None here starved the tool parser."""
|
||||
raw = (
|
||||
" to=self<|message|>Let me call the tool.<|eom|>"
|
||||
"<|start|>assistant to=weather.get<|message|>"
|
||||
'<atem:function_calls>\n<atem:invoke name="weather.get">\n'
|
||||
'<atem:parameter name="city">Paris</atem:parameter>\n'
|
||||
"</atem:invoke>\n</atem:function_calls>"
|
||||
)
|
||||
reasoning, content = MuseGlimmerReasoningParser.extract_reasoning(R, raw, _NO_REQ)
|
||||
assert reasoning == "Let me call the tool.", repr(reasoning)
|
||||
assert content is not None and "<atem:invoke" in content, repr(content)
|
||||
out = MuseGlimmerToolParser.extract_tool_calls(T, content, _NO_REQ)
|
||||
assert out.tools_called and len(out.tool_calls) == 1
|
||||
assert out.tool_calls[0].function.name == "weather.get"
|
||||
|
||||
|
||||
def test_reasoning_then_user_answer():
|
||||
raw = (
|
||||
" to=self<|message|>thinking<|eom|>"
|
||||
"<|start|>assistant to=user<|message|>The answer is 42.<|eot|>"
|
||||
)
|
||||
reasoning, content = MuseGlimmerReasoningParser.extract_reasoning(R, raw, _NO_REQ)
|
||||
assert reasoning == "thinking", repr(reasoning)
|
||||
assert content == "The answer is 42.", repr(content)
|
||||
assert not MuseGlimmerToolParser.extract_tool_calls(
|
||||
T, content, _NO_REQ
|
||||
).tools_called
|
||||
|
||||
|
||||
def test_plain_content_without_framing_passes_through():
|
||||
reasoning, content = MuseGlimmerReasoningParser.extract_reasoning(
|
||||
R, "Just a direct answer.", _NO_REQ
|
||||
)
|
||||
assert reasoning is None and content == "Just a direct answer.", (
|
||||
reasoning,
|
||||
content,
|
||||
)
|
||||
|
||||
|
||||
def test_reasoning_then_parallel_calls():
|
||||
raw = (
|
||||
" to=self<|message|>need two calls<|eom|>"
|
||||
"<|start|>assistant to=math.add<|message|>"
|
||||
'<atem:function_calls>\n<atem:invoke name="math.add">\n'
|
||||
'<atem:parameter name="a">1</atem:parameter>\n</atem:invoke>\n'
|
||||
"</atem:function_calls><|eom|>"
|
||||
"<|start|>assistant to=math.mul<|message|>"
|
||||
'<atem:function_calls>\n<atem:invoke name="math.mul">\n'
|
||||
'<atem:parameter name="a">3</atem:parameter>\n</atem:invoke>\n'
|
||||
"</atem:function_calls><|eot|>"
|
||||
)
|
||||
reasoning, content = MuseGlimmerReasoningParser.extract_reasoning(R, raw, _NO_REQ)
|
||||
assert reasoning == "need two calls", repr(reasoning)
|
||||
assert content is not None
|
||||
out = MuseGlimmerToolParser.extract_tool_calls(T, content, _NO_REQ)
|
||||
assert [t.function.name for t in out.tool_calls] == ["math.add", "math.mul"], (
|
||||
out.tool_calls
|
||||
)
|
||||
|
||||
|
||||
# ----------------------------------------------------------------- streaming
|
||||
|
||||
|
||||
def _stream(raw: str, chunk: int):
|
||||
"""Feed ``raw`` incrementally in ``chunk``-char steps through BOTH streaming
|
||||
parsers; return (reasoning, content, tool_calls)."""
|
||||
reasoning, content, toolcalls = [], [], []
|
||||
prev = ""
|
||||
i = 0
|
||||
while i < len(raw):
|
||||
cur = raw[: i + chunk]
|
||||
delta = cur[len(prev) :]
|
||||
dm = MuseGlimmerReasoningParser.extract_reasoning_streaming(
|
||||
R, prev, cur, delta, [], [], []
|
||||
)
|
||||
if dm is not None:
|
||||
if dm.reasoning:
|
||||
reasoning.append(dm.reasoning)
|
||||
content_delta = getattr(dm, "content", None)
|
||||
# Tool-channel content is an internal handoff to the tool parser,
|
||||
# not client-visible content from the unified parser.
|
||||
if content_delta and "<atem:function_calls>" not in content_delta:
|
||||
content.append(content_delta)
|
||||
dt = MuseGlimmerToolParser.extract_tool_calls_streaming(
|
||||
T, prev, cur, delta, [], [], [], _fake_req()
|
||||
)
|
||||
if dt is not None and dt.tool_calls:
|
||||
toolcalls.extend(dt.tool_calls)
|
||||
prev = cur
|
||||
i += chunk
|
||||
return "".join(reasoning), "".join(content), toolcalls
|
||||
|
||||
|
||||
def _fn_of(tc):
|
||||
fn = tc.function
|
||||
if isinstance(fn, dict):
|
||||
return fn.get("name"), fn.get("arguments")
|
||||
return fn.name, fn.arguments
|
||||
|
||||
|
||||
RAW_TOOLCALL = (
|
||||
" to=self<|message|>I should read the hostname file to answer.<|eom|>"
|
||||
"<|start|>assistant to=read.read<|message|>"
|
||||
'<atem:function_calls>\n<atem:invoke name="read.read">\n'
|
||||
'<atem:parameter name="path">/etc/hostname</atem:parameter>\n'
|
||||
"</atem:invoke>\n</atem:function_calls>"
|
||||
)
|
||||
|
||||
RAW_ANSWER = (
|
||||
" to=self<|message|>Think about it.<|eom|>"
|
||||
"<|start|>assistant to=user<|message|>The answer is 42.<|eot|>"
|
||||
)
|
||||
|
||||
# NO closing <|eom|> -> truncated CoT
|
||||
RAW_TRUNCATED = (
|
||||
" to=self<|message|>Maybe I should call "
|
||||
'<atem:function_calls>\n<atem:invoke name="read.read">\n'
|
||||
'<atem:parameter name="path">/etc/hostname</atem:parameter>\n'
|
||||
"</atem:invoke>\n</atem:function_calls> but wait"
|
||||
)
|
||||
|
||||
|
||||
def _check_toolcall_stream(chunk):
|
||||
reasoning, content, tcs = _stream(RAW_TOOLCALL, chunk)
|
||||
# (a) no framing token leaks into reasoning or content
|
||||
for f in _FRAMING:
|
||||
assert f not in reasoning, (
|
||||
f"framing {f!r} leaked into reasoning (chunk={chunk})"
|
||||
)
|
||||
assert f not in content, f"framing {f!r} leaked into content (chunk={chunk})"
|
||||
# (b) exactly one tool_call with correct name + args
|
||||
assert len(tcs) == 1, f"expected 1 tool_call, got {len(tcs)} (chunk={chunk})"
|
||||
name, args = _fn_of(tcs[0])
|
||||
assert name == "read.read", name
|
||||
assert json.loads(args) == {"path": "/etc/hostname"}, args
|
||||
assert tcs[0].index == 0 and tcs[0].type == "function" and tcs[0].id
|
||||
# (c) reasoning captured separately and clean
|
||||
assert reasoning == "I should read the hostname file to answer.", repr(reasoning)
|
||||
assert content == "", repr(content)
|
||||
|
||||
|
||||
def test_streaming_toolcall_chunk3():
|
||||
_check_toolcall_stream(3)
|
||||
|
||||
|
||||
def test_streaming_toolcall_charwise():
|
||||
# worst case: markers arrive one char at a time (mid-marker deltas)
|
||||
_check_toolcall_stream(1)
|
||||
|
||||
|
||||
def test_streaming_toolcall_bigchunks():
|
||||
_check_toolcall_stream(17)
|
||||
|
||||
|
||||
def test_streaming_reasoning_then_content():
|
||||
reasoning, content, tcs = _stream(RAW_ANSWER, 3)
|
||||
for f in _FRAMING:
|
||||
assert f not in reasoning and f not in content, f
|
||||
assert reasoning == "Think about it.", repr(reasoning)
|
||||
assert content == "The answer is 42.", repr(content)
|
||||
assert tcs == []
|
||||
|
||||
|
||||
def test_truncated_cot_no_toolcall_nonstreaming():
|
||||
out = MuseGlimmerToolParser.extract_tool_calls(T, RAW_TRUNCATED, _fake_req())
|
||||
assert not out.tools_called and out.tool_calls == []
|
||||
# partial reasoning must still be recovered by the reasoning parser
|
||||
reasoning, _ = MuseGlimmerReasoningParser.extract_reasoning(
|
||||
R, RAW_TRUNCATED, _fake_req()
|
||||
)
|
||||
assert reasoning and "Maybe I should call" in reasoning, repr(reasoning)
|
||||
|
||||
|
||||
def test_truncated_cot_no_toolcall_streaming():
|
||||
_, _, tcs = _stream(RAW_TRUNCATED, 3)
|
||||
assert tcs == [], f"truncated CoT invoke leaked as streaming tool call: {tcs}"
|
||||
|
||||
|
||||
# ------------------------------------------------------- name normalization
|
||||
#
|
||||
# MuseGlimmer emits `get_weather.get_weather` for a bare-registered
|
||||
# `get_weather`, and `weather.get` verbatim for a namespaced one. The parser
|
||||
# normalizes against the tools actually registered on the request.
|
||||
|
||||
|
||||
def test_doubled_bare_name_collapses():
|
||||
out = MuseGlimmerToolParser.extract_tool_calls(
|
||||
T, _call("get_weather.get_weather"), _req("get_weather")
|
||||
)
|
||||
assert out.tools_called and out.tool_calls[0].function.name == "get_weather", (
|
||||
out.tool_calls[0].function.name
|
||||
)
|
||||
|
||||
|
||||
def test_namespaced_name_preserved():
|
||||
out = MuseGlimmerToolParser.extract_tool_calls(
|
||||
T, _call("weather.get"), _req("weather.get")
|
||||
)
|
||||
assert out.tool_calls[0].function.name == "weather.get"
|
||||
|
||||
|
||||
def test_unregistered_namespace_is_preserved():
|
||||
# Suffix-only matching can silently dispatch a tool from the wrong namespace.
|
||||
out = MuseGlimmerToolParser.extract_tool_calls(
|
||||
T, _call("foo.get_weather"), _req("get_weather")
|
||||
)
|
||||
assert out.tool_calls[0].function.name == "foo.get_weather"
|
||||
|
||||
|
||||
def test_trailing_segment_ambiguous_left_alone():
|
||||
# two registered tools share leaf 'get' -> ambiguous -> do NOT rewrite
|
||||
out = MuseGlimmerToolParser.extract_tool_calls(
|
||||
T, _call("x.get"), _req("weather.get", "time.get")
|
||||
)
|
||||
assert out.tool_calls[0].function.name == "x.get"
|
||||
|
||||
|
||||
def test_no_registered_tools_passthrough():
|
||||
out = MuseGlimmerToolParser.extract_tool_calls(
|
||||
T, _call("get_weather.get_weather"), _NO_REQ
|
||||
)
|
||||
assert out.tool_calls[0].function.name == "get_weather.get_weather"
|
||||
|
||||
|
||||
def test_exact_match_kept():
|
||||
out = MuseGlimmerToolParser.extract_tool_calls(
|
||||
T, _call("get_weather"), _req("get_weather")
|
||||
)
|
||||
assert out.tool_calls[0].function.name == "get_weather"
|
||||
|
||||
|
||||
def test_named_custom_tool_choice_skips_structured_outputs():
|
||||
# Custom tools ride the function shim; a named custom choice must skip
|
||||
# the base adjust_request like a named function choice does.
|
||||
request = ResponsesRequest(
|
||||
input="hi",
|
||||
tools=[{"type": "custom", "name": "emit_command"}],
|
||||
tool_choice={"type": "custom", "name": "emit_command"},
|
||||
)
|
||||
T.adjust_request(request)
|
||||
assert request.structured_outputs is None
|
||||
@@ -1,187 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
"""parse_delta-level regression tests for the MuseGlimmer parsers.
|
||||
|
||||
These drive the UNIFIED parser-engine streaming API the serving layer actually
|
||||
uses (``DelegatingParser.parse_delta``), NOT the ``extract_*_streaming`` methods
|
||||
directly. This is the coverage that was missing when a live no-tools streaming
|
||||
request leaked raw harmony framing into ``content``: the phase machine marked
|
||||
``reasoning_ended=True`` at the prompt boundary (because ``is_reasoning_end`` on
|
||||
a prompt with no ``to=self`` wrongly returned True), skipping the reasoning
|
||||
phase for the whole generation.
|
||||
|
||||
Requires a real MuseGlimmer tokenizer; skipped if the checkpoint is unavailable.
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
|
||||
import pytest
|
||||
|
||||
CKPT = os.environ.get("MUSE_GLIMMER_CKPT", "")
|
||||
|
||||
pytestmark = pytest.mark.skipif(
|
||||
not os.path.isdir(CKPT), reason=f"MuseGlimmer checkpoint not found at {CKPT}"
|
||||
)
|
||||
|
||||
_FRAMING = [
|
||||
"<|start|>",
|
||||
"<|message|>",
|
||||
"<|eom|>",
|
||||
"<|eot|>",
|
||||
"to=self",
|
||||
"to=user",
|
||||
"<atem:",
|
||||
]
|
||||
|
||||
PROMPT = "<|start|>user<|message|>hi<|eom|><|start|>assistant"
|
||||
|
||||
|
||||
class _Req:
|
||||
tools = None
|
||||
tool_choice = None
|
||||
include_reasoning = True
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def tok():
|
||||
from transformers import AutoTokenizer
|
||||
|
||||
return AutoTokenizer.from_pretrained(CKPT, trust_remote_code=True)
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def parser_cls():
|
||||
from vllm.parser import ParserManager
|
||||
|
||||
return ParserManager.get_parser(
|
||||
tool_parser_name="muse_glimmer",
|
||||
reasoning_parser_name="muse_glimmer",
|
||||
enable_auto_tools=True,
|
||||
)
|
||||
|
||||
|
||||
def _drive(parser_cls, tok, gen_text, req=None):
|
||||
"""Feed gen_text token-by-token through parse_delta (as serving does)."""
|
||||
from vllm.parser.abstract_parser import StreamState
|
||||
|
||||
if req is None:
|
||||
req = _Req()
|
||||
parser = parser_cls(tok)
|
||||
parser._stream_state = StreamState()
|
||||
prompt_ids = tok.encode(PROMPT, add_special_tokens=False)
|
||||
gen_ids = tok.encode(gen_text, add_special_tokens=False)
|
||||
reasoning, content, tools = [], [], []
|
||||
for i, tid in enumerate(gen_ids):
|
||||
dm = parser.parse_delta(
|
||||
tok.decode([tid]),
|
||||
[tid],
|
||||
req,
|
||||
prompt_token_ids=prompt_ids if i == 0 else None,
|
||||
finished=(i == len(gen_ids) - 1),
|
||||
)
|
||||
if dm is None:
|
||||
continue
|
||||
if getattr(dm, "reasoning", None):
|
||||
reasoning.append(dm.reasoning)
|
||||
if getattr(dm, "content", None):
|
||||
content.append(dm.content)
|
||||
for tc in getattr(dm, "tool_calls", None) or []:
|
||||
fn = tc.function
|
||||
name = fn.get("name") if isinstance(fn, dict) else getattr(fn, "name", None)
|
||||
args = (
|
||||
fn.get("arguments")
|
||||
if isinstance(fn, dict)
|
||||
else getattr(fn, "arguments", None)
|
||||
)
|
||||
tools.append((tc.index, name, args))
|
||||
return "".join(reasoning), "".join(content), tools
|
||||
|
||||
|
||||
def _assert_no_framing(text):
|
||||
for f in _FRAMING:
|
||||
assert f not in text, f"framing {f!r} leaked: {text!r}"
|
||||
|
||||
|
||||
def test_parse_delta_no_tools_reasoning_then_answer(parser_cls, tok):
|
||||
# The core regression: reasoning phase must stay active for a no-tools turn.
|
||||
reasoning, content, tools = _drive(
|
||||
parser_cls,
|
||||
tok,
|
||||
" to=self<|message|>Let me think step by step about the sum.<|eom|>"
|
||||
"<|start|>assistant to=user<|message|>The answer is 42.<|eot|>",
|
||||
)
|
||||
_assert_no_framing(content)
|
||||
assert reasoning == "Let me think step by step about the sum.", repr(reasoning)
|
||||
assert content == "The answer is 42.", repr(content)
|
||||
assert tools == []
|
||||
|
||||
|
||||
def test_parse_delta_content_only(parser_cls, tok):
|
||||
reasoning, content, tools = _drive(
|
||||
parser_cls,
|
||||
tok,
|
||||
" to=user<|message|>Just a direct answer.<|eot|>",
|
||||
)
|
||||
_assert_no_framing(content)
|
||||
assert content == "Just a direct answer.", repr(content)
|
||||
assert tools == []
|
||||
|
||||
|
||||
def test_parse_delta_tool_call(parser_cls, tok):
|
||||
reasoning, content, tools = _drive(
|
||||
parser_cls,
|
||||
tok,
|
||||
" to=self<|message|>I should read the hostname.<|eom|>"
|
||||
"<|start|>assistant to=read.read<|message|>"
|
||||
'<atem:function_calls>\n<atem:invoke name="read.read">\n'
|
||||
'<atem:parameter name="path">/etc/hostname</atem:parameter>\n'
|
||||
"</atem:invoke>\n</atem:function_calls>",
|
||||
)
|
||||
_assert_no_framing(content)
|
||||
assert reasoning == "I should read the hostname.", repr(reasoning)
|
||||
assert len(tools) == 1, tools
|
||||
idx, name, args = tools[0]
|
||||
assert idx == 0 and name == "read.read"
|
||||
assert json.loads(args) == {"path": "/etc/hostname"}
|
||||
|
||||
|
||||
def test_parse_delta_truncated_cot_no_toolcall(parser_cls, tok):
|
||||
# Contemplated invoke inside an unterminated to=self block: no tool call,
|
||||
# no framing leak into content, partial reasoning recovered.
|
||||
reasoning, content, tools = _drive(
|
||||
parser_cls,
|
||||
tok,
|
||||
" to=self<|message|>Maybe I should call "
|
||||
'<atem:function_calls>\n<atem:invoke name="read.read">\n'
|
||||
'<atem:parameter name="path">/etc/hostname</atem:parameter>\n'
|
||||
"</atem:invoke>\n</atem:function_calls> but wait",
|
||||
)
|
||||
_assert_no_framing(content)
|
||||
assert tools == [], f"contemplated invoke leaked as tool call: {tools}"
|
||||
assert "Maybe I should call" in reasoning, repr(reasoning)
|
||||
|
||||
|
||||
def test_parse_delta_reasoning_suppressed_when_not_requested(parser_cls, tok):
|
||||
class _NoReasonReq:
|
||||
tools = None
|
||||
tool_choice = None
|
||||
include_reasoning = False
|
||||
|
||||
reasoning, content, tools = _drive(
|
||||
parser_cls,
|
||||
tok,
|
||||
" to=self<|message|>secret thoughts<|eom|>"
|
||||
"<|start|>assistant to=user<|message|>Public answer.<|eot|>",
|
||||
req=_NoReasonReq(),
|
||||
)
|
||||
assert reasoning == "", repr(reasoning) # suppressed
|
||||
assert content == "Public answer.", repr(content)
|
||||
_assert_no_framing(content)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
import sys
|
||||
|
||||
sys.exit(pytest.main([__file__, "-v"]))
|
||||
Reference in New Issue
Block a user