Add Muse Glimmer model support (#51655)
Signed-off-by: Tiezhen Wang <tiezhen@inferact.ai> Signed-off-by: Jee Jee Li <jeejeelee@inferact.ai> Signed-off-by: zixi-qi <zixi@inferact.ai> Co-authored-by: Beto de Paola <betodepaola@meta.com> Co-authored-by: Giancarlo Delfin <gdelfin@inferact.ai> Co-authored-by: Jee Jee Li <jeejeelee@inferact.ai> Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com> Co-authored-by: Jee Jee Li <pandaleefree@gmail.com> Co-authored-by: zixi-qi <zixi@inferact.ai>
This commit is contained in:
co-authored by
Beto de Paola
Giancarlo Delfin
Jee Jee Li
Claude Opus 5
Jee Jee Li
zixi-qi
parent
59d1af5698
commit
6adad08767
@@ -1192,6 +1192,12 @@ _MULTIMODAL_EXAMPLE_MODELS = {
|
||||
# required by current PrefixLM implementation
|
||||
max_num_batched_tokens=31872,
|
||||
),
|
||||
"MuseGlimmerForConditionalGeneration": _HfExamplesInfo(
|
||||
"meta-models/Muse-Glimmer-30B",
|
||||
),
|
||||
"MuseGlimmerForCausalLM": _HfExamplesInfo(
|
||||
"meta-models/Muse-Glimmer-30B",
|
||||
),
|
||||
"NVLM_D": _HfExamplesInfo("nvidia/NVLM-D-72B", trust_remote_code=True),
|
||||
"Llama_Nemotron_Nano_VL": _HfExamplesInfo(
|
||||
"nvidia/Llama-3.1-Nemotron-Nano-VL-8B-V1",
|
||||
@@ -1489,6 +1495,18 @@ _SPECULATIVE_DECODING_EXAMPLE_MODELS = {
|
||||
max_num_seqs=32,
|
||||
min_transformers_version="4.56.3", # Required for Qwen3Next
|
||||
),
|
||||
"MuseGlimmerAssistantModel": _HfExamplesInfo(
|
||||
"meta-models/Muse-Glimmer-30B",
|
||||
speculative_model="meta-models/Muse-Glimmer-30B-assistant",
|
||||
max_model_len=8192, # Reduce max len to ensure test runs in low-VRAM CI env
|
||||
max_num_seqs=32,
|
||||
),
|
||||
"DFlashMuseGlimmerAssistantModel": _HfExamplesInfo(
|
||||
"meta-models/Muse-Glimmer-30B",
|
||||
speculative_model="meta-models/Muse-Glimmer-30B-assistant",
|
||||
max_model_len=8192, # Reduce max len to ensure test runs in low-VRAM CI env
|
||||
max_num_seqs=32,
|
||||
),
|
||||
# [DSpark]
|
||||
"DSparkDraftModel": _HfExamplesInfo(
|
||||
"deepseek-ai/DeepSeek-V4-Pro-DSpark",
|
||||
|
||||
@@ -564,15 +564,20 @@ def dummy_hf_overrides(
|
||||
)
|
||||
|
||||
if hasattr(hf_config, "vision_config"):
|
||||
hf_config.vision_config.update(
|
||||
vision_config = hf_config.vision_config
|
||||
vision_config.update(
|
||||
{
|
||||
"num_layers": 1,
|
||||
"num_hidden_layers": 1,
|
||||
}
|
||||
)
|
||||
|
||||
# Keep per-layer metadata consistent with the reduced layer count.
|
||||
if layer_types := getattr(vision_config, "layer_types", None):
|
||||
vision_config.update({"layer_types": layer_types[:1]})
|
||||
|
||||
if model_arch in ("Moondream3ForCausalLM", "HfMoondream"):
|
||||
hf_config.vision_config.update({"enc_n_layers": 1})
|
||||
vision_config.update({"enc_n_layers": 1})
|
||||
|
||||
# e.g.: ibm-granite/granite-speech-3.3-2b
|
||||
if hasattr(hf_config, "encoder_config"):
|
||||
|
||||
@@ -0,0 +1,386 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
"""Unit tests for the MuseGlimmer ATEM tool parser and reasoning parser.
|
||||
|
||||
MuseGlimmer writes every turn as a sequence of channel-scoped messages rather
|
||||
than JSON, so the two parsers are tested together: the reasoning parser strips
|
||||
the reasoning span and forwards the remaining channels as content, and the tool
|
||||
parser reads ATEM markup out of those channels.
|
||||
|
||||
Four areas, in order:
|
||||
|
||||
1. non-streaming tool-call extraction, including channel scoping (an
|
||||
``<atem:invoke>`` echoed inside reasoning must never become a call);
|
||||
2. the reasoning -> tool-parser handoff, which regressed once by returning
|
||||
``content=None`` and starving the tool parser;
|
||||
3. streaming, where markers routinely straddle chunk boundaries, plus
|
||||
truncation isolation for an unterminated ``to=self`` block;
|
||||
4. tool-name normalization against the tools registered on the request.
|
||||
|
||||
These drive the parsers directly and need no checkpoint. The tests that require
|
||||
a real tokenizer live in ``test_muse_glimmer_parse_delta.py``.
|
||||
"""
|
||||
|
||||
import json
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm.reasoning.muse_glimmer_reasoning_parser import MuseGlimmerReasoningParser
|
||||
from vllm.tool_parsers.muse_glimmer_tool_parser import MuseGlimmerToolParser
|
||||
|
||||
R: MuseGlimmerReasoningParser
|
||||
T: MuseGlimmerToolParser
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _fresh_parsers():
|
||||
"""Give each test request-scoped parser state through the real constructors."""
|
||||
global R, T
|
||||
R = MuseGlimmerReasoningParser(object())
|
||||
T = MuseGlimmerToolParser(object())
|
||||
|
||||
|
||||
# Any framing token that must NEVER appear in surfaced reasoning/content.
|
||||
_FRAMING = [
|
||||
"<|start|>",
|
||||
"<|message|>",
|
||||
"<|eom|>",
|
||||
"<|eot|>",
|
||||
"to=self",
|
||||
"to=user",
|
||||
"to=read.read",
|
||||
"<atem:",
|
||||
]
|
||||
|
||||
|
||||
class _FakeReq:
|
||||
"""Minimal ChatCompletionRequest stand-in (no registered tools)."""
|
||||
|
||||
tools = None
|
||||
|
||||
|
||||
def _req(*names):
|
||||
"""A request with ``names`` registered as tools."""
|
||||
return SimpleNamespace(
|
||||
tools=[SimpleNamespace(function=SimpleNamespace(name=n)) for n in names]
|
||||
)
|
||||
|
||||
|
||||
def _call(name):
|
||||
"""One tool-call turn invoking ``name``."""
|
||||
return (
|
||||
f"<|start|>assistant to={name}<|message|>"
|
||||
f'<atem:function_calls>\n<atem:invoke name="{name}">\n'
|
||||
f'<atem:parameter name="city">Paris</atem:parameter>\n'
|
||||
f"</atem:invoke>\n</atem:function_calls>"
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- tool calls
|
||||
|
||||
|
||||
def test_single_tool_call_after_reasoning():
|
||||
raw = (
|
||||
"to=self<|message|>Let me check the weather.<|eom|>"
|
||||
"<|start|>assistant to=weather.get<|message|>"
|
||||
'<atem:function_calls>\n<atem:invoke name="weather.get">\n'
|
||||
'<atem:parameter name="city">Paris</atem:parameter>\n'
|
||||
'<atem:parameter name="units">celsius</atem:parameter>\n'
|
||||
"</atem:invoke>\n</atem:function_calls><|eot|>"
|
||||
)
|
||||
out = MuseGlimmerToolParser.extract_tool_calls(T, raw, None)
|
||||
assert out.tools_called and len(out.tool_calls) == 1
|
||||
assert out.tool_calls[0].function.name == "weather.get"
|
||||
assert json.loads(out.tool_calls[0].function.arguments) == {
|
||||
"city": "Paris",
|
||||
"units": "celsius",
|
||||
}
|
||||
|
||||
|
||||
def test_parallel_calls_across_eom_boundaries():
|
||||
raw = (
|
||||
"<|start|>assistant to=math.add<|message|>"
|
||||
'<atem:function_calls>\n<atem:invoke name="math.add">\n'
|
||||
'<atem:parameter name="a">1</atem:parameter>\n'
|
||||
'<atem:parameter name="b">2</atem:parameter>\n'
|
||||
"</atem:invoke>\n</atem:function_calls><|eom|>"
|
||||
"<|start|>assistant to=math.mul<|message|>"
|
||||
'<atem:function_calls>\n<atem:invoke name="math.mul">\n'
|
||||
'<atem:parameter name="a">3</atem:parameter>\n'
|
||||
'<atem:parameter name="b">4</atem:parameter>\n'
|
||||
"</atem:invoke>\n</atem:function_calls><|eot|>"
|
||||
)
|
||||
out = MuseGlimmerToolParser.extract_tool_calls(T, raw, None)
|
||||
assert out.tools_called and len(out.tool_calls) == 2, len(out.tool_calls)
|
||||
assert [t.function.name for t in out.tool_calls] == ["math.add", "math.mul"]
|
||||
# JSON-typed values decode to ints
|
||||
assert json.loads(out.tool_calls[0].function.arguments) == {"a": 1, "b": 2}
|
||||
|
||||
|
||||
def test_echoed_invoke_in_reasoning_is_not_parsed():
|
||||
"""Channel scoping: an invoke quoted inside reasoning is not a call."""
|
||||
raw = (
|
||||
'to=self<|message|>I could call <atem:invoke name="evil.fn">'
|
||||
'<atem:parameter name="x">1</atem:parameter></atem:invoke> '
|
||||
"but I will not.<|eom|>"
|
||||
"<|start|>assistant to=user<|message|>The answer is 42.<|eot|>"
|
||||
)
|
||||
out = MuseGlimmerToolParser.extract_tool_calls(T, raw, None)
|
||||
assert not out.tools_called, "channel scoping failed -- echoed invoke parsed!"
|
||||
assert out.content == "The answer is 42.", repr(out.content)
|
||||
|
||||
|
||||
def test_plain_answer_yields_no_tool_calls():
|
||||
out = MuseGlimmerToolParser.extract_tool_calls(
|
||||
T, "to=user<|message|>Just a plain answer.<|eot|>", None
|
||||
)
|
||||
assert not out.tools_called
|
||||
|
||||
|
||||
def test_json_object_array_and_bool_params_decode():
|
||||
raw = (
|
||||
"<|start|>assistant to=api.call<|message|>"
|
||||
'<atem:function_calls>\n<atem:invoke name="api.call">\n'
|
||||
'<atem:parameter name="payload">{"nested": [1, 2, 3]}</atem:parameter>\n'
|
||||
'<atem:parameter name="flag">true</atem:parameter>\n'
|
||||
"</atem:invoke>\n</atem:function_calls><|eot|>"
|
||||
)
|
||||
out = MuseGlimmerToolParser.extract_tool_calls(T, raw, None)
|
||||
assert json.loads(out.tool_calls[0].function.arguments) == {
|
||||
"payload": {"nested": [1, 2, 3]},
|
||||
"flag": True,
|
||||
}
|
||||
|
||||
|
||||
# ------------------------------------------------- reasoning -> tool handoff
|
||||
|
||||
|
||||
def test_reasoning_to_toolcall_handoff():
|
||||
"""The regression: content=None here starved the tool parser."""
|
||||
raw = (
|
||||
" to=self<|message|>Let me call the tool.<|eom|>"
|
||||
"<|start|>assistant to=weather.get<|message|>"
|
||||
'<atem:function_calls>\n<atem:invoke name="weather.get">\n'
|
||||
'<atem:parameter name="city">Paris</atem:parameter>\n'
|
||||
"</atem:invoke>\n</atem:function_calls>"
|
||||
)
|
||||
reasoning, content = MuseGlimmerReasoningParser.extract_reasoning(R, raw, None)
|
||||
assert reasoning == "Let me call the tool.", repr(reasoning)
|
||||
assert content is not None and "<atem:invoke" in content, repr(content)
|
||||
out = MuseGlimmerToolParser.extract_tool_calls(T, content, None)
|
||||
assert out.tools_called and len(out.tool_calls) == 1
|
||||
assert out.tool_calls[0].function.name == "weather.get"
|
||||
|
||||
|
||||
def test_reasoning_then_user_answer():
|
||||
raw = (
|
||||
" to=self<|message|>thinking<|eom|>"
|
||||
"<|start|>assistant to=user<|message|>The answer is 42.<|eot|>"
|
||||
)
|
||||
reasoning, content = MuseGlimmerReasoningParser.extract_reasoning(R, raw, None)
|
||||
assert reasoning == "thinking", repr(reasoning)
|
||||
assert content == "The answer is 42.", repr(content)
|
||||
assert not MuseGlimmerToolParser.extract_tool_calls(T, content, None).tools_called
|
||||
|
||||
|
||||
def test_plain_content_without_framing_passes_through():
|
||||
reasoning, content = MuseGlimmerReasoningParser.extract_reasoning(
|
||||
R, "Just a direct answer.", None
|
||||
)
|
||||
assert reasoning is None and content == "Just a direct answer.", (
|
||||
reasoning,
|
||||
content,
|
||||
)
|
||||
|
||||
|
||||
def test_reasoning_then_parallel_calls():
|
||||
raw = (
|
||||
" to=self<|message|>need two calls<|eom|>"
|
||||
"<|start|>assistant to=math.add<|message|>"
|
||||
'<atem:function_calls>\n<atem:invoke name="math.add">\n'
|
||||
'<atem:parameter name="a">1</atem:parameter>\n</atem:invoke>\n'
|
||||
"</atem:function_calls><|eom|>"
|
||||
"<|start|>assistant to=math.mul<|message|>"
|
||||
'<atem:function_calls>\n<atem:invoke name="math.mul">\n'
|
||||
'<atem:parameter name="a">3</atem:parameter>\n</atem:invoke>\n'
|
||||
"</atem:function_calls><|eot|>"
|
||||
)
|
||||
reasoning, content = MuseGlimmerReasoningParser.extract_reasoning(R, raw, None)
|
||||
assert reasoning == "need two calls", repr(reasoning)
|
||||
out = MuseGlimmerToolParser.extract_tool_calls(T, content, None)
|
||||
assert [t.function.name for t in out.tool_calls] == ["math.add", "math.mul"], (
|
||||
out.tool_calls
|
||||
)
|
||||
|
||||
|
||||
# ----------------------------------------------------------------- streaming
|
||||
|
||||
|
||||
def _stream(raw: str, chunk: int):
|
||||
"""Feed ``raw`` incrementally in ``chunk``-char steps through BOTH streaming
|
||||
parsers; return (reasoning, content, tool_calls)."""
|
||||
reasoning, content, toolcalls = [], [], []
|
||||
prev = ""
|
||||
i = 0
|
||||
while i < len(raw):
|
||||
cur = raw[: i + chunk]
|
||||
delta = cur[len(prev) :]
|
||||
dm = MuseGlimmerReasoningParser.extract_reasoning_streaming(
|
||||
R, prev, cur, delta, [], [], []
|
||||
)
|
||||
if dm is not None:
|
||||
if getattr(dm, "reasoning", None):
|
||||
reasoning.append(dm.reasoning)
|
||||
content_delta = getattr(dm, "content", None)
|
||||
# Tool-channel content is an internal handoff to the tool parser,
|
||||
# not client-visible content from the unified parser.
|
||||
if content_delta and "<atem:function_calls>" not in content_delta:
|
||||
content.append(content_delta)
|
||||
dt = MuseGlimmerToolParser.extract_tool_calls_streaming(
|
||||
T, prev, cur, delta, [], [], [], _FakeReq()
|
||||
)
|
||||
if dt is not None and dt.tool_calls:
|
||||
toolcalls.extend(dt.tool_calls)
|
||||
prev = cur
|
||||
i += chunk
|
||||
return "".join(reasoning), "".join(content), toolcalls
|
||||
|
||||
|
||||
def _fn_of(tc):
|
||||
fn = tc.function
|
||||
if isinstance(fn, dict):
|
||||
return fn.get("name"), fn.get("arguments")
|
||||
return fn.name, fn.arguments
|
||||
|
||||
|
||||
RAW_TOOLCALL = (
|
||||
" to=self<|message|>I should read the hostname file to answer.<|eom|>"
|
||||
"<|start|>assistant to=read.read<|message|>"
|
||||
'<atem:function_calls>\n<atem:invoke name="read.read">\n'
|
||||
'<atem:parameter name="path">/etc/hostname</atem:parameter>\n'
|
||||
"</atem:invoke>\n</atem:function_calls>"
|
||||
)
|
||||
|
||||
RAW_ANSWER = (
|
||||
" to=self<|message|>Think about it.<|eom|>"
|
||||
"<|start|>assistant to=user<|message|>The answer is 42.<|eot|>"
|
||||
)
|
||||
|
||||
# NO closing <|eom|> -> truncated CoT
|
||||
RAW_TRUNCATED = (
|
||||
" to=self<|message|>Maybe I should call "
|
||||
'<atem:function_calls>\n<atem:invoke name="read.read">\n'
|
||||
'<atem:parameter name="path">/etc/hostname</atem:parameter>\n'
|
||||
"</atem:invoke>\n</atem:function_calls> but wait"
|
||||
)
|
||||
|
||||
|
||||
def _check_toolcall_stream(chunk):
|
||||
reasoning, content, tcs = _stream(RAW_TOOLCALL, chunk)
|
||||
# (a) no framing token leaks into reasoning or content
|
||||
for f in _FRAMING:
|
||||
assert f not in reasoning, (
|
||||
f"framing {f!r} leaked into reasoning (chunk={chunk})"
|
||||
)
|
||||
assert f not in content, f"framing {f!r} leaked into content (chunk={chunk})"
|
||||
# (b) exactly one tool_call with correct name + args
|
||||
assert len(tcs) == 1, f"expected 1 tool_call, got {len(tcs)} (chunk={chunk})"
|
||||
name, args = _fn_of(tcs[0])
|
||||
assert name == "read.read", name
|
||||
assert json.loads(args) == {"path": "/etc/hostname"}, args
|
||||
assert tcs[0].index == 0 and tcs[0].type == "function" and tcs[0].id
|
||||
# (c) reasoning captured separately and clean
|
||||
assert reasoning == "I should read the hostname file to answer.", repr(reasoning)
|
||||
assert content == "", repr(content)
|
||||
|
||||
|
||||
def test_streaming_toolcall_chunk3():
|
||||
_check_toolcall_stream(3)
|
||||
|
||||
|
||||
def test_streaming_toolcall_charwise():
|
||||
# worst case: markers arrive one char at a time (mid-marker deltas)
|
||||
_check_toolcall_stream(1)
|
||||
|
||||
|
||||
def test_streaming_toolcall_bigchunks():
|
||||
_check_toolcall_stream(17)
|
||||
|
||||
|
||||
def test_streaming_reasoning_then_content():
|
||||
reasoning, content, tcs = _stream(RAW_ANSWER, 3)
|
||||
for f in _FRAMING:
|
||||
assert f not in reasoning and f not in content, f
|
||||
assert reasoning == "Think about it.", repr(reasoning)
|
||||
assert content == "The answer is 42.", repr(content)
|
||||
assert tcs == []
|
||||
|
||||
|
||||
def test_truncated_cot_no_toolcall_nonstreaming():
|
||||
out = MuseGlimmerToolParser.extract_tool_calls(T, RAW_TRUNCATED, _FakeReq())
|
||||
assert not out.tools_called and out.tool_calls == []
|
||||
# partial reasoning must still be recovered by the reasoning parser
|
||||
reasoning, _ = MuseGlimmerReasoningParser.extract_reasoning(
|
||||
R, RAW_TRUNCATED, _FakeReq()
|
||||
)
|
||||
assert reasoning and "Maybe I should call" in reasoning, repr(reasoning)
|
||||
|
||||
|
||||
def test_truncated_cot_no_toolcall_streaming():
|
||||
_, _, tcs = _stream(RAW_TRUNCATED, 3)
|
||||
assert tcs == [], f"truncated CoT invoke leaked as streaming tool call: {tcs}"
|
||||
|
||||
|
||||
# ------------------------------------------------------- name normalization
|
||||
#
|
||||
# MuseGlimmer emits `get_weather.get_weather` for a bare-registered
|
||||
# `get_weather`, and `weather.get` verbatim for a namespaced one. The parser
|
||||
# normalizes against the tools actually registered on the request.
|
||||
|
||||
|
||||
def test_doubled_bare_name_collapses():
|
||||
out = MuseGlimmerToolParser.extract_tool_calls(
|
||||
T, _call("get_weather.get_weather"), _req("get_weather")
|
||||
)
|
||||
assert out.tools_called and out.tool_calls[0].function.name == "get_weather", (
|
||||
out.tool_calls[0].function.name
|
||||
)
|
||||
|
||||
|
||||
def test_namespaced_name_preserved():
|
||||
out = MuseGlimmerToolParser.extract_tool_calls(
|
||||
T, _call("weather.get"), _req("weather.get")
|
||||
)
|
||||
assert out.tool_calls[0].function.name == "weather.get"
|
||||
|
||||
|
||||
def test_unregistered_namespace_is_preserved():
|
||||
# Suffix-only matching can silently dispatch a tool from the wrong namespace.
|
||||
out = MuseGlimmerToolParser.extract_tool_calls(
|
||||
T, _call("foo.get_weather"), _req("get_weather")
|
||||
)
|
||||
assert out.tool_calls[0].function.name == "foo.get_weather"
|
||||
|
||||
|
||||
def test_trailing_segment_ambiguous_left_alone():
|
||||
# two registered tools share leaf 'get' -> ambiguous -> do NOT rewrite
|
||||
out = MuseGlimmerToolParser.extract_tool_calls(
|
||||
T, _call("x.get"), _req("weather.get", "time.get")
|
||||
)
|
||||
assert out.tool_calls[0].function.name == "x.get"
|
||||
|
||||
|
||||
def test_no_registered_tools_passthrough():
|
||||
out = MuseGlimmerToolParser.extract_tool_calls(
|
||||
T, _call("get_weather.get_weather"), None
|
||||
)
|
||||
assert out.tool_calls[0].function.name == "get_weather.get_weather"
|
||||
|
||||
|
||||
def test_exact_match_kept():
|
||||
out = MuseGlimmerToolParser.extract_tool_calls(
|
||||
T, _call("get_weather"), _req("get_weather")
|
||||
)
|
||||
assert out.tool_calls[0].function.name == "get_weather"
|
||||
@@ -0,0 +1,187 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
"""parse_delta-level regression tests for the MuseGlimmer parsers.
|
||||
|
||||
These drive the UNIFIED parser-engine streaming API the serving layer actually
|
||||
uses (``DelegatingParser.parse_delta``), NOT the ``extract_*_streaming`` methods
|
||||
directly. This is the coverage that was missing when a live no-tools streaming
|
||||
request leaked raw harmony framing into ``content``: the phase machine marked
|
||||
``reasoning_ended=True`` at the prompt boundary (because ``is_reasoning_end`` on
|
||||
a prompt with no ``to=self`` wrongly returned True), skipping the reasoning
|
||||
phase for the whole generation.
|
||||
|
||||
Requires a real MuseGlimmer tokenizer; skipped if the checkpoint is unavailable.
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
|
||||
import pytest
|
||||
|
||||
CKPT = os.environ.get("MUSE_GLIMMER_CKPT", "")
|
||||
|
||||
pytestmark = pytest.mark.skipif(
|
||||
not os.path.isdir(CKPT), reason=f"MuseGlimmer checkpoint not found at {CKPT}"
|
||||
)
|
||||
|
||||
_FRAMING = [
|
||||
"<|start|>",
|
||||
"<|message|>",
|
||||
"<|eom|>",
|
||||
"<|eot|>",
|
||||
"to=self",
|
||||
"to=user",
|
||||
"<atem:",
|
||||
]
|
||||
|
||||
PROMPT = "<|start|>user<|message|>hi<|eom|><|start|>assistant"
|
||||
|
||||
|
||||
class _Req:
|
||||
tools = None
|
||||
tool_choice = None
|
||||
include_reasoning = True
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def tok():
|
||||
from transformers import AutoTokenizer
|
||||
|
||||
return AutoTokenizer.from_pretrained(CKPT, trust_remote_code=True)
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def parser_cls():
|
||||
from vllm.parser import ParserManager
|
||||
|
||||
return ParserManager.get_parser(
|
||||
tool_parser_name="muse_glimmer",
|
||||
reasoning_parser_name="muse_glimmer",
|
||||
enable_auto_tools=True,
|
||||
)
|
||||
|
||||
|
||||
def _drive(parser_cls, tok, gen_text, req=None):
|
||||
"""Feed gen_text token-by-token through parse_delta (as serving does)."""
|
||||
from vllm.parser.abstract_parser import StreamState
|
||||
|
||||
if req is None:
|
||||
req = _Req()
|
||||
parser = parser_cls(tok)
|
||||
parser._stream_state = StreamState()
|
||||
prompt_ids = tok.encode(PROMPT, add_special_tokens=False)
|
||||
gen_ids = tok.encode(gen_text, add_special_tokens=False)
|
||||
reasoning, content, tools = [], [], []
|
||||
for i, tid in enumerate(gen_ids):
|
||||
dm = parser.parse_delta(
|
||||
tok.decode([tid]),
|
||||
[tid],
|
||||
req,
|
||||
prompt_token_ids=prompt_ids if i == 0 else None,
|
||||
finished=(i == len(gen_ids) - 1),
|
||||
)
|
||||
if dm is None:
|
||||
continue
|
||||
if getattr(dm, "reasoning", None):
|
||||
reasoning.append(dm.reasoning)
|
||||
if getattr(dm, "content", None):
|
||||
content.append(dm.content)
|
||||
for tc in getattr(dm, "tool_calls", None) or []:
|
||||
fn = tc.function
|
||||
name = fn.get("name") if isinstance(fn, dict) else getattr(fn, "name", None)
|
||||
args = (
|
||||
fn.get("arguments")
|
||||
if isinstance(fn, dict)
|
||||
else getattr(fn, "arguments", None)
|
||||
)
|
||||
tools.append((tc.index, name, args))
|
||||
return "".join(reasoning), "".join(content), tools
|
||||
|
||||
|
||||
def _assert_no_framing(text):
|
||||
for f in _FRAMING:
|
||||
assert f not in text, f"framing {f!r} leaked: {text!r}"
|
||||
|
||||
|
||||
def test_parse_delta_no_tools_reasoning_then_answer(parser_cls, tok):
|
||||
# The core regression: reasoning phase must stay active for a no-tools turn.
|
||||
reasoning, content, tools = _drive(
|
||||
parser_cls,
|
||||
tok,
|
||||
" to=self<|message|>Let me think step by step about the sum.<|eom|>"
|
||||
"<|start|>assistant to=user<|message|>The answer is 42.<|eot|>",
|
||||
)
|
||||
_assert_no_framing(content)
|
||||
assert reasoning == "Let me think step by step about the sum.", repr(reasoning)
|
||||
assert content == "The answer is 42.", repr(content)
|
||||
assert tools == []
|
||||
|
||||
|
||||
def test_parse_delta_content_only(parser_cls, tok):
|
||||
reasoning, content, tools = _drive(
|
||||
parser_cls,
|
||||
tok,
|
||||
" to=user<|message|>Just a direct answer.<|eot|>",
|
||||
)
|
||||
_assert_no_framing(content)
|
||||
assert content == "Just a direct answer.", repr(content)
|
||||
assert tools == []
|
||||
|
||||
|
||||
def test_parse_delta_tool_call(parser_cls, tok):
|
||||
reasoning, content, tools = _drive(
|
||||
parser_cls,
|
||||
tok,
|
||||
" to=self<|message|>I should read the hostname.<|eom|>"
|
||||
"<|start|>assistant to=read.read<|message|>"
|
||||
'<atem:function_calls>\n<atem:invoke name="read.read">\n'
|
||||
'<atem:parameter name="path">/etc/hostname</atem:parameter>\n'
|
||||
"</atem:invoke>\n</atem:function_calls>",
|
||||
)
|
||||
_assert_no_framing(content)
|
||||
assert reasoning == "I should read the hostname.", repr(reasoning)
|
||||
assert len(tools) == 1, tools
|
||||
idx, name, args = tools[0]
|
||||
assert idx == 0 and name == "read.read"
|
||||
assert json.loads(args) == {"path": "/etc/hostname"}
|
||||
|
||||
|
||||
def test_parse_delta_truncated_cot_no_toolcall(parser_cls, tok):
|
||||
# Contemplated invoke inside an unterminated to=self block: no tool call,
|
||||
# no framing leak into content, partial reasoning recovered.
|
||||
reasoning, content, tools = _drive(
|
||||
parser_cls,
|
||||
tok,
|
||||
" to=self<|message|>Maybe I should call "
|
||||
'<atem:function_calls>\n<atem:invoke name="read.read">\n'
|
||||
'<atem:parameter name="path">/etc/hostname</atem:parameter>\n'
|
||||
"</atem:invoke>\n</atem:function_calls> but wait",
|
||||
)
|
||||
_assert_no_framing(content)
|
||||
assert tools == [], f"contemplated invoke leaked as tool call: {tools}"
|
||||
assert "Maybe I should call" in reasoning, repr(reasoning)
|
||||
|
||||
|
||||
def test_parse_delta_reasoning_suppressed_when_not_requested(parser_cls, tok):
|
||||
class _NoReasonReq:
|
||||
tools = None
|
||||
tool_choice = None
|
||||
include_reasoning = False
|
||||
|
||||
reasoning, content, tools = _drive(
|
||||
parser_cls,
|
||||
tok,
|
||||
" to=self<|message|>secret thoughts<|eom|>"
|
||||
"<|start|>assistant to=user<|message|>Public answer.<|eot|>",
|
||||
req=_NoReasonReq(),
|
||||
)
|
||||
assert reasoning == "", repr(reasoning) # suppressed
|
||||
assert content == "Public answer.", repr(content)
|
||||
_assert_no_framing(content)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
import sys
|
||||
|
||||
sys.exit(pytest.main([__file__, "-v"]))
|
||||
@@ -0,0 +1,174 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
"""MuseGlimmer config normalization.
|
||||
|
||||
Two schema pairs have to converge on the same model, and each has produced a
|
||||
silent-wrong-output bug:
|
||||
|
||||
1. flat (legacy converter) vs nested (canonical) ``config.json``. A FLAT
|
||||
config once deserialized to an all-default text config, silently dropping
|
||||
every checkpoint value.
|
||||
2. native vs modular attention config. The modular HF text_config OMITS
|
||||
use_qk_norm / use_attn_output_gate (read as None) and ships a PRE-FOLDED
|
||||
qk_scale_factor (43.784/sqrt(128)=3.87). Missing flags must read as True
|
||||
(MuseGlimmer always applies QK-norm + output gate) and the query pre-scale
|
||||
must normalize so native and modular land on the same number.
|
||||
"""
|
||||
|
||||
import math
|
||||
from typing import Any
|
||||
|
||||
from vllm.model_executor.models.muse_glimmer import (
|
||||
_muse_glimmer_query_prescale,
|
||||
_muse_glimmer_use_attn_output_gate,
|
||||
_muse_glimmer_use_qk_norm,
|
||||
)
|
||||
from vllm.transformers_utils.configs.muse_glimmer import MuseGlimmerConfig
|
||||
|
||||
# A representative FLAT config (Ruan rl_v1/hf shape), trimmed.
|
||||
FLAT: dict[str, Any] = {
|
||||
"architectures": ["MuseGlimmerForCausalLM"],
|
||||
"model_type": "muse_glimmer",
|
||||
"has_vision": True,
|
||||
"bos_token_id": 200000,
|
||||
"eos_token_id": 200001,
|
||||
"vocab_size": 202048,
|
||||
"hidden_size": 6656,
|
||||
"intermediate_size": 19968,
|
||||
"num_hidden_layers": 52,
|
||||
"num_attention_heads": 32,
|
||||
"num_key_value_heads": 2,
|
||||
"head_dim": 128,
|
||||
"hidden_act": "silu",
|
||||
"max_position_embeddings": 16384,
|
||||
"rms_norm_eps": 1e-5,
|
||||
"post_norm_eps": 1e-8,
|
||||
"qk_scale_factor": 43.7840518911,
|
||||
"use_qk_norm": True,
|
||||
"use_attn_output_gate": True,
|
||||
"output_multiplier": 0.19611613513818404,
|
||||
"output_soft_cap_temp": 20.0,
|
||||
"normalize_tok_embeddings": True,
|
||||
"rope_theta": 500000.0,
|
||||
"sliding_window": 2048,
|
||||
"patch_token_id": 200092,
|
||||
"vision_latent_dim": 1536,
|
||||
"vision_heads": 16,
|
||||
"vision_layers": 50,
|
||||
"vision_output_dim": 6144,
|
||||
"vision_patch_size": 14,
|
||||
"vision_patch_temporal": 2,
|
||||
"vision_adapter_dim": 4096,
|
||||
"vision_pos_emb_grid_h": 32,
|
||||
"vision_pos_emb_grid_w": 32,
|
||||
}
|
||||
|
||||
NESTED: dict[str, Any] = {
|
||||
"architectures": ["MuseGlimmerForCausalLM"],
|
||||
"model_type": "muse_glimmer",
|
||||
"image_token_id": 200092,
|
||||
"text_config": {
|
||||
"model_type": "muse_glimmer_text",
|
||||
"vocab_size": 202048,
|
||||
"hidden_size": 6656,
|
||||
"num_hidden_layers": 52,
|
||||
"hidden_activation": "silu",
|
||||
"final_logit_softcapping": 20.0,
|
||||
"qk_scale_factor": 43.7840518911,
|
||||
"rope_parameters": {"rope_type": "default", "rope_theta": 500000.0},
|
||||
},
|
||||
"vision_config": {
|
||||
"model_type": "muse_glimmer_vision",
|
||||
"hidden_size": 1536,
|
||||
"num_hidden_layers": 50,
|
||||
},
|
||||
}
|
||||
|
||||
HEAD_DIM = 128
|
||||
SQRT_HD = math.sqrt(HEAD_DIM)
|
||||
NATIVE = 43.7840518911
|
||||
FOLDED = NATIVE / SQRT_HD # 3.8700...
|
||||
|
||||
|
||||
class Cfg:
|
||||
"""Attention-config stand-in carrying only the fields under test."""
|
||||
|
||||
def __init__(self, **kw):
|
||||
self.head_dim = HEAD_DIM
|
||||
for k, v in kw.items():
|
||||
setattr(self, k, v)
|
||||
|
||||
|
||||
# ------------------------------------------------------------ flat vs nested
|
||||
|
||||
|
||||
def test_flat_config_values_respected():
|
||||
c = MuseGlimmerConfig(**FLAT)
|
||||
t = c.text_config
|
||||
assert t.hidden_size == 6656
|
||||
assert t.num_hidden_layers == 52
|
||||
assert t.vocab_size == 202048
|
||||
assert t.head_dim == 128
|
||||
assert t.hidden_activation == "silu" # renamed from hidden_act
|
||||
assert t.final_logit_softcapping == 20.0 # renamed from output_soft_cap_temp
|
||||
assert abs(t.output_multiplier - 0.19611613513818404) < 1e-12
|
||||
assert abs(t.qk_scale_factor - 43.7840518911) < 1e-9
|
||||
assert t.rope_parameters["rope_theta"] == 500000.0 # from flat rope_theta
|
||||
# vision hoisted + renamed
|
||||
assert c.vision_config.hidden_size == 1536
|
||||
assert c.vision_config.num_hidden_layers == 50
|
||||
assert c.vision_config.output_dim == 6144
|
||||
# flat patch_token_id -> image_token_id
|
||||
assert c.image_token_id == 200092
|
||||
|
||||
|
||||
def test_flat_config_no_silent_default():
|
||||
# The regression: a non-default value MUST be honored, not silently dropped.
|
||||
flat = dict(FLAT)
|
||||
flat["hidden_size"] = 4096
|
||||
flat["num_hidden_layers"] = 40
|
||||
c = MuseGlimmerConfig(**flat)
|
||||
assert c.text_config.hidden_size == 4096, "flat hidden_size silently ignored!"
|
||||
assert c.text_config.num_hidden_layers == 40, "flat num_hidden_layers ignored!"
|
||||
|
||||
|
||||
def test_nested_config_unchanged():
|
||||
c = MuseGlimmerConfig(**NESTED)
|
||||
assert c.text_config.hidden_size == 6656
|
||||
assert c.text_config.hidden_activation == "silu"
|
||||
assert c.text_config.final_logit_softcapping == 20.0
|
||||
assert c.vision_config.hidden_size == 1536
|
||||
assert c.vision_config.num_hidden_layers == 50
|
||||
assert c.image_token_id == 200092
|
||||
|
||||
|
||||
# -------------------------------------------------------- native vs modular
|
||||
|
||||
|
||||
def test_qk_norm_missing_defaults_true():
|
||||
assert _muse_glimmer_use_qk_norm(Cfg(use_qk_norm=None)) is True # modular
|
||||
assert _muse_glimmer_use_qk_norm(Cfg()) is True # absent
|
||||
assert _muse_glimmer_use_qk_norm(Cfg(use_qk_norm=True)) is True # native
|
||||
assert _muse_glimmer_use_qk_norm(Cfg(use_qk_norm=False)) is False # explicit off
|
||||
|
||||
|
||||
def test_output_gate_missing_defaults_true():
|
||||
assert _muse_glimmer_use_attn_output_gate(Cfg(use_attn_output_gate=None)) is True
|
||||
assert _muse_glimmer_use_attn_output_gate(Cfg()) is True
|
||||
assert _muse_glimmer_use_attn_output_gate(Cfg(use_attn_output_gate=False)) is False
|
||||
|
||||
|
||||
def test_query_prescale_native_and_modular_converge():
|
||||
# Both schemas must yield the SAME final scale_query_by (~3.87).
|
||||
assert (
|
||||
abs(_muse_glimmer_query_prescale(Cfg(qk_scale_factor=NATIVE)) - FOLDED) < 1e-9
|
||||
)
|
||||
assert (
|
||||
abs(_muse_glimmer_query_prescale(Cfg(qk_scale_factor=FOLDED)) - FOLDED) < 1e-9
|
||||
)
|
||||
|
||||
|
||||
def test_query_prescale_explicit_wins():
|
||||
c = Cfg(scale_query_by=FOLDED, qk_scale_factor=NATIVE)
|
||||
assert abs(_muse_glimmer_query_prescale(c) - FOLDED) < 1e-9
|
||||
Reference in New Issue
Block a user