Add Muse Glimmer model support (#51655)

Signed-off-by: Tiezhen Wang <tiezhen@inferact.ai>
Signed-off-by: Jee Jee Li <jeejeelee@inferact.ai>
Signed-off-by: zixi-qi <zixi@inferact.ai>
Co-authored-by: Beto de Paola <betodepaola@meta.com>
Co-authored-by: Giancarlo Delfin <gdelfin@inferact.ai>
Co-authored-by: Jee Jee Li <jeejeelee@inferact.ai>
Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
Co-authored-by: Jee Jee Li <pandaleefree@gmail.com>
Co-authored-by: zixi-qi <zixi@inferact.ai>
This commit is contained in:
Tiezhen WANG
2026-08-14 12:28:56 +08:00
committed by GitHub
co-authored by Beto de Paola Giancarlo Delfin Jee Jee Li Claude Opus 5 Jee Jee Li zixi-qi
parent 59d1af5698
commit 6adad08767
21 changed files with 4333 additions and 16 deletions
+18
View File
@@ -1192,6 +1192,12 @@ _MULTIMODAL_EXAMPLE_MODELS = {
# required by current PrefixLM implementation
max_num_batched_tokens=31872,
),
"MuseGlimmerForConditionalGeneration": _HfExamplesInfo(
"meta-models/Muse-Glimmer-30B",
),
"MuseGlimmerForCausalLM": _HfExamplesInfo(
"meta-models/Muse-Glimmer-30B",
),
"NVLM_D": _HfExamplesInfo("nvidia/NVLM-D-72B", trust_remote_code=True),
"Llama_Nemotron_Nano_VL": _HfExamplesInfo(
"nvidia/Llama-3.1-Nemotron-Nano-VL-8B-V1",
@@ -1489,6 +1495,18 @@ _SPECULATIVE_DECODING_EXAMPLE_MODELS = {
max_num_seqs=32,
min_transformers_version="4.56.3", # Required for Qwen3Next
),
"MuseGlimmerAssistantModel": _HfExamplesInfo(
"meta-models/Muse-Glimmer-30B",
speculative_model="meta-models/Muse-Glimmer-30B-assistant",
max_model_len=8192, # Reduce max len to ensure test runs in low-VRAM CI env
max_num_seqs=32,
),
"DFlashMuseGlimmerAssistantModel": _HfExamplesInfo(
"meta-models/Muse-Glimmer-30B",
speculative_model="meta-models/Muse-Glimmer-30B-assistant",
max_model_len=8192, # Reduce max len to ensure test runs in low-VRAM CI env
max_num_seqs=32,
),
# [DSpark]
"DSparkDraftModel": _HfExamplesInfo(
"deepseek-ai/DeepSeek-V4-Pro-DSpark",
+7 -2
View File
@@ -564,15 +564,20 @@ def dummy_hf_overrides(
)
if hasattr(hf_config, "vision_config"):
hf_config.vision_config.update(
vision_config = hf_config.vision_config
vision_config.update(
{
"num_layers": 1,
"num_hidden_layers": 1,
}
)
# Keep per-layer metadata consistent with the reduced layer count.
if layer_types := getattr(vision_config, "layer_types", None):
vision_config.update({"layer_types": layer_types[:1]})
if model_arch in ("Moondream3ForCausalLM", "HfMoondream"):
hf_config.vision_config.update({"enc_n_layers": 1})
vision_config.update({"enc_n_layers": 1})
# e.g.: ibm-granite/granite-speech-3.3-2b
if hasattr(hf_config, "encoder_config"):
+386
View File
@@ -0,0 +1,386 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Unit tests for the MuseGlimmer ATEM tool parser and reasoning parser.
MuseGlimmer writes every turn as a sequence of channel-scoped messages rather
than JSON, so the two parsers are tested together: the reasoning parser strips
the reasoning span and forwards the remaining channels as content, and the tool
parser reads ATEM markup out of those channels.
Four areas, in order:
1. non-streaming tool-call extraction, including channel scoping (an
``<atem:invoke>`` echoed inside reasoning must never become a call);
2. the reasoning -> tool-parser handoff, which regressed once by returning
``content=None`` and starving the tool parser;
3. streaming, where markers routinely straddle chunk boundaries, plus
truncation isolation for an unterminated ``to=self`` block;
4. tool-name normalization against the tools registered on the request.
These drive the parsers directly and need no checkpoint. The tests that require
a real tokenizer live in ``test_muse_glimmer_parse_delta.py``.
"""
import json
from types import SimpleNamespace
import pytest
from vllm.reasoning.muse_glimmer_reasoning_parser import MuseGlimmerReasoningParser
from vllm.tool_parsers.muse_glimmer_tool_parser import MuseGlimmerToolParser
R: MuseGlimmerReasoningParser
T: MuseGlimmerToolParser
@pytest.fixture(autouse=True)
def _fresh_parsers():
"""Give each test request-scoped parser state through the real constructors."""
global R, T
R = MuseGlimmerReasoningParser(object())
T = MuseGlimmerToolParser(object())
# Any framing token that must NEVER appear in surfaced reasoning/content.
_FRAMING = [
"<|start|>",
"<|message|>",
"<|eom|>",
"<|eot|>",
"to=self",
"to=user",
"to=read.read",
"<atem:",
]
class _FakeReq:
"""Minimal ChatCompletionRequest stand-in (no registered tools)."""
tools = None
def _req(*names):
"""A request with ``names`` registered as tools."""
return SimpleNamespace(
tools=[SimpleNamespace(function=SimpleNamespace(name=n)) for n in names]
)
def _call(name):
"""One tool-call turn invoking ``name``."""
return (
f"<|start|>assistant to={name}<|message|>"
f'<atem:function_calls>\n<atem:invoke name="{name}">\n'
f'<atem:parameter name="city">Paris</atem:parameter>\n'
f"</atem:invoke>\n</atem:function_calls>"
)
# ---------------------------------------------------------------- tool calls
def test_single_tool_call_after_reasoning():
raw = (
"to=self<|message|>Let me check the weather.<|eom|>"
"<|start|>assistant to=weather.get<|message|>"
'<atem:function_calls>\n<atem:invoke name="weather.get">\n'
'<atem:parameter name="city">Paris</atem:parameter>\n'
'<atem:parameter name="units">celsius</atem:parameter>\n'
"</atem:invoke>\n</atem:function_calls><|eot|>"
)
out = MuseGlimmerToolParser.extract_tool_calls(T, raw, None)
assert out.tools_called and len(out.tool_calls) == 1
assert out.tool_calls[0].function.name == "weather.get"
assert json.loads(out.tool_calls[0].function.arguments) == {
"city": "Paris",
"units": "celsius",
}
def test_parallel_calls_across_eom_boundaries():
raw = (
"<|start|>assistant to=math.add<|message|>"
'<atem:function_calls>\n<atem:invoke name="math.add">\n'
'<atem:parameter name="a">1</atem:parameter>\n'
'<atem:parameter name="b">2</atem:parameter>\n'
"</atem:invoke>\n</atem:function_calls><|eom|>"
"<|start|>assistant to=math.mul<|message|>"
'<atem:function_calls>\n<atem:invoke name="math.mul">\n'
'<atem:parameter name="a">3</atem:parameter>\n'
'<atem:parameter name="b">4</atem:parameter>\n'
"</atem:invoke>\n</atem:function_calls><|eot|>"
)
out = MuseGlimmerToolParser.extract_tool_calls(T, raw, None)
assert out.tools_called and len(out.tool_calls) == 2, len(out.tool_calls)
assert [t.function.name for t in out.tool_calls] == ["math.add", "math.mul"]
# JSON-typed values decode to ints
assert json.loads(out.tool_calls[0].function.arguments) == {"a": 1, "b": 2}
def test_echoed_invoke_in_reasoning_is_not_parsed():
"""Channel scoping: an invoke quoted inside reasoning is not a call."""
raw = (
'to=self<|message|>I could call <atem:invoke name="evil.fn">'
'<atem:parameter name="x">1</atem:parameter></atem:invoke> '
"but I will not.<|eom|>"
"<|start|>assistant to=user<|message|>The answer is 42.<|eot|>"
)
out = MuseGlimmerToolParser.extract_tool_calls(T, raw, None)
assert not out.tools_called, "channel scoping failed -- echoed invoke parsed!"
assert out.content == "The answer is 42.", repr(out.content)
def test_plain_answer_yields_no_tool_calls():
out = MuseGlimmerToolParser.extract_tool_calls(
T, "to=user<|message|>Just a plain answer.<|eot|>", None
)
assert not out.tools_called
def test_json_object_array_and_bool_params_decode():
raw = (
"<|start|>assistant to=api.call<|message|>"
'<atem:function_calls>\n<atem:invoke name="api.call">\n'
'<atem:parameter name="payload">{"nested": [1, 2, 3]}</atem:parameter>\n'
'<atem:parameter name="flag">true</atem:parameter>\n'
"</atem:invoke>\n</atem:function_calls><|eot|>"
)
out = MuseGlimmerToolParser.extract_tool_calls(T, raw, None)
assert json.loads(out.tool_calls[0].function.arguments) == {
"payload": {"nested": [1, 2, 3]},
"flag": True,
}
# ------------------------------------------------- reasoning -> tool handoff
def test_reasoning_to_toolcall_handoff():
"""The regression: content=None here starved the tool parser."""
raw = (
" to=self<|message|>Let me call the tool.<|eom|>"
"<|start|>assistant to=weather.get<|message|>"
'<atem:function_calls>\n<atem:invoke name="weather.get">\n'
'<atem:parameter name="city">Paris</atem:parameter>\n'
"</atem:invoke>\n</atem:function_calls>"
)
reasoning, content = MuseGlimmerReasoningParser.extract_reasoning(R, raw, None)
assert reasoning == "Let me call the tool.", repr(reasoning)
assert content is not None and "<atem:invoke" in content, repr(content)
out = MuseGlimmerToolParser.extract_tool_calls(T, content, None)
assert out.tools_called and len(out.tool_calls) == 1
assert out.tool_calls[0].function.name == "weather.get"
def test_reasoning_then_user_answer():
raw = (
" to=self<|message|>thinking<|eom|>"
"<|start|>assistant to=user<|message|>The answer is 42.<|eot|>"
)
reasoning, content = MuseGlimmerReasoningParser.extract_reasoning(R, raw, None)
assert reasoning == "thinking", repr(reasoning)
assert content == "The answer is 42.", repr(content)
assert not MuseGlimmerToolParser.extract_tool_calls(T, content, None).tools_called
def test_plain_content_without_framing_passes_through():
reasoning, content = MuseGlimmerReasoningParser.extract_reasoning(
R, "Just a direct answer.", None
)
assert reasoning is None and content == "Just a direct answer.", (
reasoning,
content,
)
def test_reasoning_then_parallel_calls():
raw = (
" to=self<|message|>need two calls<|eom|>"
"<|start|>assistant to=math.add<|message|>"
'<atem:function_calls>\n<atem:invoke name="math.add">\n'
'<atem:parameter name="a">1</atem:parameter>\n</atem:invoke>\n'
"</atem:function_calls><|eom|>"
"<|start|>assistant to=math.mul<|message|>"
'<atem:function_calls>\n<atem:invoke name="math.mul">\n'
'<atem:parameter name="a">3</atem:parameter>\n</atem:invoke>\n'
"</atem:function_calls><|eot|>"
)
reasoning, content = MuseGlimmerReasoningParser.extract_reasoning(R, raw, None)
assert reasoning == "need two calls", repr(reasoning)
out = MuseGlimmerToolParser.extract_tool_calls(T, content, None)
assert [t.function.name for t in out.tool_calls] == ["math.add", "math.mul"], (
out.tool_calls
)
# ----------------------------------------------------------------- streaming
def _stream(raw: str, chunk: int):
"""Feed ``raw`` incrementally in ``chunk``-char steps through BOTH streaming
parsers; return (reasoning, content, tool_calls)."""
reasoning, content, toolcalls = [], [], []
prev = ""
i = 0
while i < len(raw):
cur = raw[: i + chunk]
delta = cur[len(prev) :]
dm = MuseGlimmerReasoningParser.extract_reasoning_streaming(
R, prev, cur, delta, [], [], []
)
if dm is not None:
if getattr(dm, "reasoning", None):
reasoning.append(dm.reasoning)
content_delta = getattr(dm, "content", None)
# Tool-channel content is an internal handoff to the tool parser,
# not client-visible content from the unified parser.
if content_delta and "<atem:function_calls>" not in content_delta:
content.append(content_delta)
dt = MuseGlimmerToolParser.extract_tool_calls_streaming(
T, prev, cur, delta, [], [], [], _FakeReq()
)
if dt is not None and dt.tool_calls:
toolcalls.extend(dt.tool_calls)
prev = cur
i += chunk
return "".join(reasoning), "".join(content), toolcalls
def _fn_of(tc):
fn = tc.function
if isinstance(fn, dict):
return fn.get("name"), fn.get("arguments")
return fn.name, fn.arguments
RAW_TOOLCALL = (
" to=self<|message|>I should read the hostname file to answer.<|eom|>"
"<|start|>assistant to=read.read<|message|>"
'<atem:function_calls>\n<atem:invoke name="read.read">\n'
'<atem:parameter name="path">/etc/hostname</atem:parameter>\n'
"</atem:invoke>\n</atem:function_calls>"
)
RAW_ANSWER = (
" to=self<|message|>Think about it.<|eom|>"
"<|start|>assistant to=user<|message|>The answer is 42.<|eot|>"
)
# NO closing <|eom|> -> truncated CoT
RAW_TRUNCATED = (
" to=self<|message|>Maybe I should call "
'<atem:function_calls>\n<atem:invoke name="read.read">\n'
'<atem:parameter name="path">/etc/hostname</atem:parameter>\n'
"</atem:invoke>\n</atem:function_calls> but wait"
)
def _check_toolcall_stream(chunk):
reasoning, content, tcs = _stream(RAW_TOOLCALL, chunk)
# (a) no framing token leaks into reasoning or content
for f in _FRAMING:
assert f not in reasoning, (
f"framing {f!r} leaked into reasoning (chunk={chunk})"
)
assert f not in content, f"framing {f!r} leaked into content (chunk={chunk})"
# (b) exactly one tool_call with correct name + args
assert len(tcs) == 1, f"expected 1 tool_call, got {len(tcs)} (chunk={chunk})"
name, args = _fn_of(tcs[0])
assert name == "read.read", name
assert json.loads(args) == {"path": "/etc/hostname"}, args
assert tcs[0].index == 0 and tcs[0].type == "function" and tcs[0].id
# (c) reasoning captured separately and clean
assert reasoning == "I should read the hostname file to answer.", repr(reasoning)
assert content == "", repr(content)
def test_streaming_toolcall_chunk3():
_check_toolcall_stream(3)
def test_streaming_toolcall_charwise():
# worst case: markers arrive one char at a time (mid-marker deltas)
_check_toolcall_stream(1)
def test_streaming_toolcall_bigchunks():
_check_toolcall_stream(17)
def test_streaming_reasoning_then_content():
reasoning, content, tcs = _stream(RAW_ANSWER, 3)
for f in _FRAMING:
assert f not in reasoning and f not in content, f
assert reasoning == "Think about it.", repr(reasoning)
assert content == "The answer is 42.", repr(content)
assert tcs == []
def test_truncated_cot_no_toolcall_nonstreaming():
out = MuseGlimmerToolParser.extract_tool_calls(T, RAW_TRUNCATED, _FakeReq())
assert not out.tools_called and out.tool_calls == []
# partial reasoning must still be recovered by the reasoning parser
reasoning, _ = MuseGlimmerReasoningParser.extract_reasoning(
R, RAW_TRUNCATED, _FakeReq()
)
assert reasoning and "Maybe I should call" in reasoning, repr(reasoning)
def test_truncated_cot_no_toolcall_streaming():
_, _, tcs = _stream(RAW_TRUNCATED, 3)
assert tcs == [], f"truncated CoT invoke leaked as streaming tool call: {tcs}"
# ------------------------------------------------------- name normalization
#
# MuseGlimmer emits `get_weather.get_weather` for a bare-registered
# `get_weather`, and `weather.get` verbatim for a namespaced one. The parser
# normalizes against the tools actually registered on the request.
def test_doubled_bare_name_collapses():
out = MuseGlimmerToolParser.extract_tool_calls(
T, _call("get_weather.get_weather"), _req("get_weather")
)
assert out.tools_called and out.tool_calls[0].function.name == "get_weather", (
out.tool_calls[0].function.name
)
def test_namespaced_name_preserved():
out = MuseGlimmerToolParser.extract_tool_calls(
T, _call("weather.get"), _req("weather.get")
)
assert out.tool_calls[0].function.name == "weather.get"
def test_unregistered_namespace_is_preserved():
# Suffix-only matching can silently dispatch a tool from the wrong namespace.
out = MuseGlimmerToolParser.extract_tool_calls(
T, _call("foo.get_weather"), _req("get_weather")
)
assert out.tool_calls[0].function.name == "foo.get_weather"
def test_trailing_segment_ambiguous_left_alone():
# two registered tools share leaf 'get' -> ambiguous -> do NOT rewrite
out = MuseGlimmerToolParser.extract_tool_calls(
T, _call("x.get"), _req("weather.get", "time.get")
)
assert out.tool_calls[0].function.name == "x.get"
def test_no_registered_tools_passthrough():
out = MuseGlimmerToolParser.extract_tool_calls(
T, _call("get_weather.get_weather"), None
)
assert out.tool_calls[0].function.name == "get_weather.get_weather"
def test_exact_match_kept():
out = MuseGlimmerToolParser.extract_tool_calls(
T, _call("get_weather"), _req("get_weather")
)
assert out.tool_calls[0].function.name == "get_weather"
@@ -0,0 +1,187 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""parse_delta-level regression tests for the MuseGlimmer parsers.
These drive the UNIFIED parser-engine streaming API the serving layer actually
uses (``DelegatingParser.parse_delta``), NOT the ``extract_*_streaming`` methods
directly. This is the coverage that was missing when a live no-tools streaming
request leaked raw harmony framing into ``content``: the phase machine marked
``reasoning_ended=True`` at the prompt boundary (because ``is_reasoning_end`` on
a prompt with no ``to=self`` wrongly returned True), skipping the reasoning
phase for the whole generation.
Requires a real MuseGlimmer tokenizer; skipped if the checkpoint is unavailable.
"""
import json
import os
import pytest
CKPT = os.environ.get("MUSE_GLIMMER_CKPT", "")
pytestmark = pytest.mark.skipif(
not os.path.isdir(CKPT), reason=f"MuseGlimmer checkpoint not found at {CKPT}"
)
_FRAMING = [
"<|start|>",
"<|message|>",
"<|eom|>",
"<|eot|>",
"to=self",
"to=user",
"<atem:",
]
PROMPT = "<|start|>user<|message|>hi<|eom|><|start|>assistant"
class _Req:
tools = None
tool_choice = None
include_reasoning = True
@pytest.fixture(scope="module")
def tok():
from transformers import AutoTokenizer
return AutoTokenizer.from_pretrained(CKPT, trust_remote_code=True)
@pytest.fixture(scope="module")
def parser_cls():
from vllm.parser import ParserManager
return ParserManager.get_parser(
tool_parser_name="muse_glimmer",
reasoning_parser_name="muse_glimmer",
enable_auto_tools=True,
)
def _drive(parser_cls, tok, gen_text, req=None):
"""Feed gen_text token-by-token through parse_delta (as serving does)."""
from vllm.parser.abstract_parser import StreamState
if req is None:
req = _Req()
parser = parser_cls(tok)
parser._stream_state = StreamState()
prompt_ids = tok.encode(PROMPT, add_special_tokens=False)
gen_ids = tok.encode(gen_text, add_special_tokens=False)
reasoning, content, tools = [], [], []
for i, tid in enumerate(gen_ids):
dm = parser.parse_delta(
tok.decode([tid]),
[tid],
req,
prompt_token_ids=prompt_ids if i == 0 else None,
finished=(i == len(gen_ids) - 1),
)
if dm is None:
continue
if getattr(dm, "reasoning", None):
reasoning.append(dm.reasoning)
if getattr(dm, "content", None):
content.append(dm.content)
for tc in getattr(dm, "tool_calls", None) or []:
fn = tc.function
name = fn.get("name") if isinstance(fn, dict) else getattr(fn, "name", None)
args = (
fn.get("arguments")
if isinstance(fn, dict)
else getattr(fn, "arguments", None)
)
tools.append((tc.index, name, args))
return "".join(reasoning), "".join(content), tools
def _assert_no_framing(text):
for f in _FRAMING:
assert f not in text, f"framing {f!r} leaked: {text!r}"
def test_parse_delta_no_tools_reasoning_then_answer(parser_cls, tok):
# The core regression: reasoning phase must stay active for a no-tools turn.
reasoning, content, tools = _drive(
parser_cls,
tok,
" to=self<|message|>Let me think step by step about the sum.<|eom|>"
"<|start|>assistant to=user<|message|>The answer is 42.<|eot|>",
)
_assert_no_framing(content)
assert reasoning == "Let me think step by step about the sum.", repr(reasoning)
assert content == "The answer is 42.", repr(content)
assert tools == []
def test_parse_delta_content_only(parser_cls, tok):
reasoning, content, tools = _drive(
parser_cls,
tok,
" to=user<|message|>Just a direct answer.<|eot|>",
)
_assert_no_framing(content)
assert content == "Just a direct answer.", repr(content)
assert tools == []
def test_parse_delta_tool_call(parser_cls, tok):
reasoning, content, tools = _drive(
parser_cls,
tok,
" to=self<|message|>I should read the hostname.<|eom|>"
"<|start|>assistant to=read.read<|message|>"
'<atem:function_calls>\n<atem:invoke name="read.read">\n'
'<atem:parameter name="path">/etc/hostname</atem:parameter>\n'
"</atem:invoke>\n</atem:function_calls>",
)
_assert_no_framing(content)
assert reasoning == "I should read the hostname.", repr(reasoning)
assert len(tools) == 1, tools
idx, name, args = tools[0]
assert idx == 0 and name == "read.read"
assert json.loads(args) == {"path": "/etc/hostname"}
def test_parse_delta_truncated_cot_no_toolcall(parser_cls, tok):
# Contemplated invoke inside an unterminated to=self block: no tool call,
# no framing leak into content, partial reasoning recovered.
reasoning, content, tools = _drive(
parser_cls,
tok,
" to=self<|message|>Maybe I should call "
'<atem:function_calls>\n<atem:invoke name="read.read">\n'
'<atem:parameter name="path">/etc/hostname</atem:parameter>\n'
"</atem:invoke>\n</atem:function_calls> but wait",
)
_assert_no_framing(content)
assert tools == [], f"contemplated invoke leaked as tool call: {tools}"
assert "Maybe I should call" in reasoning, repr(reasoning)
def test_parse_delta_reasoning_suppressed_when_not_requested(parser_cls, tok):
class _NoReasonReq:
tools = None
tool_choice = None
include_reasoning = False
reasoning, content, tools = _drive(
parser_cls,
tok,
" to=self<|message|>secret thoughts<|eom|>"
"<|start|>assistant to=user<|message|>Public answer.<|eot|>",
req=_NoReasonReq(),
)
assert reasoning == "", repr(reasoning) # suppressed
assert content == "Public answer.", repr(content)
_assert_no_framing(content)
if __name__ == "__main__":
import sys
sys.exit(pytest.main([__file__, "-v"]))
@@ -0,0 +1,174 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""MuseGlimmer config normalization.
Two schema pairs have to converge on the same model, and each has produced a
silent-wrong-output bug:
1. flat (legacy converter) vs nested (canonical) ``config.json``. A FLAT
config once deserialized to an all-default text config, silently dropping
every checkpoint value.
2. native vs modular attention config. The modular HF text_config OMITS
use_qk_norm / use_attn_output_gate (read as None) and ships a PRE-FOLDED
qk_scale_factor (43.784/sqrt(128)=3.87). Missing flags must read as True
(MuseGlimmer always applies QK-norm + output gate) and the query pre-scale
must normalize so native and modular land on the same number.
"""
import math
from typing import Any
from vllm.model_executor.models.muse_glimmer import (
_muse_glimmer_query_prescale,
_muse_glimmer_use_attn_output_gate,
_muse_glimmer_use_qk_norm,
)
from vllm.transformers_utils.configs.muse_glimmer import MuseGlimmerConfig
# A representative FLAT config (Ruan rl_v1/hf shape), trimmed.
FLAT: dict[str, Any] = {
"architectures": ["MuseGlimmerForCausalLM"],
"model_type": "muse_glimmer",
"has_vision": True,
"bos_token_id": 200000,
"eos_token_id": 200001,
"vocab_size": 202048,
"hidden_size": 6656,
"intermediate_size": 19968,
"num_hidden_layers": 52,
"num_attention_heads": 32,
"num_key_value_heads": 2,
"head_dim": 128,
"hidden_act": "silu",
"max_position_embeddings": 16384,
"rms_norm_eps": 1e-5,
"post_norm_eps": 1e-8,
"qk_scale_factor": 43.7840518911,
"use_qk_norm": True,
"use_attn_output_gate": True,
"output_multiplier": 0.19611613513818404,
"output_soft_cap_temp": 20.0,
"normalize_tok_embeddings": True,
"rope_theta": 500000.0,
"sliding_window": 2048,
"patch_token_id": 200092,
"vision_latent_dim": 1536,
"vision_heads": 16,
"vision_layers": 50,
"vision_output_dim": 6144,
"vision_patch_size": 14,
"vision_patch_temporal": 2,
"vision_adapter_dim": 4096,
"vision_pos_emb_grid_h": 32,
"vision_pos_emb_grid_w": 32,
}
NESTED: dict[str, Any] = {
"architectures": ["MuseGlimmerForCausalLM"],
"model_type": "muse_glimmer",
"image_token_id": 200092,
"text_config": {
"model_type": "muse_glimmer_text",
"vocab_size": 202048,
"hidden_size": 6656,
"num_hidden_layers": 52,
"hidden_activation": "silu",
"final_logit_softcapping": 20.0,
"qk_scale_factor": 43.7840518911,
"rope_parameters": {"rope_type": "default", "rope_theta": 500000.0},
},
"vision_config": {
"model_type": "muse_glimmer_vision",
"hidden_size": 1536,
"num_hidden_layers": 50,
},
}
HEAD_DIM = 128
SQRT_HD = math.sqrt(HEAD_DIM)
NATIVE = 43.7840518911
FOLDED = NATIVE / SQRT_HD # 3.8700...
class Cfg:
"""Attention-config stand-in carrying only the fields under test."""
def __init__(self, **kw):
self.head_dim = HEAD_DIM
for k, v in kw.items():
setattr(self, k, v)
# ------------------------------------------------------------ flat vs nested
def test_flat_config_values_respected():
c = MuseGlimmerConfig(**FLAT)
t = c.text_config
assert t.hidden_size == 6656
assert t.num_hidden_layers == 52
assert t.vocab_size == 202048
assert t.head_dim == 128
assert t.hidden_activation == "silu" # renamed from hidden_act
assert t.final_logit_softcapping == 20.0 # renamed from output_soft_cap_temp
assert abs(t.output_multiplier - 0.19611613513818404) < 1e-12
assert abs(t.qk_scale_factor - 43.7840518911) < 1e-9
assert t.rope_parameters["rope_theta"] == 500000.0 # from flat rope_theta
# vision hoisted + renamed
assert c.vision_config.hidden_size == 1536
assert c.vision_config.num_hidden_layers == 50
assert c.vision_config.output_dim == 6144
# flat patch_token_id -> image_token_id
assert c.image_token_id == 200092
def test_flat_config_no_silent_default():
# The regression: a non-default value MUST be honored, not silently dropped.
flat = dict(FLAT)
flat["hidden_size"] = 4096
flat["num_hidden_layers"] = 40
c = MuseGlimmerConfig(**flat)
assert c.text_config.hidden_size == 4096, "flat hidden_size silently ignored!"
assert c.text_config.num_hidden_layers == 40, "flat num_hidden_layers ignored!"
def test_nested_config_unchanged():
c = MuseGlimmerConfig(**NESTED)
assert c.text_config.hidden_size == 6656
assert c.text_config.hidden_activation == "silu"
assert c.text_config.final_logit_softcapping == 20.0
assert c.vision_config.hidden_size == 1536
assert c.vision_config.num_hidden_layers == 50
assert c.image_token_id == 200092
# -------------------------------------------------------- native vs modular
def test_qk_norm_missing_defaults_true():
assert _muse_glimmer_use_qk_norm(Cfg(use_qk_norm=None)) is True # modular
assert _muse_glimmer_use_qk_norm(Cfg()) is True # absent
assert _muse_glimmer_use_qk_norm(Cfg(use_qk_norm=True)) is True # native
assert _muse_glimmer_use_qk_norm(Cfg(use_qk_norm=False)) is False # explicit off
def test_output_gate_missing_defaults_true():
assert _muse_glimmer_use_attn_output_gate(Cfg(use_attn_output_gate=None)) is True
assert _muse_glimmer_use_attn_output_gate(Cfg()) is True
assert _muse_glimmer_use_attn_output_gate(Cfg(use_attn_output_gate=False)) is False
def test_query_prescale_native_and_modular_converge():
# Both schemas must yield the SAME final scale_query_by (~3.87).
assert (
abs(_muse_glimmer_query_prescale(Cfg(qk_scale_factor=NATIVE)) - FOLDED) < 1e-9
)
assert (
abs(_muse_glimmer_query_prescale(Cfg(qk_scale_factor=FOLDED)) - FOLDED) < 1e-9
)
def test_query_prescale_explicit_wins():
c = Cfg(scale_query_by=FOLDED, qk_scale_factor=NATIVE)
assert abs(_muse_glimmer_query_prescale(c) - FOLDED) < 1e-9