Add Muse Glimmer model support (#51655)

Signed-off-by: Tiezhen Wang <tiezhen@inferact.ai>
Signed-off-by: Jee Jee Li <jeejeelee@inferact.ai>
Signed-off-by: zixi-qi <zixi@inferact.ai>
Co-authored-by: Beto de Paola <betodepaola@meta.com>
Co-authored-by: Giancarlo Delfin <gdelfin@inferact.ai>
Co-authored-by: Jee Jee Li <jeejeelee@inferact.ai>
Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
Co-authored-by: Jee Jee Li <pandaleefree@gmail.com>
Co-authored-by: zixi-qi <zixi@inferact.ai>
This commit is contained in:
Tiezhen WANG
2026-08-14 12:28:56 +08:00
committed by GitHub
co-authored by Beto de Paola Giancarlo Delfin Jee Jee Li Claude Opus 5 Jee Jee Li zixi-qi
parent 59d1af5698
commit 6adad08767
21 changed files with 4333 additions and 16 deletions
+4
View File
@@ -102,6 +102,10 @@ _CONFIG_REGISTRY: dict[str, type[PretrainedConfig]] = LazyConfigDict(
kimi_linear="KimiLinearConfig",
kimi_vl="KimiVLConfig",
kimi_k25="KimiK25Config",
muse_glimmer="MuseGlimmerConfig",
muse_glimmer_text="MuseGlimmerTextConfig",
muse_glimmer_vision="MuseGlimmerVisionConfig",
muse_glimmer_assistant="MuseGlimmerAssistantConfig",
kimi_k3="KimiK3Config",
RefinedWeb="RWConfig", # For tiiuae/falcon-40b(-instruct)
RefinedWebModel="RWConfig", # For tiiuae/falcon-7b(-instruct)
@@ -74,6 +74,10 @@ _CLASS_TO_MODULE: dict[str, str] = {
"KimiLinearConfig": "vllm.transformers_utils.configs.kimi_linear",
"KimiVLConfig": "vllm.transformers_utils.configs.kimi_vl",
"KimiK25Config": "vllm.transformers_utils.configs.kimi_k25",
"MuseGlimmerConfig": "vllm.transformers_utils.configs.muse_glimmer",
"MuseGlimmerTextConfig": "vllm.transformers_utils.configs.muse_glimmer",
"MuseGlimmerVisionConfig": "vllm.transformers_utils.configs.muse_glimmer",
"MuseGlimmerAssistantConfig": "vllm.transformers_utils.configs.muse_glimmer",
"KimiK3Config": "vllm.transformers_utils.configs.kimi_k3",
"KimiK3VisionConfig": "vllm.transformers_utils.configs.kimi_k3",
"NemotronConfig": "vllm.transformers_utils.configs.nemotron",
@@ -162,6 +166,10 @@ __all__ = [
"KimiLinearConfig",
"KimiVLConfig",
"KimiK25Config",
"MuseGlimmerConfig",
"MuseGlimmerTextConfig",
"MuseGlimmerVisionConfig",
"MuseGlimmerAssistantConfig",
"KimiK3Config",
"KimiK3VisionConfig",
"NemotronConfig",
@@ -0,0 +1,389 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""MuseGlimmer model configuration for vLLM.
Native vLLM copy of the MuseGlimmer HuggingFace configs
(``configuration_muse_glimmer.py``). MuseGlimmer's
``model_type`` is not yet registered in released transformers, so vLLM ships
this config so partners can serve MuseGlimmer checkpoints *without* trust_remote_code.
The field set and defaults are kept byte-for-byte in sync with the HF reference
so a checkpoint's ``config.json`` deserializes identically here. Both the text
and vision configs are consumed by the native multimodal serving path.
"""
from __future__ import annotations
from transformers import Qwen3Config
from transformers.configuration_utils import PretrainedConfig
def _default_no_rope_layers(num_hidden_layers: int) -> list[int]:
# iRoPE mask: NoPE every 4 layers, counted backward from the last layer.
stride = 4
return [
0 if (num_hidden_layers - 1 - i) % stride == 0 else 1
for i in range(num_hidden_layers)
]
class MuseGlimmerTextConfig(PretrainedConfig):
model_type = "muse_glimmer_text"
keys_to_ignore_at_inference = ["past_key_values"]
def __init__(
self,
vocab_size: int = 202_048,
hidden_size: int = 6656,
intermediate_size: int = 19968,
num_hidden_layers: int = 52,
num_attention_heads: int = 32,
num_key_value_heads: int = 2,
head_dim: int = 128,
hidden_activation: str = "silu",
max_position_embeddings: int = 16_384,
initializer_range: float = 0.02,
rms_norm_eps: float = 1e-5,
use_cache: bool = True,
pad_token_id: int | None = None,
eos_token_id: int | list[int] | None = 200_001,
bos_token_id: int | None = 200_000,
tie_word_embeddings: bool = False,
rope_parameters: dict | None = None,
rope_theta: float | None = None,
attention_bias: bool = False,
attention_dropout: float = 0.0,
query_pre_attn_scalar: int = 256,
sliding_window: int | None = 2048,
layer_types: list[str] | None = None,
final_logit_softcapping: float | None = 20.0,
attn_logit_softcapping: float | None = None,
use_bidirectional_attention: bool | None = None,
# MuseGlimmer-specific
qk_scale_factor: float = 43.7840518911,
use_qk_norm: bool = True,
use_attn_output_gate: bool = True,
output_multiplier: float = 0.19611613513818404,
normalize_tok_embeddings: bool = True,
post_norm_eps: float = 1e-8,
no_rope_layers: list[int] | None = None,
**kwargs,
) -> None:
self.vocab_size = vocab_size
self.hidden_size = hidden_size
self.intermediate_size = intermediate_size
self.num_hidden_layers = num_hidden_layers
self.num_attention_heads = num_attention_heads
self.num_key_value_heads = num_key_value_heads
self.head_dim = head_dim
self.hidden_activation = hidden_activation
self.max_position_embeddings = max_position_embeddings
self.initializer_range = initializer_range
self.rms_norm_eps = rms_norm_eps
self.use_cache = use_cache
self.attention_bias = attention_bias
self.attention_dropout = attention_dropout
self.query_pre_attn_scalar = query_pre_attn_scalar
self.sliding_window = sliding_window
self.final_logit_softcapping = final_logit_softcapping
self.attn_logit_softcapping = attn_logit_softcapping
self.use_bidirectional_attention = use_bidirectional_attention
# RoPE: accept either an explicit rope_parameters dict (HF 5.x) or a
# bare rope_theta; normalize to rope_parameters for vLLM's get_rope.
if rope_parameters is None:
theta = rope_theta if rope_theta is not None else 500_000.0
rope_parameters = {"rope_type": "default", "rope_theta": theta}
self.rope_parameters = rope_parameters
# vLLM reads rope_theta off the config in some codepaths.
self.rope_theta = rope_parameters.get("rope_theta", 500_000.0)
# MuseGlimmer-specific fields
self.qk_scale_factor = qk_scale_factor
self.use_qk_norm = use_qk_norm
self.use_attn_output_gate = use_attn_output_gate
self.output_multiplier = output_multiplier
self.normalize_tok_embeddings = normalize_tok_embeddings
self.post_norm_eps = post_norm_eps
self.no_rope_layers = (
no_rope_layers
if no_rope_layers is not None
else _default_no_rope_layers(num_hidden_layers)
)
if layer_types is None:
layer_types = [
"full_attention" if self.no_rope_layers[i] == 0 else "sliding_attention"
for i in range(num_hidden_layers)
]
self.layer_types = layer_types
super().__init__(
pad_token_id=pad_token_id,
bos_token_id=bos_token_id,
eos_token_id=eos_token_id,
tie_word_embeddings=tie_word_embeddings,
**kwargs,
)
class MuseGlimmerVisionConfig(PretrainedConfig):
model_type = "muse_glimmer_vision"
def __init__(
self,
patch_size: int = 14,
pos_emb_height: int = 32,
pos_emb_width: int = 32,
num_attention_heads: int = 16,
num_hidden_layers: int = 50,
hidden_size: int = 1536,
intermediate_size: int = 8960,
hidden_act: str = "gelu",
merge_kernel_size: int = 2,
rope_parameters: dict | None = None,
max_position_embeddings: int = 32 * 32,
output_dim: int = 6144,
patch_temporal: int = 2,
adapter_dim: int = 4096,
layer_norm_eps: float = 1e-5,
layer_types: list[str] | None = None,
**kwargs,
) -> None:
super().__init__(**kwargs)
self.patch_size = patch_size
self.pos_emb_height = pos_emb_height
self.pos_emb_width = pos_emb_width
self.num_attention_heads = num_attention_heads
self.num_hidden_layers = num_hidden_layers
self.hidden_size = hidden_size
self.intermediate_size = intermediate_size
self.hidden_act = hidden_act
self.merge_kernel_size = merge_kernel_size
self.rope_parameters = rope_parameters
self.max_position_embeddings = max_position_embeddings
self.output_dim = output_dim
self.patch_temporal = patch_temporal
self.adapter_dim = adapter_dim
self.layer_norm_eps = layer_norm_eps
if layer_types is None:
stride = 4
layer_types = [
"full_attention"
if (i + 1) % stride == 0 or i == num_hidden_layers - 1
else "sliding_attention"
for i in range(num_hidden_layers)
]
self.layer_types = layer_types
class MuseGlimmerConfig(PretrainedConfig):
model_type = "muse_glimmer"
sub_configs = {
"text_config": MuseGlimmerTextConfig,
"vision_config": MuseGlimmerVisionConfig,
}
# --- Flat (legacy-converter) -> canonical normalization -------------------
# MuseGlimmer checkpoints exist in two config layouts in the wild:
#
# * CANONICAL (current HF converter, transformers 5.15): nested
# ``text_config`` / ``vision_config`` sub-dicts with canonical field
# names (``hidden_activation``, ``final_logit_softcapping``,
# ``rope_parameters``, ``vision_config.hidden_size`` ...).
#
# * FLAT (older converter, e.g. Ruan's ``rl_v1/hf``, transformers 5.9):
# every field is a top-level key, with different names
# (``hidden_act``, ``output_soft_cap_temp``, ``rope_theta``,
# ``vision_latent_dim`` ...) and NO ``text_config`` nesting.
#
# Without normalization a flat config silently deserializes to an
# ALL-DEFAULT text config (every checkpoint value ignored) — a dangerous
# correctness bug: a checkpoint whose arch differs from the defaults would
# load into a wrong-shaped model with no error. So when we detect a flat
# config we hoist the flat fields into ``text_config`` / ``vision_config``
# and rename them to canonical names.
# flat text field name -> canonical MuseGlimmerTextConfig field name
_FLAT_TEXT_RENAMES = {
"hidden_act": "hidden_activation",
"output_soft_cap_temp": "final_logit_softcapping",
}
# canonical MuseGlimmerTextConfig constructor params that may appear flat
_FLAT_TEXT_KEYS = frozenset(
{
"vocab_size",
"hidden_size",
"intermediate_size",
"num_hidden_layers",
"num_attention_heads",
"num_key_value_heads",
"head_dim",
"hidden_activation",
"hidden_act",
"max_position_embeddings",
"initializer_range",
"rms_norm_eps",
"use_cache",
"tie_word_embeddings",
"rope_parameters",
"rope_theta",
"attention_bias",
"attention_dropout",
"query_pre_attn_scalar",
"sliding_window",
"layer_types",
"final_logit_softcapping",
"output_soft_cap_temp",
"attn_logit_softcapping",
"use_bidirectional_attention",
"qk_scale_factor",
"use_qk_norm",
"use_attn_output_gate",
"output_multiplier",
"normalize_tok_embeddings",
"post_norm_eps",
"no_rope_layers",
}
)
# flat vision field name -> canonical MuseGlimmerVisionConfig field name
_FLAT_VISION_RENAMES = {
"vision_latent_dim": "hidden_size",
"vision_heads": "num_attention_heads",
"vision_layers": "num_hidden_layers",
"vision_output_dim": "output_dim",
"vision_patch_size": "patch_size",
"vision_patch_temporal": "patch_temporal",
"vision_adapter_dim": "adapter_dim",
"vision_pos_emb_grid_h": "pos_emb_height",
"vision_pos_emb_grid_w": "pos_emb_width",
"vision_downsample_factor": "merge_kernel_size",
}
@classmethod
def _looks_flat(cls, kwargs: dict) -> bool:
# Flat if there is no explicit text_config but there ARE text-level
# fields at the top level (e.g. hidden_size / num_hidden_layers).
if kwargs.get("text_config") is not None:
return False
return any(k in kwargs for k in ("hidden_size", "num_hidden_layers"))
@classmethod
def _normalize_flat(cls, kwargs: dict) -> dict:
kwargs = dict(kwargs)
text: dict = {}
for key in list(kwargs.keys()):
if key in cls._FLAT_TEXT_KEYS:
canon = cls._FLAT_TEXT_RENAMES.get(key, key)
text.setdefault(canon, kwargs.pop(key))
vision_mlp_ratio = kwargs.pop("vision_mlp_ratio", None)
sparse_factor = kwargs.pop("vision_sparse_attention_factor", None)
vision: dict = {}
for key in list(kwargs.keys()):
if key in cls._FLAT_VISION_RENAMES:
vision.setdefault(cls._FLAT_VISION_RENAMES[key], kwargs.pop(key))
if vision_mlp_ratio is not None:
hidden_size = vision.get("hidden_size", 1536)
vision["intermediate_size"] = int(vision_mlp_ratio * hidden_size)
if sparse_factor is not None:
sparse_factor = int(sparse_factor)
if sparse_factor <= 0:
raise ValueError("vision_sparse_attention_factor must be positive")
num_layers = vision.get("num_hidden_layers", 50)
vision["layer_types"] = [
"full_attention"
if (layer_idx + 1) % sparse_factor == 0 or layer_idx == num_layers - 1
else "sliding_attention"
for layer_idx in range(num_layers)
]
if text:
kwargs["text_config"] = text
if vision:
kwargs["vision_config"] = vision
return kwargs
def __init__(
self,
text_config: dict | MuseGlimmerTextConfig | None = None,
vision_config: dict | MuseGlimmerVisionConfig | None = None,
image_token_id: int = 200092,
video_token_id: int = 200091,
**kwargs,
) -> None:
# Detect + fold a flat config into nested text/vision before building
# the sub-configs. (image/video token ids are handled below; a flat
# ``patch_token_id`` alias is mapped too.)
if text_config is None and self._looks_flat(kwargs):
kwargs = self._normalize_flat(kwargs)
text_config = kwargs.pop("text_config", None)
vision_config = kwargs.pop("vision_config", vision_config)
# flat image-token alias
if "patch_token_id" in kwargs and "image_token_id" not in kwargs:
image_token_id = kwargs.pop("patch_token_id")
if text_config is None:
self.text_config = MuseGlimmerTextConfig()
elif isinstance(text_config, dict):
self.text_config = MuseGlimmerTextConfig(**text_config)
else:
self.text_config = text_config
if vision_config is None:
self.vision_config = MuseGlimmerVisionConfig()
elif isinstance(vision_config, dict):
self.vision_config = MuseGlimmerVisionConfig(**vision_config)
else:
self.vision_config = vision_config
self.image_token_id = image_token_id
self.video_token_id = video_token_id
self.patch_token_id = image_token_id
super().__init__(**kwargs)
class MuseGlimmerAssistantConfig(Qwen3Config):
"""Config for the Muse Glimmer DFlash draft head.
The head is Qwen3-shaped and runs on vLLM's generic ``qwen3_dflash``
implementation, so this derives from ``Qwen3Config``. It cannot BE
``Qwen3Config``, because two of the checkpoint's values do not survive that
class:
* ``sliding_window`` is gated behind ``use_sliding_window``, which defaults
to False -- ``Qwen3Config(sliding_window=2048).sliding_window`` is None.
The checkpoint declares ``sliding_window: 2048`` and five
``sliding_attention`` layers, so the window silently disappears and the
DFlash path then raises "sliding attention requires a window size".
* ``vocab_size`` is absent from the checkpoint, so Qwen3's default of
151936 applies instead of Muse Glimmer's 202048. That one is *silent*: it
builds an all-zero ``draft_id_to_target_id`` remap, and also puts
pad/bos/eos/mask token ids out of range.
Both defaults are set here so an unmodified checkpoint loads correctly.
"""
model_type = "muse_glimmer_assistant"
def __init__(
self,
vocab_size: int = 202048,
use_sliding_window: bool = True,
dflash_config: dict | None = None,
**kwargs,
) -> None:
super().__init__(
vocab_size=vocab_size, use_sliding_window=use_sliding_window, **kwargs
)
# muse_glimmer_assistant uses non-causal attention.
self.dflash_config = {"causal": False} | (dflash_config or {})
__all__ = [
"MuseGlimmerTextConfig",
"MuseGlimmerVisionConfig",
"MuseGlimmerConfig",
"MuseGlimmerAssistantConfig",
]
@@ -0,0 +1,582 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""HuggingFace processor for the MuseGlimmer multimodal model (text + image + video).
This packages MuseGlimmer's vision preprocessing and token-span expansion the standard
HF way, so you can do:
processor = AutoProcessor.from_pretrained(hf_dir, trust_remote_code=True)
messages = [{"role": "user", "content": [
{"type": "image"},
{"type": "text", "text": "Describe this image."},
]}]
text = processor.apply_chat_template(messages, add_generation_prompt=True)
inputs = processor(text=text, images=[pil_image], return_tensors="pt")
out = model.generate(**inputs, max_new_tokens=256)
It reproduces the exact token layout the model expects:
image -> <|image_start|> + <|patch|> * N + <|image_end|>
video -> <|vid_start|>
( "Time: X.Xs" + <|video|> * P [+ <|vid_frame_separator|>] )*
+ <|vid_end|>
where N = floor(H/(patch*ds)) * floor(W/(patch*ds)) (<= 4096) for images and
P = the same grid count per temporal frame-group (<= 144) for video, with the
target (H, W) chosen by MuseGlimmerImageProcessor.compute_image_size /
MuseGlimmerVideoProcessor.compute_video_frame_size (the same grid logic as
modeling_muse_glimmer.MuseGlimmerVisionEncoder).
The chat template emits one sentinel per media item -- ``<|image|>`` for images,
``<|video|>`` for videos -- which ``MuseGlimmerProcessor.__call__`` expands into the
spans above using each media item's computed token count. Vision features are
returned in ``pixel_values`` (a list of per-image / per-frame-group tensors) in
the order the sentinels appear, matching how ``MuseGlimmerModel`` merges them at the
``<|patch|>`` / ``<|video|>`` positions.
"""
from __future__ import annotations
import itertools
import math
from pathlib import Path
import torch
from PIL import Image
from torchvision import transforms as T
from transformers.feature_extraction_utils import BatchFeature
from transformers.image_processing_utils import BaseImageProcessor
from transformers.processing_utils import ProcessorMixin
from transformers.video_processing_utils import BaseVideoProcessor
# Single-token sentinels emitted by the chat template, one per media item.
IMAGE_SENTINEL = "<|image|>" # id 200090; absent from the image/video spans
VIDEO_SENTINEL = "<|video|>" # id 200091; also the per-frame token (see __call__)
# Jinja chat template (multimodal superset of convert_muse_glimmer_to_hf.CHAT_TEMPLATE).
# Message content may be a plain string OR a list of parts
# ({"type": "text"|"image"|"video", ...}); image/video parts render as a single
# sentinel that __call__ expands.
MUSE_GLIMMER_MM_CHAT_TEMPLATE = (
"{{- bos_token -}}"
"{%- macro render_parts(content) -%}"
"{%- if content is string -%}{{- content -}}"
"{%- else -%}"
"{%- for part in content -%}"
"{%- if part['type'] == 'image' -%}{{- '<|image|>' -}}"
"{%- elif part['type'] == 'video' -%}{{- '<|video|>' -}}"
"{%- elif part['type'] == 'text' -%}{{- part['text'] -}}"
"{%- endif -%}"
"{%- endfor -%}"
"{%- endif -%}"
"{%- endmacro -%}"
# At inference (add_generation_prompt), inject a default system message when
# the caller supplied none. Full-transcript rendering
# (add_generation_prompt=False) is left unchanged.
"{%- set ns = namespace(has_system=false) -%}"
"{%- for m in messages -%}"
"{%- if m['role'] == 'system' -%}{%- set ns.has_system = true -%}{%- endif -%}"
"{%- endfor -%}"
"{%- if add_generation_prompt and not ns.has_system -%}"
"{{- '<|start|>system<|message|>You are a helpful assistant.<|eot|>' -}}"
"{%- endif -%}"
"{%- for message in messages -%}"
"{%- set role = message['role'] -%}"
"{%- if role == 'assistant' -%}"
"{%- set recipient = message.get('recipient') -%}"
"{%- set end_turn = message.get('end_turn') -%}"
"{%- if end_turn is none -%}"
"{%- set end_turn = not (recipient and recipient != 'user') -%}"
"{%- endif -%}"
"{{- '<|start|>assistant' -}}"
"{%- if recipient -%}{{- ' to=' + recipient -}}{%- endif -%}"
"{{- '<|message|>' -}}{{- render_parts(message['content']) -}}"
"{{- ('<|eot|>' if end_turn else '<|eom|>') -}}"
"{%- elif role == 'tool' -%}"
"{%- set name = message.get('name', '') -%}"
# Tool content is emitted as-is (string body or interleaved image/text parts).
# Any <tool_output ...> wrapper is baked into the SFT data content; the
# tokenizer does not add it, so the template must not either.
"{{- '<|start|>tool ' + name + '<|message|>' -}}"
"{{- render_parts(message['content']) -}}"
"{{- '<|eot|>' -}}"
"{%- else -%}"
"{%- set header = role -%}"
"{%- if message.get('name') -%}"
"{%- set header = role + ' ' + message['name'] -%}"
"{%- endif -%}"
"{{- '<|start|>' + header + '<|message|>' -}}"
"{{- render_parts(message['content']) -}}"
"{{- '<|eot|>' -}}"
"{%- endif -%}"
"{%- endfor -%}"
"{%- if add_generation_prompt -%}{{- '<|start|>assistant' -}}{%- endif -%}"
)
def _grid_size(
img_w: int, img_h: int, patch_hw: int, max_tokens: int
) -> tuple[int, int, int]:
"""Pick the integer (H, W) grid closest to the aspect ratio under the token cap.
Replicates MuseGlimmerVisionEncoder._compute_grid_size
(modeling_muse_glimmer.py) so the processor needs no torch model import.
Returns (target_h, target_w, n_tokens).
"""
i_nph = img_h / patch_hw
i_npw = img_w / patch_hw
ratio = i_npw / i_nph if i_nph > 0 else 1.0
if i_nph * i_npw > max_tokens:
i_nph = (max_tokens / ratio) ** 0.5
i_npw = i_nph * ratio
candidates = list(
set(
itertools.product(
[math.floor(i_nph), math.ceil(i_nph)],
[math.floor(i_npw), math.ceil(i_npw)],
)
)
)
candidates = [
(nph, npw)
for nph, npw in candidates
if nph >= 1 and npw >= 1 and nph * npw <= max_tokens
]
if not candidates:
candidates = [(max(1, round(i_nph)), max(1, round(i_npw)))]
nph, npw = min(candidates, key=lambda c: abs(c[0] / c[1] - img_h / img_w))
return nph * patch_hw, npw * patch_hw, nph * npw
class MuseGlimmerImageProcessor(BaseImageProcessor):
"""Resize + normalize MuseGlimmer images and compute patch-token counts.
Variable-resolution: each image is resized to the grid that best matches its
aspect ratio under the per-image token cap, then normalized with mean/std 0.5.
Returns per-image tensors (not stacked) because MuseGlimmer consumes a list of
variable-size images. Video frames are handled by MuseGlimmerVideoProcessor.
"""
model_input_names = ["pixel_values"]
def __init__(
self,
patch_size: int = 14,
downsample_factor: int = 2,
max_image_tokens: int = 4096,
image_mean: float = 0.5,
image_std: float = 0.5,
**kwargs,
):
super().__init__(**kwargs)
self.patch_size = patch_size
self.downsample_factor = downsample_factor
self.max_image_tokens = max_image_tokens
self.image_mean = image_mean
self.image_std = image_std
def _to_norm_tensor(self, image: Image.Image) -> torch.Tensor:
# Functional normalize -- no stored Normalize object, so the processor
# stays JSON-serializable for to_dict() / save_pretrained.
return T.functional.normalize(
T.functional.to_tensor(image),
[self.image_mean] * 3,
[self.image_std] * 3,
)
# -- size computation (mirrors modeling_muse_glimmer.MuseGlimmerVisionEncoder) --
def compute_image_size(self, img_w: int, img_h: int) -> tuple[int, int, int]:
ph = self.patch_size * self.downsample_factor
return _grid_size(img_w, img_h, ph, self.max_image_tokens)
# -- preprocessing ---------------------------------------------------------
def preprocess_image(self, image: Image.Image) -> tuple[torch.Tensor, int]:
"""Return (pixel tensor [3, H, W], n_patch_tokens) for one image."""
image = image.convert("RGB")
target_h, target_w, n_tokens = self.compute_image_size(
image.width, image.height
)
image = image.resize((target_w, target_h), Image.LANCZOS)
return self._to_norm_tensor(image), n_tokens
# torchcodec is the training-faithful video decoder; other decoders (torchvision /
# PyAV) seek + color-convert differently and diverge from training. Optional at
# import time so text/image paths load without it; decode_video raises if used.
try:
import torchcodec
except Exception:
torchcodec = None
class MuseGlimmerVideoProcessor(BaseVideoProcessor):
"""MuseGlimmer video preprocessing behind the standard HF
``AutoVideoProcessor`` API.
Exposes MuseGlimmer's training-faithful video handling as a first-class video
processor so you can call ``processor(videos="clip.mp4")`` and downstream
users can discover it via the standard HF ``AutoVideoProcessor`` API plus a
``video_preprocessor_config.json``. It wraps:
* torchcodec decode (matches training; other decoders diverge),
* uniform frame sampling to a whole multiple of ``patch_temporal``,
* real per-group PTS timestamps (rendered as ``Time: X.Xs`` by
MuseGlimmerProcessor),
* ``patch_temporal`` frame-grouping (frames cat on the channel axis ->
``[patch_temporal * 3, H, W]``; the encoder detects video by channel count).
It deliberately overrides ``preprocess`` instead of the BaseVideoProcessor fast
pipeline (group_videos_by_shape / smart_resize / single stacked tensor): MuseGlimmer
consumes a LIST of variable-size group tensors and needs the real per-group PTS,
neither of which the stacked fast path models.
"""
model_input_names = ["pixel_values"]
def __init__(
self,
patch_size: int = 14,
downsample_factor: int = 2,
patch_temporal: int = 2,
max_video_frame_tokens: int = 144,
image_mean: float = 0.5,
image_std: float = 0.5,
video_num_frames: int = 96,
video_sampling_fps: float = 2.0,
**kwargs,
):
super().__init__(**kwargs)
self.patch_size = patch_size
self.downsample_factor = downsample_factor
self.patch_temporal = patch_temporal
self.max_video_frame_tokens = max_video_frame_tokens
self.image_mean = image_mean
self.image_std = image_std
self.video_num_frames = video_num_frames
self.video_sampling_fps = video_sampling_fps
def _to_norm_tensor(self, image: Image.Image) -> torch.Tensor:
return T.functional.normalize(
T.functional.to_tensor(image),
[self.image_mean] * 3,
[self.image_std] * 3,
)
def decode_video(self, video_path: str) -> tuple[list[Image.Image], list[float]]:
"""Sample frames + per-group PTS with torchcodec (the training decode path).
``timestamps[g]`` is the ACTUAL decoded PTS of the first frame in temporal
group ``g``; ``len(frames)`` is a whole multiple of ``patch_temporal``.
"""
if torchcodec is None:
raise RuntimeError(
"torchcodec is required for video decoding (it matches the training "
"decode path). See the Environment Setup steps in hf/README.md."
)
pt = self.patch_temporal
reader = torchcodec.decoders.VideoDecoder(video_path)
total = len(reader)
assert reader.metadata.average_fps is not None, (
f"Video has no FPS metadata: {video_path}"
)
fps = reader.metadata.average_fps
assert self.video_sampling_fps and self.video_sampling_fps > 0, (
f"video_sampling_fps must be positive, got {self.video_sampling_fps}"
)
n = min(
int(total * self.video_sampling_fps / fps), self.video_num_frames, total
)
n = max(pt, (n // pt) * pt)
n = min(n, total)
if n < pt:
raise ValueError(
f"Video has only {total} decodable frame(s) but needs at least "
f"{pt} (one temporal patch): {video_path}"
)
indices = torch.linspace(0, total - 1, n).long().tolist()
frames: list[Image.Image] = []
timestamps: list[float] = []
for j, i in enumerate(indices):
fr = reader[i]
frames.append(
Image.fromarray(fr.data.permute(1, 2, 0).numpy()).convert("RGB")
)
if j % pt == 0:
pts = getattr(fr, "pts_seconds", None)
timestamps.append(float(pts) if pts is not None else i / fps)
return frames, timestamps
def compute_video_frame_size(self, img_w: int, img_h: int) -> tuple[int, int, int]:
"""Pick (H, W, tokens_per_frame) for a video frame under the per-frame cap.
Mirrors MuseGlimmerImageProcessor.compute_image_size but uses the (smaller)
per-frame video token budget; shares the same ``_grid_size`` grid logic as
modeling_muse_glimmer.MuseGlimmerVisionEncoder.
"""
ph = self.patch_size * self.downsample_factor
return _grid_size(img_w, img_h, ph, self.max_video_frame_tokens)
def _group_frames(
self, frames: list[Image.Image]
) -> tuple[list[torch.Tensor], int, int]:
"""Resize/normalize frames and cat ``patch_temporal`` frames per group.
Returns (group_tensors, n_groups, tokens_per_group).
"""
pt = self.patch_temporal
if pt <= 0:
raise ValueError(f"patch_temporal must be positive, got {pt}")
if not frames:
raise ValueError("video must contain at least one frame")
if padding := (-len(frames)) % pt:
frames = [*frames, *([frames[-1]] * padding)]
first = frames[0].convert("RGB")
target_h, target_w, n_tokens = self.compute_video_frame_size(
first.width, first.height
)
groups: list[torch.Tensor] = []
for i in range(0, len(frames), pt):
grp = [
self._to_norm_tensor(
frames[i + j]
.convert("RGB")
.resize((target_w, target_h), Image.LANCZOS)
)
for j in range(pt)
]
groups.append(torch.cat(grp, dim=0))
return groups, len(groups), n_tokens
@staticmethod
def _normalize_videos(videos) -> list:
"""Normalize the ``videos`` arg to a list of per-video items.
A video item is a path (str/Path) or a list of PIL frames. Accepts a
single video (one path, or one list of frames) or a list of those.
"""
if isinstance(videos, (str, Path)):
return [videos]
if videos and not isinstance(videos[0], (list, tuple, str, Path)):
return [videos] # a single list of PIL frames
return list(videos or [])
def preprocess_one(
self,
video: str | Path | list[Image.Image],
timestamps: list[float] | None = None,
) -> tuple[list[torch.Tensor], int, int, list[float]]:
"""Decode (if a path) + group ONE video.
Returns (group_tensors, n_groups, tokens_per_group, group_timestamps). This
is the single per-video implementation shared by ``preprocess`` (the HF
AutoVideoProcessor batch API) and ``MuseGlimmerProcessor.__call__`` (prompt
building), so the two paths cannot drift. For a path, frames + real
per-group PTS come from ``decode_video``; for pre-decoded frames the given
``timestamps`` are used verbatim (empty if not supplied).
"""
if isinstance(video, (str, Path)):
frames, ts = self.decode_video(str(video))
else:
frames = [f.convert("RGB") for f in video]
ts = list(timestamps) if timestamps is not None else []
groups, n_groups, tokens_per_group = self._group_frames(frames)
return groups, n_groups, tokens_per_group, ts
def preprocess(
self,
videos,
video_timestamps: list[list[float]] | None = None,
return_tensors: str | None = "pt",
**kwargs,
) -> BatchFeature:
"""Preprocess video file path(s) or pre-decoded frame list(s).
Accepts a single video (path str, or list of PIL frames) or a list of
those. Returns a BatchFeature carrying, in video order:
* ``pixel_values`` -- flat list of [pt*3, H, W] group tensors,
* ``video_num_groups`` -- groups per video,
* ``video_tokens_per_group`` -- patch-token count per group per video,
* ``video_timestamps`` -- per-group PTS per video,
i.e. exactly what MuseGlimmerProcessor needs to build the
``<|vid_start|> ( Time: X.Xs <|video|>*P [sep] )* <|vid_end|>`` block.
"""
videos = self._normalize_videos(videos)
pixel_values: list[torch.Tensor] = []
num_groups: list[int] = []
tokens_per_group: list[int] = []
out_ts: list[list[float]] = []
for idx, v in enumerate(videos):
ts_in = video_timestamps[idx] if video_timestamps else None
groups, ng, tpg, ts = self.preprocess_one(v, ts_in)
pixel_values += groups
num_groups.append(ng)
tokens_per_group.append(tpg)
out_ts.append(ts)
batch = BatchFeature(
data={
"video_num_groups": num_groups,
"video_tokens_per_group": tokens_per_group,
"video_timestamps": out_ts,
},
tensor_type=None,
)
# Variable-size list -- BatchFeature would fail to stack into one tensor.
batch["pixel_values"] = pixel_values
return batch
class MuseGlimmerProcessor(ProcessorMixin):
"""Bundle MuseGlimmerImageProcessor + MuseGlimmerVideoProcessor + tokenizer;
expand media sentinels into spans.
Images go through ``image_processor`` (MuseGlimmerImageProcessor) and videos through
``video_processor`` (MuseGlimmerVideoProcessor) -- one preprocessing implementation
each, no overlap.
"""
attributes = ["image_processor", "video_processor", "tokenizer"]
image_processor_class = "AutoImageProcessor"
video_processor_class = "AutoVideoProcessor"
tokenizer_class = "PreTrainedTokenizerFast"
def __init__(
self,
image_processor=None,
video_processor=None,
tokenizer=None,
chat_template=None,
**kwargs,
):
if image_processor is None:
image_processor = MuseGlimmerImageProcessor()
if video_processor is None:
video_processor = MuseGlimmerVideoProcessor()
super().__init__(
image_processor,
video_processor,
tokenizer,
chat_template=chat_template or MUSE_GLIMMER_MM_CHAT_TEMPLATE,
**kwargs,
)
def _sid(self, token: str) -> int:
return self.tokenizer.convert_tokens_to_ids(token)
def _image_block(self, n_tokens: int) -> list[int]:
return (
[self._sid("<|image_start|>")]
+ [self._sid("<|patch|>")] * n_tokens
+ [self._sid("<|image_end|>")]
)
def _video_block(
self,
n_groups: int,
tokens_per_group: int,
timestamps: list[float] | None = None,
) -> list[int]:
"""Per-group ``Time: X.Xs`` + <|video|>*P, separated/terminated.
``timestamps`` (one per temporal group, in seconds) should be the ACTUAL
decoded frame times -- training renders the real factored frame PTS, not a
uniform grid. Falls back to ``g*patch_temporal/fps`` only when timestamps
are not supplied (e.g. callers that don't track frame times); that
approximation can differ from training by the per-frame jitter.
"""
vid = self._sid("<|video|>")
sep = self._sid("<|vid_frame_separator|>")
pt = self.video_processor.patch_temporal
fps = self.video_processor.video_sampling_fps
block = [self._sid("<|vid_start|>")]
for g in range(n_groups):
ts = timestamps[g] if timestamps is not None else g * pt / fps
block += self.tokenizer.encode(f"Time: {ts:.1f}s", add_special_tokens=False)
block += [vid] * tokens_per_group
block.append(sep if g < n_groups - 1 else self._sid("<|vid_end|>"))
return block
def __call__(
self,
text: str | list[str] | None = None,
images: list[Image.Image] | None = None,
videos: list[list[Image.Image] | str] | None = None,
video_timestamps: list[list[float]] | None = None,
return_tensors: str | None = "pt",
**kwargs,
) -> BatchFeature:
if text is None:
raise ValueError(
"`text` is required (use apply_chat_template to build it)."
)
if isinstance(text, (list, tuple)):
if len(text) != 1:
raise ValueError(
"MuseGlimmerProcessor supports a single text sample per call."
)
text = text[0]
images = list(images or [])
videos = list(videos or [])
image_sentinel = self._sid(IMAGE_SENTINEL)
video_sentinel = self._sid(VIDEO_SENTINEL)
# Preprocess media up front so we can expand sentinels in document order.
# Images via image_processor; videos via video_processor (each video item
# is a path -- decoded here -- or a pre-decoded list of PIL frames).
prepped_images = [self.image_processor.preprocess_image(im) for im in images]
prepped_videos = [
self.video_processor.preprocess_one(
v, video_timestamps[i] if video_timestamps else None
)
for i, v in enumerate(videos)
]
ids = self.tokenizer.encode(text, add_special_tokens=False)
n_img = sum(1 for t in ids if t == image_sentinel)
n_vid = sum(1 for t in ids if t == video_sentinel)
if n_img != len(prepped_images):
raise ValueError(
f"{n_img} image sentinel(s) in text but "
f"{len(prepped_images)} image(s) given."
)
if n_vid != len(prepped_videos):
raise ValueError(
f"{n_vid} video sentinel(s) in text but "
f"{len(prepped_videos)} video(s) given."
)
out_ids: list[int] = []
pixel_values: list[torch.Tensor] = []
img_i = vid_i = 0
for tid in ids:
if tid == image_sentinel:
tensor, n_tokens = prepped_images[img_i]
img_i += 1
out_ids += self._image_block(n_tokens)
pixel_values.append(tensor)
elif tid == video_sentinel:
groups, n_groups, tokens_per_group, ts = prepped_videos[vid_i]
vid_i += 1
out_ids += self._video_block(n_groups, tokens_per_group, ts or None)
pixel_values += groups
else:
out_ids.append(tid)
data: dict = {
"input_ids": [out_ids],
"attention_mask": [[1] * len(out_ids)],
}
batch = BatchFeature(data=data, tensor_type=return_tensors)
# Keep pixel_values as a list of variable-size tensors (MuseGlimmerModel
# consumes a list); BatchFeature would fail to stack them into one tensor.
if pixel_values:
batch["pixel_values"] = pixel_values
return batch
__all__ = [
"MuseGlimmerImageProcessor",
"MuseGlimmerVideoProcessor",
"MuseGlimmerProcessor",
"MUSE_GLIMMER_MM_CHAT_TEMPLATE",
]