Add Muse Glimmer model support (#51655)
Signed-off-by: Tiezhen Wang <tiezhen@inferact.ai> Signed-off-by: Jee Jee Li <jeejeelee@inferact.ai> Signed-off-by: zixi-qi <zixi@inferact.ai> Co-authored-by: Beto de Paola <betodepaola@meta.com> Co-authored-by: Giancarlo Delfin <gdelfin@inferact.ai> Co-authored-by: Jee Jee Li <jeejeelee@inferact.ai> Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com> Co-authored-by: Jee Jee Li <pandaleefree@gmail.com> Co-authored-by: zixi-qi <zixi@inferact.ai>
This commit is contained in:
co-authored by
Beto de Paola
Giancarlo Delfin
Jee Jee Li
Claude Opus 5
Jee Jee Li
zixi-qi
parent
59d1af5698
commit
6adad08767
@@ -102,6 +102,10 @@ _CONFIG_REGISTRY: dict[str, type[PretrainedConfig]] = LazyConfigDict(
|
||||
kimi_linear="KimiLinearConfig",
|
||||
kimi_vl="KimiVLConfig",
|
||||
kimi_k25="KimiK25Config",
|
||||
muse_glimmer="MuseGlimmerConfig",
|
||||
muse_glimmer_text="MuseGlimmerTextConfig",
|
||||
muse_glimmer_vision="MuseGlimmerVisionConfig",
|
||||
muse_glimmer_assistant="MuseGlimmerAssistantConfig",
|
||||
kimi_k3="KimiK3Config",
|
||||
RefinedWeb="RWConfig", # For tiiuae/falcon-40b(-instruct)
|
||||
RefinedWebModel="RWConfig", # For tiiuae/falcon-7b(-instruct)
|
||||
|
||||
@@ -74,6 +74,10 @@ _CLASS_TO_MODULE: dict[str, str] = {
|
||||
"KimiLinearConfig": "vllm.transformers_utils.configs.kimi_linear",
|
||||
"KimiVLConfig": "vllm.transformers_utils.configs.kimi_vl",
|
||||
"KimiK25Config": "vllm.transformers_utils.configs.kimi_k25",
|
||||
"MuseGlimmerConfig": "vllm.transformers_utils.configs.muse_glimmer",
|
||||
"MuseGlimmerTextConfig": "vllm.transformers_utils.configs.muse_glimmer",
|
||||
"MuseGlimmerVisionConfig": "vllm.transformers_utils.configs.muse_glimmer",
|
||||
"MuseGlimmerAssistantConfig": "vllm.transformers_utils.configs.muse_glimmer",
|
||||
"KimiK3Config": "vllm.transformers_utils.configs.kimi_k3",
|
||||
"KimiK3VisionConfig": "vllm.transformers_utils.configs.kimi_k3",
|
||||
"NemotronConfig": "vllm.transformers_utils.configs.nemotron",
|
||||
@@ -162,6 +166,10 @@ __all__ = [
|
||||
"KimiLinearConfig",
|
||||
"KimiVLConfig",
|
||||
"KimiK25Config",
|
||||
"MuseGlimmerConfig",
|
||||
"MuseGlimmerTextConfig",
|
||||
"MuseGlimmerVisionConfig",
|
||||
"MuseGlimmerAssistantConfig",
|
||||
"KimiK3Config",
|
||||
"KimiK3VisionConfig",
|
||||
"NemotronConfig",
|
||||
|
||||
@@ -0,0 +1,389 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""MuseGlimmer model configuration for vLLM.
|
||||
|
||||
Native vLLM copy of the MuseGlimmer HuggingFace configs
|
||||
(``configuration_muse_glimmer.py``). MuseGlimmer's
|
||||
``model_type`` is not yet registered in released transformers, so vLLM ships
|
||||
this config so partners can serve MuseGlimmer checkpoints *without* trust_remote_code.
|
||||
|
||||
The field set and defaults are kept byte-for-byte in sync with the HF reference
|
||||
so a checkpoint's ``config.json`` deserializes identically here. Both the text
|
||||
and vision configs are consumed by the native multimodal serving path.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from transformers import Qwen3Config
|
||||
from transformers.configuration_utils import PretrainedConfig
|
||||
|
||||
|
||||
def _default_no_rope_layers(num_hidden_layers: int) -> list[int]:
|
||||
# iRoPE mask: NoPE every 4 layers, counted backward from the last layer.
|
||||
stride = 4
|
||||
return [
|
||||
0 if (num_hidden_layers - 1 - i) % stride == 0 else 1
|
||||
for i in range(num_hidden_layers)
|
||||
]
|
||||
|
||||
|
||||
class MuseGlimmerTextConfig(PretrainedConfig):
|
||||
model_type = "muse_glimmer_text"
|
||||
keys_to_ignore_at_inference = ["past_key_values"]
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
vocab_size: int = 202_048,
|
||||
hidden_size: int = 6656,
|
||||
intermediate_size: int = 19968,
|
||||
num_hidden_layers: int = 52,
|
||||
num_attention_heads: int = 32,
|
||||
num_key_value_heads: int = 2,
|
||||
head_dim: int = 128,
|
||||
hidden_activation: str = "silu",
|
||||
max_position_embeddings: int = 16_384,
|
||||
initializer_range: float = 0.02,
|
||||
rms_norm_eps: float = 1e-5,
|
||||
use_cache: bool = True,
|
||||
pad_token_id: int | None = None,
|
||||
eos_token_id: int | list[int] | None = 200_001,
|
||||
bos_token_id: int | None = 200_000,
|
||||
tie_word_embeddings: bool = False,
|
||||
rope_parameters: dict | None = None,
|
||||
rope_theta: float | None = None,
|
||||
attention_bias: bool = False,
|
||||
attention_dropout: float = 0.0,
|
||||
query_pre_attn_scalar: int = 256,
|
||||
sliding_window: int | None = 2048,
|
||||
layer_types: list[str] | None = None,
|
||||
final_logit_softcapping: float | None = 20.0,
|
||||
attn_logit_softcapping: float | None = None,
|
||||
use_bidirectional_attention: bool | None = None,
|
||||
# MuseGlimmer-specific
|
||||
qk_scale_factor: float = 43.7840518911,
|
||||
use_qk_norm: bool = True,
|
||||
use_attn_output_gate: bool = True,
|
||||
output_multiplier: float = 0.19611613513818404,
|
||||
normalize_tok_embeddings: bool = True,
|
||||
post_norm_eps: float = 1e-8,
|
||||
no_rope_layers: list[int] | None = None,
|
||||
**kwargs,
|
||||
) -> None:
|
||||
self.vocab_size = vocab_size
|
||||
self.hidden_size = hidden_size
|
||||
self.intermediate_size = intermediate_size
|
||||
self.num_hidden_layers = num_hidden_layers
|
||||
self.num_attention_heads = num_attention_heads
|
||||
self.num_key_value_heads = num_key_value_heads
|
||||
self.head_dim = head_dim
|
||||
self.hidden_activation = hidden_activation
|
||||
self.max_position_embeddings = max_position_embeddings
|
||||
self.initializer_range = initializer_range
|
||||
self.rms_norm_eps = rms_norm_eps
|
||||
self.use_cache = use_cache
|
||||
self.attention_bias = attention_bias
|
||||
self.attention_dropout = attention_dropout
|
||||
self.query_pre_attn_scalar = query_pre_attn_scalar
|
||||
self.sliding_window = sliding_window
|
||||
self.final_logit_softcapping = final_logit_softcapping
|
||||
self.attn_logit_softcapping = attn_logit_softcapping
|
||||
self.use_bidirectional_attention = use_bidirectional_attention
|
||||
|
||||
# RoPE: accept either an explicit rope_parameters dict (HF 5.x) or a
|
||||
# bare rope_theta; normalize to rope_parameters for vLLM's get_rope.
|
||||
if rope_parameters is None:
|
||||
theta = rope_theta if rope_theta is not None else 500_000.0
|
||||
rope_parameters = {"rope_type": "default", "rope_theta": theta}
|
||||
self.rope_parameters = rope_parameters
|
||||
# vLLM reads rope_theta off the config in some codepaths.
|
||||
self.rope_theta = rope_parameters.get("rope_theta", 500_000.0)
|
||||
|
||||
# MuseGlimmer-specific fields
|
||||
self.qk_scale_factor = qk_scale_factor
|
||||
self.use_qk_norm = use_qk_norm
|
||||
self.use_attn_output_gate = use_attn_output_gate
|
||||
self.output_multiplier = output_multiplier
|
||||
self.normalize_tok_embeddings = normalize_tok_embeddings
|
||||
self.post_norm_eps = post_norm_eps
|
||||
|
||||
self.no_rope_layers = (
|
||||
no_rope_layers
|
||||
if no_rope_layers is not None
|
||||
else _default_no_rope_layers(num_hidden_layers)
|
||||
)
|
||||
if layer_types is None:
|
||||
layer_types = [
|
||||
"full_attention" if self.no_rope_layers[i] == 0 else "sliding_attention"
|
||||
for i in range(num_hidden_layers)
|
||||
]
|
||||
self.layer_types = layer_types
|
||||
|
||||
super().__init__(
|
||||
pad_token_id=pad_token_id,
|
||||
bos_token_id=bos_token_id,
|
||||
eos_token_id=eos_token_id,
|
||||
tie_word_embeddings=tie_word_embeddings,
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
|
||||
class MuseGlimmerVisionConfig(PretrainedConfig):
|
||||
model_type = "muse_glimmer_vision"
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
patch_size: int = 14,
|
||||
pos_emb_height: int = 32,
|
||||
pos_emb_width: int = 32,
|
||||
num_attention_heads: int = 16,
|
||||
num_hidden_layers: int = 50,
|
||||
hidden_size: int = 1536,
|
||||
intermediate_size: int = 8960,
|
||||
hidden_act: str = "gelu",
|
||||
merge_kernel_size: int = 2,
|
||||
rope_parameters: dict | None = None,
|
||||
max_position_embeddings: int = 32 * 32,
|
||||
output_dim: int = 6144,
|
||||
patch_temporal: int = 2,
|
||||
adapter_dim: int = 4096,
|
||||
layer_norm_eps: float = 1e-5,
|
||||
layer_types: list[str] | None = None,
|
||||
**kwargs,
|
||||
) -> None:
|
||||
super().__init__(**kwargs)
|
||||
self.patch_size = patch_size
|
||||
self.pos_emb_height = pos_emb_height
|
||||
self.pos_emb_width = pos_emb_width
|
||||
self.num_attention_heads = num_attention_heads
|
||||
self.num_hidden_layers = num_hidden_layers
|
||||
self.hidden_size = hidden_size
|
||||
self.intermediate_size = intermediate_size
|
||||
self.hidden_act = hidden_act
|
||||
self.merge_kernel_size = merge_kernel_size
|
||||
self.rope_parameters = rope_parameters
|
||||
self.max_position_embeddings = max_position_embeddings
|
||||
self.output_dim = output_dim
|
||||
self.patch_temporal = patch_temporal
|
||||
self.adapter_dim = adapter_dim
|
||||
self.layer_norm_eps = layer_norm_eps
|
||||
if layer_types is None:
|
||||
stride = 4
|
||||
layer_types = [
|
||||
"full_attention"
|
||||
if (i + 1) % stride == 0 or i == num_hidden_layers - 1
|
||||
else "sliding_attention"
|
||||
for i in range(num_hidden_layers)
|
||||
]
|
||||
self.layer_types = layer_types
|
||||
|
||||
|
||||
class MuseGlimmerConfig(PretrainedConfig):
|
||||
model_type = "muse_glimmer"
|
||||
sub_configs = {
|
||||
"text_config": MuseGlimmerTextConfig,
|
||||
"vision_config": MuseGlimmerVisionConfig,
|
||||
}
|
||||
|
||||
# --- Flat (legacy-converter) -> canonical normalization -------------------
|
||||
# MuseGlimmer checkpoints exist in two config layouts in the wild:
|
||||
#
|
||||
# * CANONICAL (current HF converter, transformers 5.15): nested
|
||||
# ``text_config`` / ``vision_config`` sub-dicts with canonical field
|
||||
# names (``hidden_activation``, ``final_logit_softcapping``,
|
||||
# ``rope_parameters``, ``vision_config.hidden_size`` ...).
|
||||
#
|
||||
# * FLAT (older converter, e.g. Ruan's ``rl_v1/hf``, transformers 5.9):
|
||||
# every field is a top-level key, with different names
|
||||
# (``hidden_act``, ``output_soft_cap_temp``, ``rope_theta``,
|
||||
# ``vision_latent_dim`` ...) and NO ``text_config`` nesting.
|
||||
#
|
||||
# Without normalization a flat config silently deserializes to an
|
||||
# ALL-DEFAULT text config (every checkpoint value ignored) — a dangerous
|
||||
# correctness bug: a checkpoint whose arch differs from the defaults would
|
||||
# load into a wrong-shaped model with no error. So when we detect a flat
|
||||
# config we hoist the flat fields into ``text_config`` / ``vision_config``
|
||||
# and rename them to canonical names.
|
||||
|
||||
# flat text field name -> canonical MuseGlimmerTextConfig field name
|
||||
_FLAT_TEXT_RENAMES = {
|
||||
"hidden_act": "hidden_activation",
|
||||
"output_soft_cap_temp": "final_logit_softcapping",
|
||||
}
|
||||
# canonical MuseGlimmerTextConfig constructor params that may appear flat
|
||||
_FLAT_TEXT_KEYS = frozenset(
|
||||
{
|
||||
"vocab_size",
|
||||
"hidden_size",
|
||||
"intermediate_size",
|
||||
"num_hidden_layers",
|
||||
"num_attention_heads",
|
||||
"num_key_value_heads",
|
||||
"head_dim",
|
||||
"hidden_activation",
|
||||
"hidden_act",
|
||||
"max_position_embeddings",
|
||||
"initializer_range",
|
||||
"rms_norm_eps",
|
||||
"use_cache",
|
||||
"tie_word_embeddings",
|
||||
"rope_parameters",
|
||||
"rope_theta",
|
||||
"attention_bias",
|
||||
"attention_dropout",
|
||||
"query_pre_attn_scalar",
|
||||
"sliding_window",
|
||||
"layer_types",
|
||||
"final_logit_softcapping",
|
||||
"output_soft_cap_temp",
|
||||
"attn_logit_softcapping",
|
||||
"use_bidirectional_attention",
|
||||
"qk_scale_factor",
|
||||
"use_qk_norm",
|
||||
"use_attn_output_gate",
|
||||
"output_multiplier",
|
||||
"normalize_tok_embeddings",
|
||||
"post_norm_eps",
|
||||
"no_rope_layers",
|
||||
}
|
||||
)
|
||||
# flat vision field name -> canonical MuseGlimmerVisionConfig field name
|
||||
_FLAT_VISION_RENAMES = {
|
||||
"vision_latent_dim": "hidden_size",
|
||||
"vision_heads": "num_attention_heads",
|
||||
"vision_layers": "num_hidden_layers",
|
||||
"vision_output_dim": "output_dim",
|
||||
"vision_patch_size": "patch_size",
|
||||
"vision_patch_temporal": "patch_temporal",
|
||||
"vision_adapter_dim": "adapter_dim",
|
||||
"vision_pos_emb_grid_h": "pos_emb_height",
|
||||
"vision_pos_emb_grid_w": "pos_emb_width",
|
||||
"vision_downsample_factor": "merge_kernel_size",
|
||||
}
|
||||
|
||||
@classmethod
|
||||
def _looks_flat(cls, kwargs: dict) -> bool:
|
||||
# Flat if there is no explicit text_config but there ARE text-level
|
||||
# fields at the top level (e.g. hidden_size / num_hidden_layers).
|
||||
if kwargs.get("text_config") is not None:
|
||||
return False
|
||||
return any(k in kwargs for k in ("hidden_size", "num_hidden_layers"))
|
||||
|
||||
@classmethod
|
||||
def _normalize_flat(cls, kwargs: dict) -> dict:
|
||||
kwargs = dict(kwargs)
|
||||
text: dict = {}
|
||||
for key in list(kwargs.keys()):
|
||||
if key in cls._FLAT_TEXT_KEYS:
|
||||
canon = cls._FLAT_TEXT_RENAMES.get(key, key)
|
||||
text.setdefault(canon, kwargs.pop(key))
|
||||
|
||||
vision_mlp_ratio = kwargs.pop("vision_mlp_ratio", None)
|
||||
sparse_factor = kwargs.pop("vision_sparse_attention_factor", None)
|
||||
vision: dict = {}
|
||||
for key in list(kwargs.keys()):
|
||||
if key in cls._FLAT_VISION_RENAMES:
|
||||
vision.setdefault(cls._FLAT_VISION_RENAMES[key], kwargs.pop(key))
|
||||
|
||||
if vision_mlp_ratio is not None:
|
||||
hidden_size = vision.get("hidden_size", 1536)
|
||||
vision["intermediate_size"] = int(vision_mlp_ratio * hidden_size)
|
||||
if sparse_factor is not None:
|
||||
sparse_factor = int(sparse_factor)
|
||||
if sparse_factor <= 0:
|
||||
raise ValueError("vision_sparse_attention_factor must be positive")
|
||||
num_layers = vision.get("num_hidden_layers", 50)
|
||||
vision["layer_types"] = [
|
||||
"full_attention"
|
||||
if (layer_idx + 1) % sparse_factor == 0 or layer_idx == num_layers - 1
|
||||
else "sliding_attention"
|
||||
for layer_idx in range(num_layers)
|
||||
]
|
||||
|
||||
if text:
|
||||
kwargs["text_config"] = text
|
||||
if vision:
|
||||
kwargs["vision_config"] = vision
|
||||
return kwargs
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
text_config: dict | MuseGlimmerTextConfig | None = None,
|
||||
vision_config: dict | MuseGlimmerVisionConfig | None = None,
|
||||
image_token_id: int = 200092,
|
||||
video_token_id: int = 200091,
|
||||
**kwargs,
|
||||
) -> None:
|
||||
# Detect + fold a flat config into nested text/vision before building
|
||||
# the sub-configs. (image/video token ids are handled below; a flat
|
||||
# ``patch_token_id`` alias is mapped too.)
|
||||
if text_config is None and self._looks_flat(kwargs):
|
||||
kwargs = self._normalize_flat(kwargs)
|
||||
text_config = kwargs.pop("text_config", None)
|
||||
vision_config = kwargs.pop("vision_config", vision_config)
|
||||
# flat image-token alias
|
||||
if "patch_token_id" in kwargs and "image_token_id" not in kwargs:
|
||||
image_token_id = kwargs.pop("patch_token_id")
|
||||
|
||||
if text_config is None:
|
||||
self.text_config = MuseGlimmerTextConfig()
|
||||
elif isinstance(text_config, dict):
|
||||
self.text_config = MuseGlimmerTextConfig(**text_config)
|
||||
else:
|
||||
self.text_config = text_config
|
||||
|
||||
if vision_config is None:
|
||||
self.vision_config = MuseGlimmerVisionConfig()
|
||||
elif isinstance(vision_config, dict):
|
||||
self.vision_config = MuseGlimmerVisionConfig(**vision_config)
|
||||
else:
|
||||
self.vision_config = vision_config
|
||||
|
||||
self.image_token_id = image_token_id
|
||||
self.video_token_id = video_token_id
|
||||
self.patch_token_id = image_token_id
|
||||
super().__init__(**kwargs)
|
||||
|
||||
|
||||
class MuseGlimmerAssistantConfig(Qwen3Config):
|
||||
"""Config for the Muse Glimmer DFlash draft head.
|
||||
|
||||
The head is Qwen3-shaped and runs on vLLM's generic ``qwen3_dflash``
|
||||
implementation, so this derives from ``Qwen3Config``. It cannot BE
|
||||
``Qwen3Config``, because two of the checkpoint's values do not survive that
|
||||
class:
|
||||
|
||||
* ``sliding_window`` is gated behind ``use_sliding_window``, which defaults
|
||||
to False -- ``Qwen3Config(sliding_window=2048).sliding_window`` is None.
|
||||
The checkpoint declares ``sliding_window: 2048`` and five
|
||||
``sliding_attention`` layers, so the window silently disappears and the
|
||||
DFlash path then raises "sliding attention requires a window size".
|
||||
* ``vocab_size`` is absent from the checkpoint, so Qwen3's default of
|
||||
151936 applies instead of Muse Glimmer's 202048. That one is *silent*: it
|
||||
builds an all-zero ``draft_id_to_target_id`` remap, and also puts
|
||||
pad/bos/eos/mask token ids out of range.
|
||||
|
||||
Both defaults are set here so an unmodified checkpoint loads correctly.
|
||||
"""
|
||||
|
||||
model_type = "muse_glimmer_assistant"
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
vocab_size: int = 202048,
|
||||
use_sliding_window: bool = True,
|
||||
dflash_config: dict | None = None,
|
||||
**kwargs,
|
||||
) -> None:
|
||||
super().__init__(
|
||||
vocab_size=vocab_size, use_sliding_window=use_sliding_window, **kwargs
|
||||
)
|
||||
# muse_glimmer_assistant uses non-causal attention.
|
||||
self.dflash_config = {"causal": False} | (dflash_config or {})
|
||||
|
||||
|
||||
__all__ = [
|
||||
"MuseGlimmerTextConfig",
|
||||
"MuseGlimmerVisionConfig",
|
||||
"MuseGlimmerConfig",
|
||||
"MuseGlimmerAssistantConfig",
|
||||
]
|
||||
@@ -0,0 +1,582 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""HuggingFace processor for the MuseGlimmer multimodal model (text + image + video).
|
||||
|
||||
This packages MuseGlimmer's vision preprocessing and token-span expansion the standard
|
||||
HF way, so you can do:
|
||||
|
||||
processor = AutoProcessor.from_pretrained(hf_dir, trust_remote_code=True)
|
||||
messages = [{"role": "user", "content": [
|
||||
{"type": "image"},
|
||||
{"type": "text", "text": "Describe this image."},
|
||||
]}]
|
||||
text = processor.apply_chat_template(messages, add_generation_prompt=True)
|
||||
inputs = processor(text=text, images=[pil_image], return_tensors="pt")
|
||||
out = model.generate(**inputs, max_new_tokens=256)
|
||||
|
||||
It reproduces the exact token layout the model expects:
|
||||
|
||||
image -> <|image_start|> + <|patch|> * N + <|image_end|>
|
||||
video -> <|vid_start|>
|
||||
( "Time: X.Xs" + <|video|> * P [+ <|vid_frame_separator|>] )*
|
||||
+ <|vid_end|>
|
||||
|
||||
where N = floor(H/(patch*ds)) * floor(W/(patch*ds)) (<= 4096) for images and
|
||||
P = the same grid count per temporal frame-group (<= 144) for video, with the
|
||||
target (H, W) chosen by MuseGlimmerImageProcessor.compute_image_size /
|
||||
MuseGlimmerVideoProcessor.compute_video_frame_size (the same grid logic as
|
||||
modeling_muse_glimmer.MuseGlimmerVisionEncoder).
|
||||
|
||||
The chat template emits one sentinel per media item -- ``<|image|>`` for images,
|
||||
``<|video|>`` for videos -- which ``MuseGlimmerProcessor.__call__`` expands into the
|
||||
spans above using each media item's computed token count. Vision features are
|
||||
returned in ``pixel_values`` (a list of per-image / per-frame-group tensors) in
|
||||
the order the sentinels appear, matching how ``MuseGlimmerModel`` merges them at the
|
||||
``<|patch|>`` / ``<|video|>`` positions.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import itertools
|
||||
import math
|
||||
from pathlib import Path
|
||||
|
||||
import torch
|
||||
from PIL import Image
|
||||
from torchvision import transforms as T
|
||||
from transformers.feature_extraction_utils import BatchFeature
|
||||
from transformers.image_processing_utils import BaseImageProcessor
|
||||
from transformers.processing_utils import ProcessorMixin
|
||||
from transformers.video_processing_utils import BaseVideoProcessor
|
||||
|
||||
# Single-token sentinels emitted by the chat template, one per media item.
|
||||
IMAGE_SENTINEL = "<|image|>" # id 200090; absent from the image/video spans
|
||||
VIDEO_SENTINEL = "<|video|>" # id 200091; also the per-frame token (see __call__)
|
||||
|
||||
# Jinja chat template (multimodal superset of convert_muse_glimmer_to_hf.CHAT_TEMPLATE).
|
||||
# Message content may be a plain string OR a list of parts
|
||||
# ({"type": "text"|"image"|"video", ...}); image/video parts render as a single
|
||||
# sentinel that __call__ expands.
|
||||
MUSE_GLIMMER_MM_CHAT_TEMPLATE = (
|
||||
"{{- bos_token -}}"
|
||||
"{%- macro render_parts(content) -%}"
|
||||
"{%- if content is string -%}{{- content -}}"
|
||||
"{%- else -%}"
|
||||
"{%- for part in content -%}"
|
||||
"{%- if part['type'] == 'image' -%}{{- '<|image|>' -}}"
|
||||
"{%- elif part['type'] == 'video' -%}{{- '<|video|>' -}}"
|
||||
"{%- elif part['type'] == 'text' -%}{{- part['text'] -}}"
|
||||
"{%- endif -%}"
|
||||
"{%- endfor -%}"
|
||||
"{%- endif -%}"
|
||||
"{%- endmacro -%}"
|
||||
# At inference (add_generation_prompt), inject a default system message when
|
||||
# the caller supplied none. Full-transcript rendering
|
||||
# (add_generation_prompt=False) is left unchanged.
|
||||
"{%- set ns = namespace(has_system=false) -%}"
|
||||
"{%- for m in messages -%}"
|
||||
"{%- if m['role'] == 'system' -%}{%- set ns.has_system = true -%}{%- endif -%}"
|
||||
"{%- endfor -%}"
|
||||
"{%- if add_generation_prompt and not ns.has_system -%}"
|
||||
"{{- '<|start|>system<|message|>You are a helpful assistant.<|eot|>' -}}"
|
||||
"{%- endif -%}"
|
||||
"{%- for message in messages -%}"
|
||||
"{%- set role = message['role'] -%}"
|
||||
"{%- if role == 'assistant' -%}"
|
||||
"{%- set recipient = message.get('recipient') -%}"
|
||||
"{%- set end_turn = message.get('end_turn') -%}"
|
||||
"{%- if end_turn is none -%}"
|
||||
"{%- set end_turn = not (recipient and recipient != 'user') -%}"
|
||||
"{%- endif -%}"
|
||||
"{{- '<|start|>assistant' -}}"
|
||||
"{%- if recipient -%}{{- ' to=' + recipient -}}{%- endif -%}"
|
||||
"{{- '<|message|>' -}}{{- render_parts(message['content']) -}}"
|
||||
"{{- ('<|eot|>' if end_turn else '<|eom|>') -}}"
|
||||
"{%- elif role == 'tool' -%}"
|
||||
"{%- set name = message.get('name', '') -%}"
|
||||
# Tool content is emitted as-is (string body or interleaved image/text parts).
|
||||
# Any <tool_output ...> wrapper is baked into the SFT data content; the
|
||||
# tokenizer does not add it, so the template must not either.
|
||||
"{{- '<|start|>tool ' + name + '<|message|>' -}}"
|
||||
"{{- render_parts(message['content']) -}}"
|
||||
"{{- '<|eot|>' -}}"
|
||||
"{%- else -%}"
|
||||
"{%- set header = role -%}"
|
||||
"{%- if message.get('name') -%}"
|
||||
"{%- set header = role + ' ' + message['name'] -%}"
|
||||
"{%- endif -%}"
|
||||
"{{- '<|start|>' + header + '<|message|>' -}}"
|
||||
"{{- render_parts(message['content']) -}}"
|
||||
"{{- '<|eot|>' -}}"
|
||||
"{%- endif -%}"
|
||||
"{%- endfor -%}"
|
||||
"{%- if add_generation_prompt -%}{{- '<|start|>assistant' -}}{%- endif -%}"
|
||||
)
|
||||
|
||||
|
||||
def _grid_size(
|
||||
img_w: int, img_h: int, patch_hw: int, max_tokens: int
|
||||
) -> tuple[int, int, int]:
|
||||
"""Pick the integer (H, W) grid closest to the aspect ratio under the token cap.
|
||||
|
||||
Replicates MuseGlimmerVisionEncoder._compute_grid_size
|
||||
(modeling_muse_glimmer.py) so the processor needs no torch model import.
|
||||
Returns (target_h, target_w, n_tokens).
|
||||
"""
|
||||
i_nph = img_h / patch_hw
|
||||
i_npw = img_w / patch_hw
|
||||
ratio = i_npw / i_nph if i_nph > 0 else 1.0
|
||||
if i_nph * i_npw > max_tokens:
|
||||
i_nph = (max_tokens / ratio) ** 0.5
|
||||
i_npw = i_nph * ratio
|
||||
candidates = list(
|
||||
set(
|
||||
itertools.product(
|
||||
[math.floor(i_nph), math.ceil(i_nph)],
|
||||
[math.floor(i_npw), math.ceil(i_npw)],
|
||||
)
|
||||
)
|
||||
)
|
||||
candidates = [
|
||||
(nph, npw)
|
||||
for nph, npw in candidates
|
||||
if nph >= 1 and npw >= 1 and nph * npw <= max_tokens
|
||||
]
|
||||
if not candidates:
|
||||
candidates = [(max(1, round(i_nph)), max(1, round(i_npw)))]
|
||||
nph, npw = min(candidates, key=lambda c: abs(c[0] / c[1] - img_h / img_w))
|
||||
return nph * patch_hw, npw * patch_hw, nph * npw
|
||||
|
||||
|
||||
class MuseGlimmerImageProcessor(BaseImageProcessor):
|
||||
"""Resize + normalize MuseGlimmer images and compute patch-token counts.
|
||||
|
||||
Variable-resolution: each image is resized to the grid that best matches its
|
||||
aspect ratio under the per-image token cap, then normalized with mean/std 0.5.
|
||||
Returns per-image tensors (not stacked) because MuseGlimmer consumes a list of
|
||||
variable-size images. Video frames are handled by MuseGlimmerVideoProcessor.
|
||||
"""
|
||||
|
||||
model_input_names = ["pixel_values"]
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
patch_size: int = 14,
|
||||
downsample_factor: int = 2,
|
||||
max_image_tokens: int = 4096,
|
||||
image_mean: float = 0.5,
|
||||
image_std: float = 0.5,
|
||||
**kwargs,
|
||||
):
|
||||
super().__init__(**kwargs)
|
||||
self.patch_size = patch_size
|
||||
self.downsample_factor = downsample_factor
|
||||
self.max_image_tokens = max_image_tokens
|
||||
self.image_mean = image_mean
|
||||
self.image_std = image_std
|
||||
|
||||
def _to_norm_tensor(self, image: Image.Image) -> torch.Tensor:
|
||||
# Functional normalize -- no stored Normalize object, so the processor
|
||||
# stays JSON-serializable for to_dict() / save_pretrained.
|
||||
return T.functional.normalize(
|
||||
T.functional.to_tensor(image),
|
||||
[self.image_mean] * 3,
|
||||
[self.image_std] * 3,
|
||||
)
|
||||
|
||||
# -- size computation (mirrors modeling_muse_glimmer.MuseGlimmerVisionEncoder) --
|
||||
def compute_image_size(self, img_w: int, img_h: int) -> tuple[int, int, int]:
|
||||
ph = self.patch_size * self.downsample_factor
|
||||
return _grid_size(img_w, img_h, ph, self.max_image_tokens)
|
||||
|
||||
# -- preprocessing ---------------------------------------------------------
|
||||
def preprocess_image(self, image: Image.Image) -> tuple[torch.Tensor, int]:
|
||||
"""Return (pixel tensor [3, H, W], n_patch_tokens) for one image."""
|
||||
image = image.convert("RGB")
|
||||
target_h, target_w, n_tokens = self.compute_image_size(
|
||||
image.width, image.height
|
||||
)
|
||||
image = image.resize((target_w, target_h), Image.LANCZOS)
|
||||
return self._to_norm_tensor(image), n_tokens
|
||||
|
||||
|
||||
# torchcodec is the training-faithful video decoder; other decoders (torchvision /
|
||||
# PyAV) seek + color-convert differently and diverge from training. Optional at
|
||||
# import time so text/image paths load without it; decode_video raises if used.
|
||||
try:
|
||||
import torchcodec
|
||||
except Exception:
|
||||
torchcodec = None
|
||||
|
||||
|
||||
class MuseGlimmerVideoProcessor(BaseVideoProcessor):
|
||||
"""MuseGlimmer video preprocessing behind the standard HF
|
||||
``AutoVideoProcessor`` API.
|
||||
|
||||
Exposes MuseGlimmer's training-faithful video handling as a first-class video
|
||||
processor so you can call ``processor(videos="clip.mp4")`` and downstream
|
||||
users can discover it via the standard HF ``AutoVideoProcessor`` API plus a
|
||||
``video_preprocessor_config.json``. It wraps:
|
||||
|
||||
* torchcodec decode (matches training; other decoders diverge),
|
||||
* uniform frame sampling to a whole multiple of ``patch_temporal``,
|
||||
* real per-group PTS timestamps (rendered as ``Time: X.Xs`` by
|
||||
MuseGlimmerProcessor),
|
||||
* ``patch_temporal`` frame-grouping (frames cat on the channel axis ->
|
||||
``[patch_temporal * 3, H, W]``; the encoder detects video by channel count).
|
||||
|
||||
It deliberately overrides ``preprocess`` instead of the BaseVideoProcessor fast
|
||||
pipeline (group_videos_by_shape / smart_resize / single stacked tensor): MuseGlimmer
|
||||
consumes a LIST of variable-size group tensors and needs the real per-group PTS,
|
||||
neither of which the stacked fast path models.
|
||||
"""
|
||||
|
||||
model_input_names = ["pixel_values"]
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
patch_size: int = 14,
|
||||
downsample_factor: int = 2,
|
||||
patch_temporal: int = 2,
|
||||
max_video_frame_tokens: int = 144,
|
||||
image_mean: float = 0.5,
|
||||
image_std: float = 0.5,
|
||||
video_num_frames: int = 96,
|
||||
video_sampling_fps: float = 2.0,
|
||||
**kwargs,
|
||||
):
|
||||
super().__init__(**kwargs)
|
||||
self.patch_size = patch_size
|
||||
self.downsample_factor = downsample_factor
|
||||
self.patch_temporal = patch_temporal
|
||||
self.max_video_frame_tokens = max_video_frame_tokens
|
||||
self.image_mean = image_mean
|
||||
self.image_std = image_std
|
||||
self.video_num_frames = video_num_frames
|
||||
self.video_sampling_fps = video_sampling_fps
|
||||
|
||||
def _to_norm_tensor(self, image: Image.Image) -> torch.Tensor:
|
||||
return T.functional.normalize(
|
||||
T.functional.to_tensor(image),
|
||||
[self.image_mean] * 3,
|
||||
[self.image_std] * 3,
|
||||
)
|
||||
|
||||
def decode_video(self, video_path: str) -> tuple[list[Image.Image], list[float]]:
|
||||
"""Sample frames + per-group PTS with torchcodec (the training decode path).
|
||||
|
||||
``timestamps[g]`` is the ACTUAL decoded PTS of the first frame in temporal
|
||||
group ``g``; ``len(frames)`` is a whole multiple of ``patch_temporal``.
|
||||
"""
|
||||
if torchcodec is None:
|
||||
raise RuntimeError(
|
||||
"torchcodec is required for video decoding (it matches the training "
|
||||
"decode path). See the Environment Setup steps in hf/README.md."
|
||||
)
|
||||
pt = self.patch_temporal
|
||||
reader = torchcodec.decoders.VideoDecoder(video_path)
|
||||
total = len(reader)
|
||||
assert reader.metadata.average_fps is not None, (
|
||||
f"Video has no FPS metadata: {video_path}"
|
||||
)
|
||||
fps = reader.metadata.average_fps
|
||||
assert self.video_sampling_fps and self.video_sampling_fps > 0, (
|
||||
f"video_sampling_fps must be positive, got {self.video_sampling_fps}"
|
||||
)
|
||||
n = min(
|
||||
int(total * self.video_sampling_fps / fps), self.video_num_frames, total
|
||||
)
|
||||
n = max(pt, (n // pt) * pt)
|
||||
n = min(n, total)
|
||||
if n < pt:
|
||||
raise ValueError(
|
||||
f"Video has only {total} decodable frame(s) but needs at least "
|
||||
f"{pt} (one temporal patch): {video_path}"
|
||||
)
|
||||
indices = torch.linspace(0, total - 1, n).long().tolist()
|
||||
frames: list[Image.Image] = []
|
||||
timestamps: list[float] = []
|
||||
for j, i in enumerate(indices):
|
||||
fr = reader[i]
|
||||
frames.append(
|
||||
Image.fromarray(fr.data.permute(1, 2, 0).numpy()).convert("RGB")
|
||||
)
|
||||
if j % pt == 0:
|
||||
pts = getattr(fr, "pts_seconds", None)
|
||||
timestamps.append(float(pts) if pts is not None else i / fps)
|
||||
return frames, timestamps
|
||||
|
||||
def compute_video_frame_size(self, img_w: int, img_h: int) -> tuple[int, int, int]:
|
||||
"""Pick (H, W, tokens_per_frame) for a video frame under the per-frame cap.
|
||||
|
||||
Mirrors MuseGlimmerImageProcessor.compute_image_size but uses the (smaller)
|
||||
per-frame video token budget; shares the same ``_grid_size`` grid logic as
|
||||
modeling_muse_glimmer.MuseGlimmerVisionEncoder.
|
||||
"""
|
||||
ph = self.patch_size * self.downsample_factor
|
||||
return _grid_size(img_w, img_h, ph, self.max_video_frame_tokens)
|
||||
|
||||
def _group_frames(
|
||||
self, frames: list[Image.Image]
|
||||
) -> tuple[list[torch.Tensor], int, int]:
|
||||
"""Resize/normalize frames and cat ``patch_temporal`` frames per group.
|
||||
|
||||
Returns (group_tensors, n_groups, tokens_per_group).
|
||||
"""
|
||||
pt = self.patch_temporal
|
||||
if pt <= 0:
|
||||
raise ValueError(f"patch_temporal must be positive, got {pt}")
|
||||
if not frames:
|
||||
raise ValueError("video must contain at least one frame")
|
||||
if padding := (-len(frames)) % pt:
|
||||
frames = [*frames, *([frames[-1]] * padding)]
|
||||
first = frames[0].convert("RGB")
|
||||
target_h, target_w, n_tokens = self.compute_video_frame_size(
|
||||
first.width, first.height
|
||||
)
|
||||
groups: list[torch.Tensor] = []
|
||||
for i in range(0, len(frames), pt):
|
||||
grp = [
|
||||
self._to_norm_tensor(
|
||||
frames[i + j]
|
||||
.convert("RGB")
|
||||
.resize((target_w, target_h), Image.LANCZOS)
|
||||
)
|
||||
for j in range(pt)
|
||||
]
|
||||
groups.append(torch.cat(grp, dim=0))
|
||||
return groups, len(groups), n_tokens
|
||||
|
||||
@staticmethod
|
||||
def _normalize_videos(videos) -> list:
|
||||
"""Normalize the ``videos`` arg to a list of per-video items.
|
||||
|
||||
A video item is a path (str/Path) or a list of PIL frames. Accepts a
|
||||
single video (one path, or one list of frames) or a list of those.
|
||||
"""
|
||||
if isinstance(videos, (str, Path)):
|
||||
return [videos]
|
||||
if videos and not isinstance(videos[0], (list, tuple, str, Path)):
|
||||
return [videos] # a single list of PIL frames
|
||||
return list(videos or [])
|
||||
|
||||
def preprocess_one(
|
||||
self,
|
||||
video: str | Path | list[Image.Image],
|
||||
timestamps: list[float] | None = None,
|
||||
) -> tuple[list[torch.Tensor], int, int, list[float]]:
|
||||
"""Decode (if a path) + group ONE video.
|
||||
|
||||
Returns (group_tensors, n_groups, tokens_per_group, group_timestamps). This
|
||||
is the single per-video implementation shared by ``preprocess`` (the HF
|
||||
AutoVideoProcessor batch API) and ``MuseGlimmerProcessor.__call__`` (prompt
|
||||
building), so the two paths cannot drift. For a path, frames + real
|
||||
per-group PTS come from ``decode_video``; for pre-decoded frames the given
|
||||
``timestamps`` are used verbatim (empty if not supplied).
|
||||
"""
|
||||
if isinstance(video, (str, Path)):
|
||||
frames, ts = self.decode_video(str(video))
|
||||
else:
|
||||
frames = [f.convert("RGB") for f in video]
|
||||
ts = list(timestamps) if timestamps is not None else []
|
||||
groups, n_groups, tokens_per_group = self._group_frames(frames)
|
||||
return groups, n_groups, tokens_per_group, ts
|
||||
|
||||
def preprocess(
|
||||
self,
|
||||
videos,
|
||||
video_timestamps: list[list[float]] | None = None,
|
||||
return_tensors: str | None = "pt",
|
||||
**kwargs,
|
||||
) -> BatchFeature:
|
||||
"""Preprocess video file path(s) or pre-decoded frame list(s).
|
||||
|
||||
Accepts a single video (path str, or list of PIL frames) or a list of
|
||||
those. Returns a BatchFeature carrying, in video order:
|
||||
* ``pixel_values`` -- flat list of [pt*3, H, W] group tensors,
|
||||
* ``video_num_groups`` -- groups per video,
|
||||
* ``video_tokens_per_group`` -- patch-token count per group per video,
|
||||
* ``video_timestamps`` -- per-group PTS per video,
|
||||
i.e. exactly what MuseGlimmerProcessor needs to build the
|
||||
``<|vid_start|> ( Time: X.Xs <|video|>*P [sep] )* <|vid_end|>`` block.
|
||||
"""
|
||||
videos = self._normalize_videos(videos)
|
||||
pixel_values: list[torch.Tensor] = []
|
||||
num_groups: list[int] = []
|
||||
tokens_per_group: list[int] = []
|
||||
out_ts: list[list[float]] = []
|
||||
for idx, v in enumerate(videos):
|
||||
ts_in = video_timestamps[idx] if video_timestamps else None
|
||||
groups, ng, tpg, ts = self.preprocess_one(v, ts_in)
|
||||
pixel_values += groups
|
||||
num_groups.append(ng)
|
||||
tokens_per_group.append(tpg)
|
||||
out_ts.append(ts)
|
||||
batch = BatchFeature(
|
||||
data={
|
||||
"video_num_groups": num_groups,
|
||||
"video_tokens_per_group": tokens_per_group,
|
||||
"video_timestamps": out_ts,
|
||||
},
|
||||
tensor_type=None,
|
||||
)
|
||||
# Variable-size list -- BatchFeature would fail to stack into one tensor.
|
||||
batch["pixel_values"] = pixel_values
|
||||
return batch
|
||||
|
||||
|
||||
class MuseGlimmerProcessor(ProcessorMixin):
|
||||
"""Bundle MuseGlimmerImageProcessor + MuseGlimmerVideoProcessor + tokenizer;
|
||||
expand media sentinels into spans.
|
||||
|
||||
Images go through ``image_processor`` (MuseGlimmerImageProcessor) and videos through
|
||||
``video_processor`` (MuseGlimmerVideoProcessor) -- one preprocessing implementation
|
||||
each, no overlap.
|
||||
"""
|
||||
|
||||
attributes = ["image_processor", "video_processor", "tokenizer"]
|
||||
image_processor_class = "AutoImageProcessor"
|
||||
video_processor_class = "AutoVideoProcessor"
|
||||
tokenizer_class = "PreTrainedTokenizerFast"
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
image_processor=None,
|
||||
video_processor=None,
|
||||
tokenizer=None,
|
||||
chat_template=None,
|
||||
**kwargs,
|
||||
):
|
||||
if image_processor is None:
|
||||
image_processor = MuseGlimmerImageProcessor()
|
||||
if video_processor is None:
|
||||
video_processor = MuseGlimmerVideoProcessor()
|
||||
super().__init__(
|
||||
image_processor,
|
||||
video_processor,
|
||||
tokenizer,
|
||||
chat_template=chat_template or MUSE_GLIMMER_MM_CHAT_TEMPLATE,
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
def _sid(self, token: str) -> int:
|
||||
return self.tokenizer.convert_tokens_to_ids(token)
|
||||
|
||||
def _image_block(self, n_tokens: int) -> list[int]:
|
||||
return (
|
||||
[self._sid("<|image_start|>")]
|
||||
+ [self._sid("<|patch|>")] * n_tokens
|
||||
+ [self._sid("<|image_end|>")]
|
||||
)
|
||||
|
||||
def _video_block(
|
||||
self,
|
||||
n_groups: int,
|
||||
tokens_per_group: int,
|
||||
timestamps: list[float] | None = None,
|
||||
) -> list[int]:
|
||||
"""Per-group ``Time: X.Xs`` + <|video|>*P, separated/terminated.
|
||||
|
||||
``timestamps`` (one per temporal group, in seconds) should be the ACTUAL
|
||||
decoded frame times -- training renders the real factored frame PTS, not a
|
||||
uniform grid. Falls back to ``g*patch_temporal/fps`` only when timestamps
|
||||
are not supplied (e.g. callers that don't track frame times); that
|
||||
approximation can differ from training by the per-frame jitter.
|
||||
"""
|
||||
vid = self._sid("<|video|>")
|
||||
sep = self._sid("<|vid_frame_separator|>")
|
||||
pt = self.video_processor.patch_temporal
|
||||
fps = self.video_processor.video_sampling_fps
|
||||
block = [self._sid("<|vid_start|>")]
|
||||
for g in range(n_groups):
|
||||
ts = timestamps[g] if timestamps is not None else g * pt / fps
|
||||
block += self.tokenizer.encode(f"Time: {ts:.1f}s", add_special_tokens=False)
|
||||
block += [vid] * tokens_per_group
|
||||
block.append(sep if g < n_groups - 1 else self._sid("<|vid_end|>"))
|
||||
return block
|
||||
|
||||
def __call__(
|
||||
self,
|
||||
text: str | list[str] | None = None,
|
||||
images: list[Image.Image] | None = None,
|
||||
videos: list[list[Image.Image] | str] | None = None,
|
||||
video_timestamps: list[list[float]] | None = None,
|
||||
return_tensors: str | None = "pt",
|
||||
**kwargs,
|
||||
) -> BatchFeature:
|
||||
if text is None:
|
||||
raise ValueError(
|
||||
"`text` is required (use apply_chat_template to build it)."
|
||||
)
|
||||
if isinstance(text, (list, tuple)):
|
||||
if len(text) != 1:
|
||||
raise ValueError(
|
||||
"MuseGlimmerProcessor supports a single text sample per call."
|
||||
)
|
||||
text = text[0]
|
||||
|
||||
images = list(images or [])
|
||||
videos = list(videos or [])
|
||||
image_sentinel = self._sid(IMAGE_SENTINEL)
|
||||
video_sentinel = self._sid(VIDEO_SENTINEL)
|
||||
|
||||
# Preprocess media up front so we can expand sentinels in document order.
|
||||
# Images via image_processor; videos via video_processor (each video item
|
||||
# is a path -- decoded here -- or a pre-decoded list of PIL frames).
|
||||
prepped_images = [self.image_processor.preprocess_image(im) for im in images]
|
||||
prepped_videos = [
|
||||
self.video_processor.preprocess_one(
|
||||
v, video_timestamps[i] if video_timestamps else None
|
||||
)
|
||||
for i, v in enumerate(videos)
|
||||
]
|
||||
|
||||
ids = self.tokenizer.encode(text, add_special_tokens=False)
|
||||
n_img = sum(1 for t in ids if t == image_sentinel)
|
||||
n_vid = sum(1 for t in ids if t == video_sentinel)
|
||||
if n_img != len(prepped_images):
|
||||
raise ValueError(
|
||||
f"{n_img} image sentinel(s) in text but "
|
||||
f"{len(prepped_images)} image(s) given."
|
||||
)
|
||||
if n_vid != len(prepped_videos):
|
||||
raise ValueError(
|
||||
f"{n_vid} video sentinel(s) in text but "
|
||||
f"{len(prepped_videos)} video(s) given."
|
||||
)
|
||||
|
||||
out_ids: list[int] = []
|
||||
pixel_values: list[torch.Tensor] = []
|
||||
img_i = vid_i = 0
|
||||
for tid in ids:
|
||||
if tid == image_sentinel:
|
||||
tensor, n_tokens = prepped_images[img_i]
|
||||
img_i += 1
|
||||
out_ids += self._image_block(n_tokens)
|
||||
pixel_values.append(tensor)
|
||||
elif tid == video_sentinel:
|
||||
groups, n_groups, tokens_per_group, ts = prepped_videos[vid_i]
|
||||
vid_i += 1
|
||||
out_ids += self._video_block(n_groups, tokens_per_group, ts or None)
|
||||
pixel_values += groups
|
||||
else:
|
||||
out_ids.append(tid)
|
||||
|
||||
data: dict = {
|
||||
"input_ids": [out_ids],
|
||||
"attention_mask": [[1] * len(out_ids)],
|
||||
}
|
||||
batch = BatchFeature(data=data, tensor_type=return_tensors)
|
||||
# Keep pixel_values as a list of variable-size tensors (MuseGlimmerModel
|
||||
# consumes a list); BatchFeature would fail to stack them into one tensor.
|
||||
if pixel_values:
|
||||
batch["pixel_values"] = pixel_values
|
||||
return batch
|
||||
|
||||
|
||||
__all__ = [
|
||||
"MuseGlimmerImageProcessor",
|
||||
"MuseGlimmerVideoProcessor",
|
||||
"MuseGlimmerProcessor",
|
||||
"MUSE_GLIMMER_MM_CHAT_TEMPLATE",
|
||||
]
|
||||
Reference in New Issue
Block a user