[Mini-VLLM] Cut dead subsystems across serving, connectors, parsers

- Remove P/D transfer connectors: nixl, nixl_chat, mooncake x2, hf3fs,
  moriio, hisparse, example and hidden-states connectors
- Reduce weight_transfer to the base API used by model loading
- Remove kv_offload tiering, simple_kv_offload, ec cpu/example,
  snapshot command, engine snapshot server
- Remove transformers model fallback, dsv4 tokenizers, dead platforms
  cpu/tpu/xpu, flashmla third_party, dsa, megamoe/flydsl, dead attn
  ops, dead layers (short_conv, fused_embed_norm, indexer_topk,
  relu2 gemma_ar, lightning_attn, minimax_rms_norm), gemm_rs_ar
- Keep MoE LoRA op (imported eagerly by punica_gpu); drop fp8 variant
- Rebuild parser hubs around Qwen3 adapters only (36 dead adapters)
- Remove dead tests with the code they covered

Co-authored-by: Hinata <hinata@idihore.id>
Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
2026-10-19 02:19:03 +07:00
co-authored by Hinata
parent 54a396586e
commit 2b544b7ba0
353 changed files with 29 additions and 121269 deletions
+1 -61
View File
@@ -14,71 +14,11 @@ from __future__ import annotations
import importlib
_CLASS_TO_MODULE: dict[str, str] = {
"BagelConfig": "vllm.transformers_utils.configs.bagel",
"BailingMoeV3TextConfig": "vllm.transformers_utils.configs.bailing_moe_v3_vl",
"BailingMoeV3VisionConfig": "vllm.transformers_utils.configs.bailing_moe_v3_vl",
"BailingMoeV3VLConfig": "vllm.transformers_utils.configs.bailing_moe_v3_vl",
"ChatGLMConfig": "vllm.transformers_utils.configs.chatglm",
"ColModernVBertConfig": "vllm.transformers_utils.configs.colmodernvbert",
"ColPaliConfig": "vllm.transformers_utils.configs.colpali",
"ColQwen3Config": "vllm.transformers_utils.configs.colqwen3",
"OpsColQwen3Config": "vllm.transformers_utils.configs.colqwen3",
"Qwen3VLNemotronEmbedConfig": "vllm.transformers_utils.configs.colqwen3",
"Cosmos3EdgeConfig": "vllm.transformers_utils.configs.cosmos3_edge",
"Cosmos3EdgeProjectorConfig": "vllm.transformers_utils.configs.cosmos3_edge",
"Cosmos3EdgeTextConfig": "vllm.transformers_utils.configs.cosmos3_edge",
"Cosmos3EdgeVisionConfig": "vllm.transformers_utils.configs.cosmos3_edge",
"DeepseekVLV2Config": "vllm.transformers_utils.configs.deepseek_vl2",
"DeepseekV41Config": "vllm.transformers_utils.configs.deepseek_v41",
"Dots3NoteConfig": "vllm.transformers_utils.configs.dots3_note",
"K3DSparkConfig": "vllm.transformers_utils.configs.k3_dspark",
"DotsOCRConfig": "vllm.transformers_utils.configs.dotsocr",
"EAGLEConfig": "vllm.transformers_utils.configs.eagle",
"FunAudioChatConfig": "vllm.transformers_utils.configs.funaudiochat",
"FunAudioChatAudioEncoderConfig": "vllm.transformers_utils.configs.funaudiochat",
"HYV4Config": "vllm.transformers_utils.configs.hy_v4",
"IsaacConfig": "vllm.transformers_utils.configs.isaac",
# RWConfig is for the original tiiuae/falcon-40b(-instruct) and
# tiiuae/falcon-7b(-instruct) models. Newer Falcon models will use the
# `FalconConfig` class from the official HuggingFace transformers library.
"RWConfig": "vllm.transformers_utils.configs.falcon",
"MiDashengLMConfig": "vllm.transformers_utils.configs.midashenglm",
"MiniMaxM3Config": "vllm.transformers_utils.configs.minimax_m3",
"MiniMaxM3MTPConfig": "vllm.transformers_utils.configs.minimax_m3",
"MiniMaxM3TextConfig": "vllm.transformers_utils.configs.minimax_m3",
"Moondream3Config": "vllm.transformers_utils.configs.moondream3",
"Moondream3TextConfig": "vllm.transformers_utils.configs.moondream3",
"Moondream3VisionConfig": "vllm.transformers_utils.configs.moondream3",
"MossTranscribeDiarizeConfig": (
"vllm.transformers_utils.configs.moss_transcribe_diarize"
),
"MoonViTConfig": "vllm.transformers_utils.configs.moonvit",
"KimiLinearConfig": "vllm.transformers_utils.configs.kimi_linear",
"KimiVLConfig": "vllm.transformers_utils.configs.kimi_vl",
"MuseGlimmerConfig": "vllm.transformers_utils.configs.muse_glimmer",
"MuseGlimmerTextConfig": "vllm.transformers_utils.configs.muse_glimmer",
"MuseGlimmerVisionConfig": "vllm.transformers_utils.configs.muse_glimmer",
"MuseGlimmerAssistantConfig": "vllm.transformers_utils.configs.muse_glimmer",
"KimiK3Config": "vllm.transformers_utils.configs.kimi_k3",
"KimiK3VisionConfig": "vllm.transformers_utils.configs.kimi_k3",
"OpenVLAConfig": "vllm.transformers_utils.configs.openvla",
"OvisConfig": "vllm.transformers_utils.configs.ovis",
"PixelShuffleSiglip2VisionConfig": "vllm.transformers_utils.configs.isaac",
"RadioConfig": "vllm.transformers_utils.configs.radio",
"SpeculatorsConfig": "vllm.transformers_utils.configs.speculators",
"UltravoxConfig": "vllm.transformers_utils.configs.ultravox",
"UnlimitedOCRConfig": "vllm.transformers_utils.configs.unlimited_ocr",
"Step3VLConfig": "vllm.transformers_utils.configs.step3_vl",
"Step3VisionEncoderConfig": "vllm.transformers_utils.configs.step3_vl",
"Step3TextConfig": "vllm.transformers_utils.configs.step3_vl",
"Step3p5Config": "vllm.transformers_utils.configs.step3p5",
"QianfanOCRConfig": "vllm.transformers_utils.configs.qianfan_ocr",
"QianfanOCRVisionConfig": "vllm.transformers_utils.configs.qianfan_ocr",
"Qwen3ASRConfig": "vllm.transformers_utils.configs.qwen3_asr",
"InklingModelConfig": "vllm.models.inkling.configs",
"InklingAudioConfig": "vllm.models.inkling.configs",
"InklingVisionConfig": "vllm.models.inkling.configs",
"InklingMMConfig": "vllm.models.inkling.configs",
"MossTranscribeDiarizeConfig": (),
# Upstream Transformers classes registered in _CONFIG_REGISTRY
"DeepseekV3Config": "transformers",
"HyperCLOVAXConfig": "transformers",
@@ -1,345 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
# Copyright 2025 Antgroup and The HuggingFace Inc. team. All rights reserved.
# Adapted from
# https://huggingface.co/inclusionAI/Ling-3.0-flash/blob/51fd444268df5267074ef0b289de0e1c4cb9b381/configuration_bailing_moe_v3.py
"""Bailing V3 vision-language model configuration."""
from __future__ import annotations
from typing import Any
from transformers.configuration_utils import PreTrainedConfig
def _normalize_mrope_parameters(
config: PreTrainedConfig,
mrope_section: list[int],
) -> None:
"""Add the top-level Bailing M-RoPE split to a text config."""
rope_parameters = dict(getattr(config, "rope_parameters", None) or {})
text_mrope_section = rope_parameters.get("mrope_section")
if text_mrope_section is not None and list(text_mrope_section) != mrope_section:
raise ValueError(
"Bailing V3 VL has conflicting M-RoPE sections: "
f"text={text_mrope_section}, top-level={mrope_section}"
)
if "rope_type" not in rope_parameters and "type" not in rope_parameters:
rope_parameters["rope_type"] = "default"
rope_parameters.setdefault("rope_theta", getattr(config, "rope_theta", 10_000))
rope_parameters["mrope_section"] = list(mrope_section)
config.rope_parameters = rope_parameters
def _build_layer_types(
num_hidden_layers: int,
layer_group_size: int,
) -> list[str]:
if layer_group_size <= 0:
raise ValueError(
f"Bailing V3 VL layer_group_size must be positive, got {layer_group_size}"
)
grouped_layers = num_hidden_layers // layer_group_size * layer_group_size
return [
"linear_attention"
if ((layer_idx + 1) % layer_group_size != 0 and layer_idx < grouped_layers)
else "full_attention"
for layer_idx in range(num_hidden_layers)
]
class BailingMoeV3TextConfig(PreTrainedConfig):
model_type = "bailing_hybrid"
base_config_key = "text_config"
keys_to_ignore_at_inference = ["past_key_values"]
ignore_keys_at_rope_validation = {"mrope_section"}
def __init__(
self,
*,
vocab_size: int = 157184,
hidden_size: int = 2048,
intermediate_size: int = 5120,
num_hidden_layers: int = 20,
num_attention_heads: int = 16,
num_key_value_heads: int = 4,
hidden_act: str = "silu",
use_qkv_bias: bool = False,
use_bias: bool = False,
rms_norm_eps: float = 1e-6,
tie_word_embeddings: bool = False,
embedding_dropout: float = 0.0,
attention_dropout: float = 0.0,
output_dropout: float = 0.0,
initializer_range: float = 0.02,
max_position_embeddings: int = 32768,
rope_theta: float = 600000.0,
rope_parameters: dict[str, Any] | None = None,
rope_scaling: dict[str, Any] | None = None,
partial_rotary_factor: float | None = None,
mrope_section: list[int] | tuple[int, ...] | None = None,
use_cache: bool = True,
max_window_layers: int = 20,
pad_token_id: int = 156892,
eos_token_id: int = 156892,
num_experts: int = 256,
num_shared_experts: int = 1,
num_experts_per_tok: int = 8,
n_group: int = 8,
topk_group: int = 4,
moe_intermediate_size: int = 512,
moe_shared_expert_intermediate_size: int = 512,
first_k_dense_replace: int = 1,
head_dim: int | None = 128,
output_router_logits: bool = False,
use_qk_norm: bool = True,
num_nextn_predict_layers: int = 0,
mtp_loss_scaling_factor: float = 0,
moe_router_enable_expert_bias: bool = True,
routed_scaling_factor: float = 1.0,
layer_group_size: int = 5,
layer_types: list[str] | None = None,
layers_block_type: list[str] | None = None,
kv_lora_rank: int = 512,
q_lora_rank: int | None = None,
qk_rope_head_dim: int = 64,
v_head_dim: int = 128,
qk_nope_head_dim: int = 128,
rope_interleave: bool = True,
score_function: str = "sigmoid",
scoring_func: str = "sigmoid",
seq_aux: bool = True,
topk_method: str = "noaux_tc",
router_dtype: str = "fp32",
gated_attention_proj_granularity_type: str | None = None,
no_kda_lora: bool = False,
kda_safe_gate: bool = False,
kda_lower_bound: float | None = None,
short_conv_kernel_size: int = 4,
architectures: list[str] | None = None,
**kwargs: Any,
) -> None:
kwargs.pop("model_type", None)
if architectures is None:
architectures = ["BailingMoeV3ForCausalLM"]
if rope_parameters is not None:
self.rope_parameters = dict(rope_parameters)
elif rope_scaling is not None:
self.rope_parameters = dict(rope_scaling)
else:
self.rope_parameters = {}
if (
"rope_type" not in self.rope_parameters
and "type" not in self.rope_parameters
):
self.rope_parameters["rope_type"] = "default"
self.rope_parameters.setdefault("rope_theta", rope_theta)
if mrope_section is not None:
_normalize_mrope_parameters(self, list(mrope_section))
self.vocab_size = vocab_size
self.hidden_size = hidden_size
self.intermediate_size = intermediate_size
self.num_hidden_layers = num_hidden_layers
self.num_attention_heads = num_attention_heads
self.num_key_value_heads = num_key_value_heads
self.hidden_act = hidden_act
self.use_qkv_bias = use_qkv_bias
self.use_bias = use_bias
self.rms_norm_eps = rms_norm_eps
self.embedding_dropout = embedding_dropout
self.attention_dropout = attention_dropout
self.output_dropout = output_dropout
self.initializer_range = initializer_range
self.max_position_embeddings = max_position_embeddings
self.rope_theta = rope_theta
self.partial_rotary_factor = partial_rotary_factor
self.use_cache = use_cache
self.max_window_layers = max_window_layers
self.num_experts = num_experts
self.num_shared_experts = num_shared_experts
self.num_experts_per_tok = num_experts_per_tok
self.n_group = n_group
self.topk_group = topk_group
self.moe_intermediate_size = moe_intermediate_size
self.moe_shared_expert_intermediate_size = moe_shared_expert_intermediate_size
self.first_k_dense_replace = first_k_dense_replace
self.head_dim = head_dim or hidden_size // num_attention_heads
self.output_router_logits = output_router_logits
self.use_qk_norm = use_qk_norm
self.num_nextn_predict_layers = num_nextn_predict_layers
self.mtp_loss_scaling_factor = mtp_loss_scaling_factor
self.moe_router_enable_expert_bias = moe_router_enable_expert_bias
self.routed_scaling_factor = routed_scaling_factor
self.layer_group_size = layer_group_size
self.kv_lora_rank = kv_lora_rank
self.q_lora_rank = q_lora_rank
self.qk_rope_head_dim = qk_rope_head_dim
self.v_head_dim = v_head_dim
self.qk_nope_head_dim = qk_nope_head_dim
self.qk_head_dim = qk_nope_head_dim + qk_rope_head_dim
self.rope_interleave = rope_interleave
self.score_function = score_function
self.scoring_func = scoring_func
self.seq_aux = seq_aux
self.topk_method = topk_method
self.router_dtype = router_dtype
self.gated_attention_proj_granularity_type = (
gated_attention_proj_granularity_type
)
self.no_kda_lora = no_kda_lora
self.kda_safe_gate = kda_safe_gate
self.kda_lower_bound = kda_lower_bound
self.short_conv_kernel_size = short_conv_kernel_size
expected_layer_types = _build_layer_types(
num_hidden_layers,
layer_group_size,
)
if layers_block_type is not None:
legacy_type_mapping = {
"attention": "full_attention",
"mamba": "linear_attention",
"conv": "linear_attention",
}
normalized_block_types = [
legacy_type_mapping.get(layer_type, layer_type)
for layer_type in layers_block_type
]
if layer_types is not None and layer_types != normalized_block_types:
raise ValueError(
"Bailing V3 VL has conflicting layer_types and layers_block_type"
)
layer_types = normalized_block_types
if layer_types is not None and layer_types != expected_layer_types:
raise ValueError(
"Bailing V3 VL layer_types must match the layer_group_size schedule"
)
self.layer_types = expected_layer_types
super().__init__(
architectures=architectures,
pad_token_id=pad_token_id,
eos_token_id=eos_token_id,
tie_word_embeddings=tie_word_embeddings,
**kwargs,
)
self.validate_layer_type()
class BailingMoeV3VisionConfig(PreTrainedConfig):
model_type = "qwen3_moe_vit"
base_config_key = "vision_config"
def __init__(
self,
*,
depth: int = 27,
hidden_size: int = 1152,
hidden_act: str = "gelu_pytorch_tanh",
intermediate_size: int = 4304,
num_heads: int = 16,
in_channels: int = 3,
patch_size: int = 16,
spatial_merge_size: int = 2,
temporal_patch_size: int = 2,
out_hidden_size: int = 3584,
num_position_embeddings: int = 2304,
deepstack_visual_indexes: list[int] | None = None,
initializer_range: float = 0.02,
**kwargs: Any,
) -> None:
kwargs.pop("model_type", None)
super().__init__(**kwargs)
self.depth = depth
self.hidden_size = hidden_size
self.hidden_act = hidden_act
self.intermediate_size = intermediate_size
self.num_heads = num_heads
self.in_channels = in_channels
self.patch_size = patch_size
self.spatial_merge_size = spatial_merge_size
self.temporal_patch_size = temporal_patch_size
self.out_hidden_size = out_hidden_size
self.num_position_embeddings = num_position_embeddings
self.deepstack_visual_indexes = (
[8, 16, 24]
if deepstack_visual_indexes is None
else deepstack_visual_indexes
)
self.initializer_range = initializer_range
class BailingMoeV3VLConfig(PreTrainedConfig):
model_type = "bailing_moe_v3_vl"
keys_to_ignore_at_inference = ["past_key_values"]
sub_configs = {
"text_config": BailingMoeV3TextConfig,
"vision_config": BailingMoeV3VisionConfig,
}
def __init__(
self,
*,
text_config: BailingMoeV3TextConfig | dict[str, Any] | None = None,
vision_config: BailingMoeV3VisionConfig | dict[str, Any] | None = None,
image_token_id: int = 151655,
video_token_id: int = 151656,
vision_start_token_id: int = 151652,
vision_end_token_id: int = 151653,
tie_word_embeddings: bool = False,
mrope_section: list[int] | tuple[int, ...] | None = None,
architectures: list[str] | None = None,
**kwargs: Any,
) -> None:
kwargs.pop("model_type", None)
if mrope_section is None:
mrope_section = [12, 10, 10]
self.mrope_section = list(mrope_section)
if text_config is None:
text_config = BailingMoeV3TextConfig()
elif isinstance(text_config, dict):
text_config = dict(text_config)
nested_mrope_section = text_config.pop("mrope_section", None)
if (
nested_mrope_section is not None
and list(nested_mrope_section) != self.mrope_section
):
raise ValueError(
"Bailing V3 VL has conflicting M-RoPE sections: "
f"text={nested_mrope_section}, "
f"top-level={self.mrope_section}"
)
text_config = BailingMoeV3TextConfig(**text_config)
self.text_config = text_config
_normalize_mrope_parameters(self.text_config, self.mrope_section)
if vision_config is None:
vision_config = BailingMoeV3VisionConfig()
elif isinstance(vision_config, dict):
vision_config = BailingMoeV3VisionConfig(**vision_config)
self.vision_config = vision_config
self.image_token_id = image_token_id
self.video_token_id = video_token_id
self.vision_start_token_id = vision_start_token_id
self.vision_end_token_id = vision_end_token_id
if architectures is None:
architectures = ["BailingMoeV3VLForConditionalGeneration"]
super().__init__(
architectures=architectures,
tie_word_embeddings=tie_word_embeddings,
**kwargs,
)
__all__ = [
"BailingMoeV3TextConfig",
"BailingMoeV3VisionConfig",
"BailingMoeV3VLConfig",
]
@@ -1,75 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
# Adapted from
# https://github.com/zai-org/ChatGLM2-6B
from transformers import PreTrainedConfig
class ChatGLMConfig(PreTrainedConfig):
model_type = "chatglm"
attribute_map = {
"num_hidden_layers": "num_layers",
"n_head_kv": "multi_query_group_num",
}
def __init__(
self,
num_layers=28,
padded_vocab_size=65024,
hidden_size=4096,
ffn_hidden_size=13696,
kv_channels=128,
num_attention_heads=32,
seq_length=2048,
hidden_dropout=0.0,
attention_dropout=0.0,
layernorm_epsilon=1e-5,
rmsnorm=True,
apply_residual_connection_post_layernorm=False,
post_layer_norm=True,
add_bias_linear=False,
add_qkv_bias=False,
interleaved_qkv=False,
bias_dropout_fusion=True,
multi_query_attention=False,
multi_query_group_num=1,
apply_query_key_layer_scaling=True,
attention_softmax_in_fp32=True,
fp32_residual_connection=False,
quantization_bit=0,
pre_seq_len=None,
prefix_projection=False,
**kwargs,
):
self.num_layers = num_layers
self.vocab_size = padded_vocab_size
self.padded_vocab_size = padded_vocab_size
self.hidden_size = hidden_size
self.ffn_hidden_size = ffn_hidden_size
self.kv_channels = kv_channels
self.num_attention_heads = num_attention_heads
self.seq_length = seq_length
# It is to be compatible with long lora.
self.max_position_embeddings = seq_length
self.hidden_dropout = hidden_dropout
self.attention_dropout = attention_dropout
self.layernorm_epsilon = layernorm_epsilon
self.rmsnorm = rmsnorm
self.apply_residual_connection_post_layernorm = (
apply_residual_connection_post_layernorm
)
self.post_layer_norm = post_layer_norm
self.add_bias_linear = add_bias_linear
self.add_qkv_bias = add_qkv_bias
self.bias_dropout_fusion = bias_dropout_fusion
self.multi_query_attention = multi_query_attention
self.multi_query_group_num = multi_query_group_num
self.apply_query_key_layer_scaling = apply_query_key_layer_scaling
self.attention_softmax_in_fp32 = attention_softmax_in_fp32
self.fp32_residual_connection = fp32_residual_connection
self.quantization_bit = quantization_bit
self.pre_seq_len = pre_seq_len
self.prefix_projection = prefix_projection
self.interleaved_qkv = interleaved_qkv
super().__init__(**kwargs)
@@ -1,63 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Configuration for ColModernVBERT visual document retrieval model.
ColModernVBERT combines SigLIP vision encoder + ModernBERT text encoder
with a pixel shuffle connector and ColBERT-style 128-dim per-token embeddings.
Reference: https://huggingface.co/ModernVBERT/colmodernvbert-merged
"""
from transformers import ModernBertConfig, PreTrainedConfig, SiglipVisionConfig
class ColModernVBertConfig(PreTrainedConfig):
model_type = "colmodernvbert"
def __init__(
self,
embedding_dim: int = 128,
image_token_id: int = 50407,
pixel_shuffle_factor: int = 4,
text_config: dict | None = None,
vision_config: dict | None = None,
**kwargs,
):
self.embedding_dim = embedding_dim
self.image_token_id = image_token_id
self.pixel_shuffle_factor = pixel_shuffle_factor
text_config = text_config or {}
self.hidden_size = text_config.get("hidden_size", 768)
self.text_config = ModernBertConfig(
vocab_size=text_config.get("vocab_size", 50408),
hidden_size=text_config.get("hidden_size", 768),
intermediate_size=text_config.get("intermediate_size", 1152),
num_hidden_layers=text_config.get("num_hidden_layers", 22),
num_attention_heads=text_config.get("num_attention_heads", 12),
mlp_bias=text_config.get("mlp_bias", False),
max_position_embeddings=text_config.get("max_position_embeddings", 8192),
)
vision_config = vision_config or {}
self.vision_config = SiglipVisionConfig(
hidden_size=vision_config.get("hidden_size", 768),
image_size=vision_config.get("image_size", 512),
patch_size=vision_config.get("patch_size", 16),
num_hidden_layers=vision_config.get("num_hidden_layers", 12),
intermediate_size=vision_config.get("intermediate_size", 3072),
num_attention_heads=vision_config.get("num_attention_heads", 12),
)
# Ensure architectures is set so vLLM routes to our model class
kwargs.setdefault("architectures", ["ColModernVBertForRetrieval"])
super().__init__(**kwargs)
@property
def image_seq_len(self) -> int:
ps = self.vision_config.image_size // self.vision_config.patch_size
return (ps * ps) // (self.pixel_shuffle_factor**2)
def get_text_config(self, **kwargs):
return self.text_config
@@ -1,56 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""ColPali configuration that extends PaliGemmaConfig with embedding projection
fields. This allows ColPali models to be loaded without trust_remote_code
by mapping their custom model_type (colpali) to a standard config class
that vLLM understands.
Supported model_types:
- colpali (vidore/colpali-v1.3-hf)
"""
from transformers import PaliGemmaConfig
class ColPaliConfig(PaliGemmaConfig):
"""Configuration class for ColPali models.
Extends PaliGemmaConfig with additional fields used by ColPali variants
for the embedding projection layer.
"""
model_type = "colpali"
def __init__(
self,
embedding_dim: int | None = None,
embed_dim: int | None = None,
dim: int | None = None,
colbert_dim: int | None = None,
pooling: str | None = None,
vlm_config: dict | None = None,
**kwargs,
):
# Store embedding projection config fields
self.embedding_dim = embedding_dim
self.embed_dim = embed_dim
self.dim = dim
self.colbert_dim = colbert_dim
self.pooling = pooling
# The HF checkpoint nests PaliGemma config inside "vlm_config".
# Flatten it so PaliGemmaConfig receives vision_config, text_config,
# image_token_index, etc. directly.
# Use setdefault to avoid overwriting keys already set (e.g.
# model_type="colpali" would be clobbered by "paligemma" from
# vlm_config).
if vlm_config is not None:
vlm_dict = (
vlm_config if isinstance(vlm_config, dict) else vlm_config.to_dict()
)
_conflicting = {"model_type", "_name_or_path"}
for key, value in vlm_dict.items():
if key not in _conflicting:
kwargs.setdefault(key, value)
super().__init__(**kwargs)
@@ -1,147 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from transformers import NemotronHConfig
from transformers.configuration_utils import PreTrainedConfig
from transformers.models.siglip2.configuration_siglip2 import Siglip2VisionConfig
def _normalize_mrope_parameters(config: NemotronHConfig) -> None:
"""Normalize the checkpoint's mRoPE configuration for vLLM."""
rope_parameters = dict(getattr(config, "rope_parameters", None) or {})
mrope_section = rope_parameters.get("mrope_section")
if mrope_section is None:
return
rope_parameters["mrope_section"] = list(mrope_section)
rope_parameters.setdefault("mrope_interleaved", True)
config.rope_parameters = rope_parameters
class Cosmos3EdgeTextConfig(NemotronHConfig):
"""Dense Cosmos3 Edge text config backed by Nemotron-H layers."""
model_type = "cosmos3_edge_text"
has_no_defaults_at_init = True
ignore_keys_at_rope_validation = {"mrope_section", "mrope_interleaved"}
def __init__(
self,
num_hidden_layers: int = 28,
hidden_act: str = "relu2",
rms_norm_eps: float = 1e-5,
layers_block_type: list[str] | None = None,
**kwargs,
) -> None:
# Each decoder block is an attention layer followed by an MLP layer
if layers_block_type is None:
layers_block_type = ["full_attention", "mlp"] * num_hidden_layers
super().__init__(
layers_block_type=layers_block_type,
mlp_hidden_act=hidden_act,
layer_norm_epsilon=rms_norm_eps,
**kwargs,
)
def to_dict(self) -> dict:
config_dict = super().to_dict()
config_dict["num_hidden_layers"] = self.num_hidden_layers // 2
config_dict["hidden_act"] = config_dict.pop("mlp_hidden_act")
config_dict["rms_norm_eps"] = config_dict.pop("layer_norm_epsilon")
return config_dict
class Cosmos3EdgeProjectorConfig(PreTrainedConfig):
model_type = "cosmos3_edge_projector"
def __init__(
self,
input_hidden_size: int = 1024,
use_postshuffle_norm: bool = False,
spatial_merge_size: int = 2,
merger_intermediate_size: int = 11520,
out_hidden_size: int = 1024,
**kwargs,
) -> None:
super().__init__(**kwargs)
self.input_hidden_size = input_hidden_size
self.use_postshuffle_norm = use_postshuffle_norm
self.spatial_merge_size = spatial_merge_size
self.merger_intermediate_size = merger_intermediate_size
self.out_hidden_size = out_hidden_size
class Cosmos3EdgeVisionConfig(Siglip2VisionConfig):
model_type = "cosmos3_edge_vision"
class Cosmos3EdgeConfig(PreTrainedConfig):
model_type = "cosmos3_edge"
keys_to_ignore_at_inference = ["past_key_values"]
has_no_defaults_at_init = True
sub_configs = {
"vision_config": Cosmos3EdgeVisionConfig,
"projector_config": Cosmos3EdgeProjectorConfig,
"text_config": Cosmos3EdgeTextConfig,
}
def __init__(
self,
text_config: Cosmos3EdgeTextConfig | dict | None = None,
vision_config: Cosmos3EdgeVisionConfig | dict | None = None,
projector_config: Cosmos3EdgeProjectorConfig | dict | None = None,
image_token_id: int = 19,
video_token_id: int = 18,
vision_start_token_id: int = 20,
vision_end_token_id: int = 21,
tie_word_embeddings: bool = False,
**kwargs,
) -> None:
if text_config is None:
text_config = {}
if vision_config is None:
vision_config = {}
if projector_config is None:
projector_config = {}
self.text_config = (
Cosmos3EdgeTextConfig(**text_config)
if isinstance(text_config, dict)
else text_config
)
_normalize_mrope_parameters(self.text_config)
self.vision_config = (
Cosmos3EdgeVisionConfig(**vision_config)
if isinstance(vision_config, dict)
else vision_config
)
self.projector_config = (
Cosmos3EdgeProjectorConfig(**projector_config)
if isinstance(projector_config, dict)
else projector_config
)
# Qwen3-VL processing and M-RoPE read these attributes from the
# vision config, while this checkpoint stores them in projector config.
self.vision_config.spatial_merge_size = self.projector_config.spatial_merge_size
self.vision_config.temporal_patch_size = 1
self.vision_config.out_hidden_size = self.projector_config.out_hidden_size
self.image_token_id = image_token_id
self.video_token_id = video_token_id
self.vision_start_token_id = vision_start_token_id
self.vision_end_token_id = vision_end_token_id
super().__init__(
tie_word_embeddings=tie_word_embeddings,
**kwargs,
)
__all__ = [
"Cosmos3EdgeConfig",
"Cosmos3EdgeProjectorConfig",
"Cosmos3EdgeTextConfig",
"Cosmos3EdgeVisionConfig",
]
@@ -1,62 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from typing import Any
from transformers import PreTrainedConfig
class DeepseekV41Config(PreTrainedConfig):
"""DeepSeek V4.1 config.
The HF config nests the text model under ``text_config`` and the vision
tower under ``vision_config``. The vLLM model code (ported from
``deepseek_v4``) reads flat attributes, so both sub-configs are flattened
onto the top level here: text fields are exposed as-is, vision fields
with the ``vision_*`` naming used by ``deepseek_v41.common.vision``.
"""
model_type = "deepseek_v41"
def __init__(
self,
text_config: dict[str, Any] | None = None,
vision_config: dict[str, Any] | None = None,
**kwargs,
):
text_config = dict(text_config or {})
vision_config = dict(vision_config or {})
# ``rope_scaling`` is a property in Transformers v5 (backed by
# ``rope_parameters``); capture it here and restore after
# super().__init__, which re-standardizes rope params.
rope_scaling = text_config.pop("rope_scaling", None)
for key, value in text_config.items():
if key == "model_type":
continue
# Don't clobber PreTrainedConfig properties (e.g. is_encoder_decoder).
if isinstance(getattr(type(self), key, None), property):
continue
setattr(self, key, value)
super().__init__(**kwargs)
if rope_scaling is not None:
self.rope_parameters = rope_scaling
# The v4.1 quantization_config carries the expert dtype; the model
# code reads it from the top-level hf_config.
quant_cfg = getattr(self, "quantization_config", None) or {}
if not hasattr(self, "expert_dtype") and "expert_dtype" in quant_cfg:
self.expert_dtype = quant_cfg["expert_dtype"]
vision_n_layers = vision_config.get("num_hidden_layers", 0)
self.vision_n_layers = vision_n_layers
self.vision_dim = vision_config.get("hidden_size", 1024)
self.vision_n_heads = vision_config.get("num_attention_heads", 16)
self.vision_inter_dim = vision_config.get("intermediate_size", 2816)
self.vision_patch_size = vision_config.get("patch_size", 14)
self.vision_rope_theta = vision_config.get("rope_theta", 10000.0)
self.vision_downsample_ratio = vision_config.get("downsample_ratio", 3)
self.vision_max_n_token = vision_config.get("max_image_tokens", 1024)
self.vision_min_pixels = vision_config.get("min_pixels", 295936)
self.vision_max_wh_ratio = vision_config.get("max_wh_ratio")
@@ -1,126 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
# adapted from https://github.com/deepseek-ai/DeepSeek-VL2/blob/faf18023f24b962b32d9f0a2d89e402a8d383a78/deepseek_vl2/models/modeling_deepseek_vl_v2.py#L115-L268
from huggingface_hub.dataclasses import strict
from transformers import DeepseekV2Config, PreTrainedConfig
class VisionEncoderConfig(PreTrainedConfig):
model_type: str = "vision"
model_name: str = "vit_so400m_patch14_siglip_384.webli"
image_size: int = 384
patch_size: int = 16
width: int = 1024
layers: int = 24
heads: int = 16
mlp_ratio: int = 4
global_pool: str = "map"
ignore_head: bool = True
class_token: bool = False
num_classes: int = 0
use_checkpoint: bool = False
weight_init: str = "skip"
deterministic: bool = False
num_recomputing_layers: int = 0
def __init__(
self,
model_name: str = "vit_so400m_patch14_siglip_384.webli",
image_size: int = 384,
patch_size: int = 16,
width: int = 1024,
layers: int = 24,
heads: int = 16,
mlp_ratio: int = 4,
global_pool: str = "map",
ignore_head: bool = True,
class_token: bool = False,
num_classes: int = 0,
use_checkpoint: bool = False,
**kwargs,
):
self.model_name = model_name
self.image_size = image_size
self.patch_size = patch_size
self.width = width
self.layers = layers
self.heads = heads
self.mlp_ratio = mlp_ratio
self.global_pool = global_pool
self.ignore_head = ignore_head
self.class_token = class_token
self.num_classes = num_classes
self.use_checkpoint = use_checkpoint
super().__init__(**kwargs)
class MlpProjectorConfig(PreTrainedConfig):
model_type = "mlp_projector"
projector_type: str = "downsample_mlp_gelu"
input_dim: int = 1152
n_embed: int = 2048
depth: int = 2
mlp_ratio: int = 1
downsample_ratio: int = 2
token_pooling: bool = False
def __init__(
self,
projector_type: str = "downsample_mlp_gelu",
input_dim: int = 1152,
n_embed: int = 2048,
depth: int = 2,
mlp_ratio: int = 1,
downsample_ratio: int = 2,
**kwargs,
):
self.projector_type = projector_type
self.input_dim = input_dim
self.n_embed = n_embed
self.depth = depth
self.mlp_ratio = mlp_ratio
self.downsample_ratio = downsample_ratio
super().__init__(**kwargs)
@strict
class DeepseekVLV2TextConfig(DeepseekV2Config):
kv_lora_rank: int | None = DeepseekV2Config.kv_lora_rank
class DeepseekVLV2Config(PreTrainedConfig):
model_type = "deepseek_vl_v2"
tile_tag: str = "2D"
global_view_pos: str = "head"
candidate_resolutions: tuple[tuple[int, int]] = ((384, 384),)
def __init__(
self,
tile_tag: str = "tile_tag",
global_view_pos: str = "head",
candidate_resolutions: tuple[tuple[int, int]] = ((384, 384),),
**kwargs,
):
architectures = kwargs.setdefault("architectures", ["DeepseekVLV2ForCausalLM"])
self.vision_config = VisionEncoderConfig(**kwargs.pop("vision_config", {}))
self.projector_config = MlpProjectorConfig(**kwargs.pop("projector_config", {}))
self.text_config = DeepseekVLV2TextConfig(**kwargs.pop("language_config", {}))
self.tile_tag = tile_tag
self.global_view_pos = global_view_pos
self.candidate_resolutions = candidate_resolutions
self.vocab_size = self.text_config.vocab_size
# update model_type for OCR models
if "DeepseekOCRForCausalLM" in architectures:
kwargs["model_type"] = "deepseek_ocr"
elif "DeepseekOCR2ForCausalLM" in architectures:
kwargs["model_type"] = "deepseek_ocr2"
super().__init__(**kwargs)
@@ -1,25 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from transformers import DeepseekV3Config
class Dots3NoteConfig(DeepseekV3Config):
"""Configuration for the Dots3Note hybrid DSA/SWA model."""
model_type = "dots3_note"
def __init__(self, **kwargs):
# These fields are absent from existing Note config.json files. Do
# not inherit DeepSeek-V3's 8-group/4-group router defaults: Note was
# trained with an ungrouped (1/1) noaux_tc router, as defined by the
# reference Dots3Config in SGLang haimain. A different grouping
# changes the selected experts at every MoE layer.
kwargs.setdefault("n_group", 1)
kwargs.setdefault("topk_group", 1)
# Dots3 projects the indexer RoPE coordinates in adjacent (GPT-J)
# pairs. DeepSeek-V3.2 defaults to the split-half NeoX layout when
# this flag is absent, which rotates different learned coordinates.
kwargs.setdefault("indexer_rope_interleave", True)
kwargs.setdefault("num_nextn_predict_layers", 1)
super().__init__(**kwargs)
-89
View File
@@ -1,89 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
# Adapted from
# https://huggingface.co/tiiuae/falcon-7b/blob/main/configuration_RW.py
# Copyright 2023 The vLLM team.
# Copyright 2022 the Big Science Workshop and HuggingFace Inc. team.
# All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Falcon configuration."""
from transformers.configuration_utils import PreTrainedConfig
class RWConfig(PreTrainedConfig):
model_type = "falcon"
keys_to_ignore_at_inference = ["past_key_values"]
attribute_map = {
"num_hidden_layers": "n_layer",
"num_attention_heads": "n_head",
"num_kv_heads": "n_head_kv",
}
def __init__(
self,
vocab_size=250880,
hidden_size=64,
n_layer=2,
n_head=8,
layer_norm_epsilon=1e-5,
initializer_range=0.02,
use_cache=True,
bos_token_id=1,
eos_token_id=2,
hidden_dropout=0.0,
attention_dropout=0.0,
multi_query=True,
n_head_kv=None,
alibi=False,
bias=False,
parallel_attn=False,
new_decoder_architecture=False,
**kwargs,
) -> None:
self.vocab_size = vocab_size
# Backward compatibility with n_embed kwarg
n_embed = kwargs.pop("n_embed", None)
self.hidden_size = hidden_size if n_embed is None else n_embed
self.n_layer = n_layer
self.n_head = n_head
self.layer_norm_epsilon = layer_norm_epsilon
self.initializer_range = initializer_range
self.use_cache = use_cache
self.hidden_dropout = hidden_dropout
self.attention_dropout = attention_dropout
self.bos_token_id = bos_token_id
self.eos_token_id = eos_token_id
self.multi_query = multi_query
self.n_head_kv = 1 if n_head_kv is None else n_head_kv
self.alibi = alibi
self.bias = bias
self.parallel_attn = parallel_attn
self.new_decoder_architecture = new_decoder_architecture
if self.hidden_size == 8192:
# Hack for falcon-40b
self.new_decoder_architecture = True
super().__init__(bos_token_id=bos_token_id, eos_token_id=eos_token_id, **kwargs)
@property
def head_dim(self):
return self.hidden_size // self.n_head
@property
def rotary(self):
return not self.alibi
@@ -1,120 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from __future__ import annotations
from transformers import CONFIG_MAPPING, PreTrainedConfig
# NOTE: Temporary shim for FunAudioChat checkpoints.
# These checkpoints use `model_type="funaudiochat"`, which is not currently
# recognized by released Transformers, and the public checkpoint does not
# provide an `auto_map` to enable `trust_remote_code=True`.
# Remove this file once Transformers adds native support (or the checkpoint
# provides an `auto_map`) and vLLM can rely on `AutoConfig.from_pretrained()`.
class FunAudioChatAudioEncoderConfig(PreTrainedConfig):
model_type = "funaudiochat_audio_encoder"
def __init__(
self,
_attn_implementation: str | None = None,
num_mel_bins: int = 128,
encoder_layers: int = 32,
encoder_attention_heads: int = 20,
encoder_ffn_dim: int = 5120,
d_model: int = 1280,
dropout: float = 0.0,
attention_dropout: float = 0.0,
activation_function: str = "gelu",
activation_dropout: float = 0.0,
scale_embedding: bool = False,
initializer_range: float = 0.02,
max_source_positions: int = 1500,
n_window: int = 100,
output_dim: int = 3584,
bos_token_id: int | None = None,
codebook_size: int | None = None,
continuous_features_mode: str = "replace",
crq_transformer_config: dict | None = None,
eos_token_id: int | None = None,
group_size: int = 5,
enable_audio_invert_tower: bool = True,
pad_token_id: int | None = None,
**kwargs,
) -> None:
attn_impl = kwargs.pop("_attn_implementation", None) or _attn_implementation
super().__init__(**kwargs)
# Match HF default for attention implementation selection.
self._attn_implementation = attn_impl or "sdpa"
self.num_mel_bins = num_mel_bins
self.d_model = d_model
self.encoder_layers = encoder_layers
self.encoder_attention_heads = encoder_attention_heads
self.encoder_ffn_dim = encoder_ffn_dim
self.dropout = dropout
self.attention_dropout = attention_dropout
self.activation_function = activation_function
self.activation_dropout = activation_dropout
self.num_hidden_layers = encoder_layers
self.initializer_range = initializer_range
self.scale_embedding = scale_embedding
self.max_source_positions = max_source_positions
self.n_window = n_window
self.output_dim = output_dim
self.bos_token_id = bos_token_id
self.codebook_size = codebook_size
self.continuous_features_mode = continuous_features_mode
self.crq_transformer_config = crq_transformer_config
self.eos_token_id = eos_token_id
self.group_size = group_size
self.enable_audio_invert_tower = enable_audio_invert_tower
self.pad_token_id = pad_token_id
class FunAudioChatConfig(PreTrainedConfig):
model_type = "funaudiochat"
attribute_map = {
"audio_token_id": "audio_token_index",
}
def __init__(
self,
audio_config: PreTrainedConfig | dict | None = None,
text_config: PreTrainedConfig | dict | None = None,
audio_token_index: int = 151646,
ignore_index: int = -100,
hidden_size: int | None = None,
**kwargs,
) -> None:
self.audio_token_index = audio_token_index
self.ignore_index = ignore_index
if audio_config is None:
self.audio_config = FunAudioChatAudioEncoderConfig()
elif isinstance(audio_config, dict):
default_model_type = FunAudioChatAudioEncoderConfig.model_type
audio_config.setdefault("model_type", default_model_type)
self.audio_config = FunAudioChatAudioEncoderConfig(**audio_config)
else:
self.audio_config = audio_config
if text_config is None:
self.text_config = CONFIG_MAPPING["qwen2"]()
elif isinstance(text_config, dict):
# Default to qwen2 for backwards compatibility; FunAudioChat uses
# qwen3 in practice for recent checkpoints.
text_config.setdefault("model_type", "qwen2")
self.text_config = CONFIG_MAPPING[text_config["model_type"]](**text_config)
else:
self.text_config = text_config
self.hidden_size = (
int(self.text_config.hidden_size)
if hidden_size is None
else int(hidden_size)
)
super().__init__(**kwargs)
-173
View File
@@ -1,173 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from huggingface_hub.dataclasses import strict
from transformers.configuration_utils import PreTrainedConfig
from transformers.modeling_rope_utils import RopeParameters
@strict
class HYV4Config(PreTrainedConfig):
r"""Configuration class for the HY V4 (`HYV4ForCausalLM`) model.
Kept field-for-field in sync with
`transformers.models.hy_v4.configuration_hy_v4.HYV4Config` so that a
checkpoint resolves to the same values through either class. Registering it
in vLLM's `_CONFIG_REGISTRY` lets the architecture load without
`trust_remote_code` and without an `auto_map` module next to the weights.
HYV4 is a mixture-of-experts causal language model using Multi-head Latent
Attention (MLA), DeepSeek-style sparse attention (DSA), gated MLA,
learnable attention sinks, and independent Hyper-Connections (iHC).
The `layer_types`, `num_nextn_predict_layers` and `mtp_loss_factor` fields
below are vLLM-only additions: upstream leaves them to `**kwargs`, but
vLLM reads them directly (sparse-attention selection and MTP speculative
decoding), so they are declared to give checkpoints that omit them a
working default instead of a `None`.
Args:
mlp_layer_types: Per-layer MLP kind, `"dense"` or `"sparse"`. Defaults
to one dense layer followed by sparse MoE layers.
layer_types: Per-layer attention kind, e.g. `"full_attention"` or
`"deepseek_sparse_attention"`.
indexer_types: Per-layer DSA indexer kind, `"full"` or `"shared"`. A
shared layer reuses the most recent full indexer in the same
forward request.
index_topk: Maximum number of key positions selected by each DSA query.
index_head_dim: Hidden dimension of each DSA indexer head.
index_n_heads: Number of DSA indexer heads.
enable_lm_head_fp32: Whether the language-model head emits float32
logits. Surfaced as ``head_dtype`` so `LogitsProcessor` accumulates
the projection into fp32 while the weight stays in the model dtype.
enable_ihc: Whether independent Hyper-Connections are enabled.
hc_mult: Number of hidden-state channels maintained by iHC.
hc_magnitude: Scale applied to the iHC post-gating branch.
hc_eps: Numerical epsilon added to iHC sigmoid gates.
gated_mla: Whether to gate the MLA output.
gating_type: MLA gate granularity, `"elementwise"` or `"headwise"`.
learnable_sink: Whether to add a learned per-head attention sink.
learnable_sink_init: Initial value of each learned attention-sink
logit.
swiglu_limit: Magnitude of the routed-expert SwiGLU clamp. Values at or
below zero disable the clamp.
num_nextn_predict_layers: Number of MTP layers in the checkpoint.
mtp_loss_factor: Training-time MTP loss weight (unused at inference).
"""
model_type = "hy_v4"
keys_to_ignore_at_inference = ["past_key_values"]
attribute_map = {
"num_local_experts": "n_routed_experts",
}
base_model_tp_plan = {
"layers.*.self_attn.q_b_proj": "colwise",
"layers.*.self_attn.kv_a_proj_with_mqa": "mla_kv_a_proj",
"layers.*.self_attn.kv_b_proj": "colwise",
"layers.*.self_attn.o_proj": "rowwise",
"layers.*.self_attn.linear_gate": "colwise",
"layers.*.mlp.experts.gate_up_proj": "packed_colwise",
"layers.*.mlp.experts.down_proj": "rowwise",
"layers.*.mlp.experts": "moe_tp_experts",
"layers.*.mlp.shared_experts.gate_proj": "colwise",
"layers.*.mlp.shared_experts.up_proj": "colwise",
"layers.*.mlp.shared_experts.down_proj": "rowwise",
"layers.*.mlp.gate_proj": "colwise",
"layers.*.mlp.up_proj": "colwise",
"layers.*.mlp.down_proj": "rowwise",
}
base_model_pp_plan = {
"embed_tokens": (["input_ids"], ["inputs_embeds"]),
"layers": (["hidden_states", "attention_mask"], ["hidden_states"]),
"norm": (["hidden_states"], ["hidden_states"]),
}
vocab_size: int = 120832
hidden_size: int = 2816
intermediate_size: int = 6912
moe_intermediate_size: int = 768
num_hidden_layers: int = 34
num_attention_heads: int = 32
num_key_value_heads: int = 32
head_dim: int = 256
hidden_act: str = "silu"
max_position_embeddings: int = 262144
initializer_range: float = 0.006
rms_norm_eps: float = 1e-5
use_cache: bool = True
pad_token_id: int | None = 120002
bos_token_id: int | None = 120000
eos_token_id: int | list[int] | None = 120025
tie_word_embeddings: bool = False
attention_bias: bool = False
attention_dropout: float = 0.0
n_routed_experts: int = 256
n_shared_experts: int = 1
num_experts_per_tok: int = 8
routed_scaling_factor: float = 2.827
norm_topk_prob: bool = True
q_lora_rank: int = 1536
kv_lora_rank: int = 512
qk_nope_head_dim: int = 192
qk_rope_head_dim: int = 64
v_head_dim: int = 256
mlp_layer_types: list[str] | None = None
layer_types: list[str] | None = None
index_topk: int = 2048
index_head_dim: int = 128
index_n_heads: int = 16
indexer_types: list[str] | None = None
enable_lm_head_fp32: bool = True
enable_ihc: bool = True
hc_mult: int = 4
hc_magnitude: float = 2.0
hc_eps: float = 1e-6
gated_mla: bool = True
gating_type: str = "elementwise"
learnable_sink: bool = True
learnable_sink_init: float = 0.0
swiglu_limit: float = 10.0
rope_parameters: RopeParameters | dict | None = None
num_nextn_predict_layers: int = 1
mtp_loss_factor: float = 0.1
def __post_init__(self, **kwargs):
self.qk_head_dim = self.qk_nope_head_dim + self.qk_rope_head_dim
# RoPE applies only to the rope slice, so point `head_dim` at it: the
# inherited rotary embedding reads `config.head_dim` and then computes
# the right frequencies with no override.
self.head_dim = self.qk_rope_head_dim
if self.mlp_layer_types is None:
self.mlp_layer_types = ["dense"] * min(1, self.num_hidden_layers) + [
"sparse"
] * max(self.num_hidden_layers - 1, 0)
if self.indexer_types is None:
self.indexer_types = [
"full" if layer_idx == 0 or (layer_idx - 1) % 4 == 0 else "shared"
for layer_idx in range(self.num_hidden_layers)
]
# `PreTrainedConfig` validates `layer_types` against a fixed allow-list
# in which "deepseek_sparse_attention" is the only accepted spelling of
# a DSA layer. The model treats "sparse_attention" and "sparse" as
# equivalent, so fold them in before the base class validates.
if self.layer_types is not None:
self.layer_types = [
"deepseek_sparse_attention"
if lt in ("sparse_attention", "sparse")
else lt
for lt in self.layer_types
]
# `ModelConfig.head_dtype` reads this off the HF config, so an fp32 head
# needs no model-side dtype juggling: LogitsProcessor picks the
# `torch.mm(out_dtype=float32)` path and leaves the weight in the model
# dtype. Only fill it in when the checkpoint (or --hf-overrides) has not
# already pinned a value.
if self.enable_lm_head_fp32 and getattr(self, "head_dtype", None) is None:
self.head_dtype = "float32"
super().__post_init__(**kwargs)
__all__ = ["HYV4Config"]
-99
View File
@@ -1,99 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from __future__ import annotations
from transformers import Qwen3Config
from transformers.models.siglip2.configuration_siglip2 import Siglip2VisionConfig
class PixelShuffleSiglip2VisionConfig(Siglip2VisionConfig):
"""Vision configuration for Isaac with Pixel Shuffle support.
Extends Siglip2VisionConfig with additional fields for pixel shuffle.
"""
model_type = "pixel_shuffle_siglip2"
base_config_key = "vision_config"
def __init__(
self,
pixel_shuffle_scale_factor: int = 1,
num_patches: int = 256,
**kwargs,
):
super().__init__(**kwargs)
# Add our custom fields
self.pixel_shuffle_scale_factor = pixel_shuffle_scale_factor
self.num_patches = num_patches
class IsaacConfig(Qwen3Config):
"""Configuration class for Isaac multimodal model."""
model_type = "isaac"
sub_configs = {
"vision_config": PixelShuffleSiglip2VisionConfig,
"text_config": Qwen3Config,
}
def __init__(
self,
text_config=None,
vision_config=None,
vision_patch_size: int = 16,
vision_max_num_patches: int = 256,
vision_min_num_patches: int | None = None,
pixel_shuffle_scale: int = 1,
max_sequence_length: int = 16384,
vision_token: str = "<image>",
vision_attn_implementation: str | None = None,
**kwargs,
):
if isinstance(text_config, dict):
# from HF config
self.text_config = self.sub_configs["text_config"](**text_config)
elif text_config is None:
# For BC use all kwargs to init text config.
self.text_config = self.sub_configs["text_config"](**kwargs)
else:
# from Qwen3Config
self.text_config = text_config
# EventStreamProcessor parameters (for backward compatibility)
self.video_patch_size = vision_patch_size
self.vision_max_num_patches = vision_max_num_patches
self.vision_min_num_patches = vision_min_num_patches
self.pixel_shuffle_scale = pixel_shuffle_scale
# Processing parameters
self.max_sequence_length = max_sequence_length
self.vision_token = vision_token
# Handle vision config - PixelShuffleSiglip2VisionConfig instance
if isinstance(vision_config, dict):
self.vision_config = PixelShuffleSiglip2VisionConfig(**vision_config)
elif vision_config is None:
self.vision_config = PixelShuffleSiglip2VisionConfig()
else:
self.vision_config = vision_config
# Ensure compatibility with pretrained checkpoints
self.vision_config.pixel_shuffle_scale_factor = getattr(
self.vision_config,
"pixel_shuffle_scale_factor",
pixel_shuffle_scale,
)
self.vision_config.num_patches = getattr(
self.vision_config,
"num_patches",
vision_max_num_patches,
)
self.vision_attn_implementation = vision_attn_implementation
super().__init__(**kwargs)
__all__ = [
"IsaacConfig",
"PixelShuffleSiglip2VisionConfig",
]
@@ -1,77 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from transformers import DeepseekV3Config
class K3DSparkConfig(DeepseekV3Config):
"""Configuration for a dense MLA DSpark draft model."""
model_type = "k3_dspark"
has_no_defaults_at_init = True
def __init__(
self,
mla_use_nope: bool = False,
mla_use_output_gate: bool = False,
mla_use_qk_norm: bool = False,
rope_theta: float = 50000.0,
**kwargs,
) -> None:
# DeepseekV3Config defaults to a MoE topology. Zero these fields so
# generic vLLM config logic also recognizes this draft as dense.
kwargs.setdefault("n_routed_experts", 0)
kwargs.setdefault("n_shared_experts", 0)
kwargs.setdefault("num_experts_per_tok", 0)
rope_parameters = kwargs.get("rope_parameters")
if rope_parameters is None:
kwargs["rope_parameters"] = {
"rope_type": "default",
"rope_theta": rope_theta,
}
else:
rope_parameters = dict(rope_parameters)
rope_parameters.setdefault("rope_type", "default")
rope_parameters.setdefault("rope_theta", rope_theta)
kwargs["rope_parameters"] = rope_parameters
super().__init__(**kwargs)
self.mla_use_nope = mla_use_nope
self.mla_use_output_gate = mla_use_output_gate
self.mla_use_qk_norm = mla_use_qk_norm
unsupported = [
name
for name in (
"mla_use_nope",
"mla_use_output_gate",
"mla_use_qk_norm",
"dspark_bonus_anchor",
)
if getattr(self, name, False)
]
if self.q_lora_rank is None:
unsupported.append("q_lora_rank=None")
if unsupported:
raise ValueError(
"MLA DSpark does not support " + ", ".join(unsupported) + "."
)
self.draft_vocab_size = (
getattr(self, "draft_vocab_size", None) or self.vocab_size
)
if self.draft_vocab_size != self.vocab_size:
raise ValueError(
"MLA DSpark requires draft_vocab_size to equal vocab_size when "
"sharing the target embedding and LM head."
)
target_layer_ids = getattr(self, "target_layer_ids", None)
if not target_layer_ids or getattr(self, "num_target_layers", None) != len(
target_layer_ids
):
raise ValueError(
"MLA DSpark requires non-empty target_layer_ids and a matching "
"num_target_layers."
)
-139
View File
@@ -1,139 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Kimi-K3 multimodal configuration."""
from transformers.configuration_utils import PreTrainedConfig
from vllm.logger import init_logger
from vllm.transformers_utils.configs.kimi_linear import KimiLinearConfig
logger = init_logger(__name__)
class KimiK3VisionConfig(PreTrainedConfig):
model_type = "kimi_k3_vision"
def __init__(
self,
patch_size: int = 14,
init_pos_emb_height: int = 64,
init_pos_emb_width: int = 64,
init_pos_emb_time: int = 4,
pos_emb_type: str = "divided_fixed",
vt_num_attention_heads: int = 12,
vt_num_hidden_layers: int = 27,
vt_hidden_size: int = 1024,
vt_intermediate_size: int = 4096,
merge_kernel_size: tuple[int, int] = (2, 2),
video_attn_type: str = "spatial_temporal",
merge_type: str = "sd2_tpool",
_attn_implementation: str = "flash_attention_2",
mm_projector_type: str = "patchmergerv2",
mm_hidden_size: int | None = None,
projector_hidden_act: str = "gelu",
projector_ln_eps: float = 1e-5,
qkv_hidden_size: int = 1536,
norm_type: str = "rmsnorm",
attn_bias: bool = False,
patch_embed_proj_bias: bool = False,
mlp_type: str = "mlp2",
linear_bias: bool = False,
activation_func: str = "gelu_pytorch_tanh",
pos_emb_interpolation_mode: str = "bilinear",
text_hidden_size: int = 2304,
**kwargs,
):
super().__init__(**kwargs)
self.patch_size = patch_size
self.init_pos_emb_height = init_pos_emb_height
self.init_pos_emb_width = init_pos_emb_width
self.init_pos_emb_time = init_pos_emb_time
self.pos_emb_type = pos_emb_type
self.vt_num_attention_heads = vt_num_attention_heads
self.vt_num_hidden_layers = vt_num_hidden_layers
self.vt_hidden_size = vt_hidden_size
self.vt_intermediate_size = vt_intermediate_size
self.merge_kernel_size = tuple(merge_kernel_size)
self.video_attn_type = video_attn_type
self.merge_type = merge_type
self._attn_implementation = _attn_implementation
self.mm_projector_type = mm_projector_type
self.mm_hidden_size = (
mm_hidden_size if mm_hidden_size is not None else vt_hidden_size
)
self.projector_hidden_act = projector_hidden_act
self.projector_ln_eps = projector_ln_eps
self.text_hidden_size = text_hidden_size
self.qkv_hidden_size = qkv_hidden_size
self.norm_type = norm_type
self.attn_bias = attn_bias
self.patch_embed_proj_bias = patch_embed_proj_bias
self.mlp_type = mlp_type
self.linear_bias = linear_bias
self.activation_func = activation_func
self.pos_emb_interpolation_mode = pos_emb_interpolation_mode
# Aliases consumed by the Kimi-K2.5 vision implementation.
self.num_attention_heads = vt_num_attention_heads
self.num_hidden_layers = vt_num_hidden_layers
self.hidden_size = vt_hidden_size
self.intermediate_size = vt_intermediate_size
class KimiK3Config(PreTrainedConfig):
model_type = "kimi_k3"
def __init__(
self,
text_config: dict | KimiLinearConfig | None = None,
vision_config: dict | KimiK3VisionConfig | None = None,
ignore_index: int = -100,
media_placeholder_token_id: int = 163605,
pad_token_id: int = 0,
image_placeholder: str = "<|kimi_image_placeholder|>",
**kwargs,
):
if text_config is None:
self.text_config = KimiLinearConfig()
elif isinstance(text_config, dict):
self.text_config = KimiLinearConfig(**text_config)
else:
self.text_config = text_config
if vision_config is None:
self.vision_config = KimiK3VisionConfig()
elif isinstance(vision_config, dict):
self.vision_config = KimiK3VisionConfig(**vision_config)
else:
self.vision_config = vision_config
# K3's vision projector output must match the text model's hidden size.
# Override any value provided in vision_config for safety.
if self.vision_config.text_hidden_size != self.text_config.hidden_size:
logger.info(
"Overriding vision_config.text_hidden_size from %s to %s "
"to match text_config.hidden_size",
self.vision_config.text_hidden_size,
self.text_config.hidden_size,
)
self.vision_config.text_hidden_size = self.text_config.hidden_size
self.ignore_index = ignore_index
self.media_placeholder_token_id = media_placeholder_token_id
self.image_placeholder = image_placeholder
if getattr(self.text_config, "quantization_config", None) is not None:
self.quantization_config = self.text_config.quantization_config
super().__init__(pad_token_id=pad_token_id, **kwargs)
@property
def hidden_size(self) -> int:
return self.text_config.hidden_size
@property
def vocab_size(self) -> int:
return self.text_config.vocab_size
@@ -1,161 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from transformers.configuration_utils import PreTrainedConfig
class KimiLinearConfig(PreTrainedConfig):
model_type = "kimi_linear"
keys_to_ignore_at_inference = ["past_key_values"]
def __init__(
self,
model_type="kimi_linear",
vocab_size=163840,
hidden_size=4096,
head_dim=None,
intermediate_size=11008,
num_hidden_layers=32,
num_attention_heads=32,
num_key_value_heads=None,
hidden_act="silu",
initializer_range=0.02,
rms_norm_eps=1e-6,
use_cache=True,
pad_token_id=0,
bos_token_id=1,
eos_token_id=2,
rope_parameters=None,
tie_word_embeddings=False,
moe_intermediate_size: int | None = None,
moe_renormalize: bool = True,
moe_router_activation_func: str = "sigmoid",
num_experts: int | None = None,
num_experts_per_token: int | None = None,
num_shared_experts: int = 0,
routed_scaling_factor: float = 1.0,
first_k_dense_replace: int = 0,
moe_layer_freq: int = 1,
use_grouped_topk: bool = True,
num_expert_group: int = 1,
topk_group: int = 1,
q_lora_rank: int | None = None,
kv_lora_rank: int | None = None,
qk_nope_head_dim: int | None = None,
qk_rope_head_dim: int | None = None,
v_head_dim: int | None = None,
mla_use_nope: bool | None = False,
mla_use_output_gate: bool | None = False,
num_nextn_predict_layers: int = 0,
linear_attn_config: dict | None = None,
attn_res_block_size: int | None = None,
latent_moe_use_norm: bool = False,
activation_situ_beta: float | None = None,
activation_situ_linear_beta: float | None = None,
max_position_embeddings: int = 4096,
routed_expert_hidden_size: int | None = None,
topk_method: str = "noaux_tc",
**kwargs,
):
self.model_type = model_type
self.vocab_size = vocab_size
self.hidden_size = hidden_size
self.head_dim = (
head_dim if head_dim is not None else hidden_size // num_attention_heads
)
self.intermediate_size = intermediate_size
self.num_hidden_layers = num_hidden_layers
self.num_attention_heads = num_attention_heads
# for backward compatibility
if num_key_value_heads is None:
num_key_value_heads = num_attention_heads
self.num_key_value_heads = num_key_value_heads
self.hidden_act = hidden_act
self.initializer_range = initializer_range
self.rms_norm_eps = rms_norm_eps
self.use_cache = use_cache
# Try to set `rope_scaling` if available, otherwise use `rope_parameters`
rope_scaling = kwargs.pop("rope_scaling", None)
rope_parameters = rope_scaling or rope_parameters or {"rope_type": "default"}
rope_theta = kwargs.pop("rope_theta", 10000.0)
if "rope_theta" not in rope_parameters:
rope_parameters["rope_theta"] = rope_theta
self.rope_parameters = rope_parameters
self.q_lora_rank = q_lora_rank
self.kv_lora_rank = kv_lora_rank
self.qk_nope_head_dim = qk_nope_head_dim
self.qk_rope_head_dim = qk_rope_head_dim
self.v_head_dim = v_head_dim
self.mla_use_nope = mla_use_nope
self.mla_use_output_gate = mla_use_output_gate
# moe config
self.num_experts = num_experts
self.num_experts_per_token = num_experts_per_token
self.moe_renormalize = moe_renormalize
self.num_shared_experts = num_shared_experts
self.routed_scaling_factor = routed_scaling_factor
self.moe_router_activation_func = moe_router_activation_func
assert self.moe_router_activation_func in ("softmax", "sigmoid")
self.moe_intermediate_size = moe_intermediate_size
self.first_k_dense_replace = first_k_dense_replace
self.moe_layer_freq = moe_layer_freq
self.use_grouped_topk = use_grouped_topk
self.num_expert_group = num_expert_group
self.topk_group = topk_group
self.num_nextn_predict_layers = num_nextn_predict_layers
self.attn_res_block_size = attn_res_block_size
self.latent_moe_use_norm = latent_moe_use_norm
self.activation_situ_beta = activation_situ_beta
self.activation_situ_linear_beta = activation_situ_linear_beta
self.max_position_embeddings = max_position_embeddings
self.routed_expert_hidden_size = routed_expert_hidden_size
self.topk_method = topk_method
if linear_attn_config is not None:
assert linear_attn_config["kda_layers"] is not None
assert linear_attn_config["full_attn_layers"] is not None
self.linear_attn_config = linear_attn_config
super().__init__(
pad_token_id=pad_token_id,
bos_token_id=bos_token_id,
eos_token_id=eos_token_id,
tie_word_embeddings=tie_word_embeddings,
**kwargs,
)
@property
def is_mla(self):
return (
self.q_lora_rank is not None
or self.kv_lora_rank is not None
or self.qk_nope_head_dim is not None
or self.qk_rope_head_dim is not None
or self.v_head_dim is not None
or self.mla_use_nope is True
)
@property
def is_moe(self):
return self.num_experts is not None
@property
def is_linear_attn(self) -> bool:
return not (
self.linear_attn_config is None
or (
isinstance(self.linear_attn_config, dict)
and self.linear_attn_config["kda_layers"] is not None
and len(self.linear_attn_config["kda_layers"]) == 0
)
)
def is_kda_layer(self, layer_idx: int):
return (
self.linear_attn_config is not None
and (layer_idx + 1) in self.linear_attn_config["kda_layers"]
)
@@ -1,38 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
# Adapted from https://huggingface.co/moonshotai/Kimi-VL-A3B-Instruct/blob/main/configuration_kimi_vl.py
from transformers import DeepseekV2Config
from transformers.configuration_utils import PreTrainedConfig
from vllm.transformers_utils.configs.moonvit import MoonViTConfig
class KimiVLConfig(PreTrainedConfig):
model_type = "kimi_vl"
def __init__(
self,
vision_config: dict | MoonViTConfig | None = None,
text_config: dict | DeepseekV2Config | None = None,
ignore_index: int = -100,
media_placeholder_token_id: int = 163605,
pad_token_id: int = 0,
**kwargs,
):
if vision_config is None:
vision_config = MoonViTConfig()
elif isinstance(vision_config, dict):
vision_config = MoonViTConfig(**vision_config)
self.vision_config = vision_config
if text_config is None:
text_config = DeepseekV2Config()
elif isinstance(text_config, dict):
text_config = DeepseekV2Config(**text_config)
self.text_config: DeepseekV2Config = text_config
self.ignore_index = ignore_index
self.media_placeholder_token_id = media_placeholder_token_id
super().__init__(pad_token_id=pad_token_id, **kwargs)
@@ -1,103 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
# Copyright 2025 Horizon team, Xiaomi MiLM Plus.
# Copyright 2024 The Qwen team.
# Copyright 2023 The vLLM team.
# Copyright 2022 EleutherAI and the HuggingFace Inc. team. All rights reserved.
#
# This code is based on EleutherAI's GPT-NeoX library and the GPT-NeoX
# and OPT implementations in this library. It has been modified from its
# original forms to accommodate minor architectural differences compared
# to GPT-NeoX and OPT used by the Meta AI team that trained the model.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from transformers import PreTrainedConfig
from transformers.models.qwen2_5_omni.configuration_qwen2_5_omni import (
Qwen2_5OmniTextConfig,
)
class DashengConfig(PreTrainedConfig):
model_type = "midashenglm_dasheng_encoder"
def __init__(
self,
embed_dim: int = 768,
outputdim: int = 527,
patch_size: int | tuple[int, int] = 16,
patch_stride: int | tuple[int, int] = 16,
input_channels: int = 1,
target_length: int = 1012,
depth: int = 12,
num_heads: int = 12,
mlp_ratio: float = 4.0,
qkv_bias: bool = True,
init_values: float | None = None,
drop_rate: float = 0.0,
attn_drop_rate: float = 0.0,
f_min: float = 0.0,
f_max: float = 8000.0,
center: bool = True,
win_length: int = 512,
hop_length: int = 160,
sample_rate: int = 16000,
n_fft: int = 512,
n_mels: int = 64,
**kwargs,
):
self.embed_dim = embed_dim
self.outputdim = outputdim
self.patch_size = patch_size
self.patch_stride = patch_stride
self.input_channels = input_channels
self.target_length = target_length
self.depth = depth
self.num_heads = num_heads
self.mlp_ratio = mlp_ratio
self.qkv_bias = qkv_bias
self.init_values = init_values
self.drop_rate = drop_rate
self.attn_drop_rate = attn_drop_rate
self.f_min = f_min
self.f_max = f_max
self.center = center
self.win_length = win_length
self.hop_length = hop_length
self.sample_rate = sample_rate
self.n_fft = n_fft
self.n_mels = n_mels
super().__init__(**kwargs)
class MiDashengLMConfig(PreTrainedConfig):
model_type = "midashenglm"
def __init__(
self,
audio_encoder_config: dict | None = None,
subsample_factor: int = 5,
text_config: dict | None = None,
audio_token_id: int | None = None,
**kwargs,
):
self.audio_encoder_config = DashengConfig(**(audio_encoder_config or {}))
self.subsample_factor = subsample_factor
self.text_config = (
Qwen2_5OmniTextConfig(**text_config)
if text_config
else Qwen2_5OmniTextConfig()
)
self.text_config.rope_parameters = None # uses_mrope is false
self.audio_token_id = audio_token_id
super().__init__(**kwargs)
@@ -1,149 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from typing import Any
from transformers import PreTrainedConfig
class MiniMaxM3TextConfig(PreTrainedConfig):
"""Config for the MiniMax M3 text backbone (MiniMaxM3SparseForCausalLM).
Defaults mirror the ``text_config`` of the MiniMax-M3-preview checkpoint.
"""
model_type = "minimax_m3_text"
architectures = ["MiniMaxM3SparseForCausalLM"]
def __init__(
self,
vocab_size: int = 200064,
hidden_size: int = 6144,
intermediate_size: int = 3072,
dense_intermediate_size: int = 12288,
shared_intermediate_size: int = 3072,
num_hidden_layers: int = 60,
num_attention_heads: int = 64,
num_key_value_heads: int = 4,
head_dim: int = 128,
max_position_embeddings: int = 524288,
rms_norm_eps: float = 1e-6,
use_gemma_norm: bool = True,
attention_output_gate: bool = False,
rope_theta: float = 5000000,
rotary_dim: int = 64,
partial_rotary_factor: float = 0.5,
hidden_act: str = "swigluoai",
swiglu_alpha: float = 1.702,
# SwiGLU-OAI uses the (up + 1) bias, i.e. beta=1.0 (matches the
# reference: gate * sigmoid(gate * alpha) * (up + 1)). The checkpoint
# config omits swiglu_beta, so this default must stay 1.0.
swiglu_beta: float = 1.0,
swiglu_limit: float = 7.0,
use_qk_norm: bool = True,
qk_norm_type: str = "per_head",
num_local_experts: int = 128,
num_experts_per_tok: int = 4,
n_shared_experts: int = 1,
scoring_func: str = "sigmoid",
use_routing_bias: bool = True,
routed_scaling_factor: float = 2.0,
num_mtp_modules: int = 1,
moe_layer_freq: list[int] | None = None,
sparse_attention_config: dict[str, Any] | None = None,
tie_word_embeddings: bool = False,
**kwargs,
):
self.vocab_size = vocab_size
self.hidden_size = hidden_size
self.intermediate_size = intermediate_size
self.dense_intermediate_size = dense_intermediate_size
self.shared_intermediate_size = shared_intermediate_size
self.num_hidden_layers = num_hidden_layers
self.num_attention_heads = num_attention_heads
self.num_key_value_heads = num_key_value_heads
self.head_dim = head_dim
self.max_position_embeddings = max_position_embeddings
self.rms_norm_eps = rms_norm_eps
self.use_gemma_norm = use_gemma_norm
self.attention_output_gate = attention_output_gate
self.rope_theta = rope_theta
self.rotary_dim = rotary_dim
self.partial_rotary_factor = partial_rotary_factor
self.hidden_act = hidden_act
self.swiglu_alpha = swiglu_alpha
self.swiglu_beta = swiglu_beta
self.swiglu_limit = swiglu_limit
self.use_qk_norm = use_qk_norm
self.qk_norm_type = qk_norm_type
self.num_local_experts = num_local_experts
self.num_experts_per_tok = num_experts_per_tok
self.n_shared_experts = n_shared_experts
self.scoring_func = scoring_func
self.use_routing_bias = use_routing_bias
self.routed_scaling_factor = routed_scaling_factor
self.num_mtp_modules = num_mtp_modules
# First 3 layers are dense; the remaining 57 are sparse MoE.
self.moe_layer_freq = (
moe_layer_freq if moe_layer_freq is not None else [0] * 3 + [1] * 57
)
self.sparse_attention_config = (
sparse_attention_config
if sparse_attention_config is not None
else {
"use_sparse_attention": True,
"sparse_index_dim": 128,
"sparse_num_index_heads": 4,
"sparse_topk_blocks": 16,
"sparse_block_size": 128,
"sparse_disable_index_value": [0] * 3 + [1] * 57,
"sparse_score_type": "max",
"sparse_init_block": 0,
"sparse_local_block": 1,
"sparse_attention_freq": [0] * 3 + [1] * 57,
}
)
super().__init__(tie_word_embeddings=tie_word_embeddings, **kwargs)
class MiniMaxM3MTPConfig(MiniMaxM3TextConfig):
"""Config for a standalone MiniMax M3 MTP (multi-token prediction) head.
The MTP transformer layer is structurally a single MiniMax M3 decoder
layer, so this reuses the text backbone schema. Standalone MTP checkpoints
use ``model_type='minimax_m3_mtp'`` and a single hidden layer.
"""
model_type = "minimax_m3_mtp"
architectures = ["MiniMaxM3MTP"]
def __init__(self, num_hidden_layers: int = 1, **kwargs):
super().__init__(num_hidden_layers=num_hidden_layers, **kwargs)
class MiniMaxM3Config(PreTrainedConfig):
"""Top-level MiniMax M3 (VL) config.
Holds the text backbone as ``text_config`` so that
``config.get_text_config()`` extracts the MiniMaxM3SparseForCausalLM
backbone. Vision components are kept as a raw dict passthrough and are
not modeled here.
"""
model_type = "minimax_m3_vl"
def __init__(
self,
text_config: dict | MiniMaxM3TextConfig | None = None,
vision_config: dict | None = None,
**kwargs,
):
if text_config is None:
text_config = MiniMaxM3TextConfig()
elif isinstance(text_config, dict):
text_config = MiniMaxM3TextConfig(**text_config)
self.text_config = text_config
self.vision_config = vision_config
self.hidden_size = text_config.hidden_size
super().__init__(**kwargs)
@@ -1,153 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Configuration for Moondream3 model."""
from transformers import PreTrainedConfig
class Moondream3VisionConfig(PreTrainedConfig):
"""Vision encoder configuration for Moondream3."""
model_type = "moondream3_vision"
def __init__(
self,
enc_dim: int = 1152,
enc_patch_size: int = 14,
enc_n_layers: int = 27,
enc_ff_dim: int = 4304,
enc_n_heads: int = 16,
proj_inner_dim: int = 8192,
crop_size: int = 378,
max_crops: int = 12,
overlap_margin: int = 4,
**kwargs,
):
super().__init__(**kwargs)
self.enc_dim = enc_dim
self.enc_patch_size = enc_patch_size
self.enc_n_layers = enc_n_layers
self.enc_ff_dim = enc_ff_dim
self.enc_n_heads = enc_n_heads
self.proj_inner_dim = proj_inner_dim
self.crop_size = crop_size
self.max_crops = max_crops
self.overlap_margin = overlap_margin
# Standard HuggingFace attributes for vision config
self.hidden_size = enc_dim
self.num_attention_heads = enc_n_heads
self.num_hidden_layers = enc_n_layers
self.intermediate_size = enc_ff_dim
self.patch_size = enc_patch_size
self.image_size = crop_size
class Moondream3TextConfig(PreTrainedConfig):
"""Text decoder configuration for Moondream3."""
model_type = "moondream3_text"
def __init__(
self,
dim: int = 2048,
ff_dim: int = 8192,
n_layers: int = 24,
vocab_size: int = 51200,
max_context: int = 4096,
n_heads: int = 32,
n_kv_heads: int = 32,
prefix_attn: int = 730,
rope_theta: float = 1500000.0,
moe: dict | None = None,
**kwargs,
):
super().__init__(**kwargs)
# Store original moondream3 config names
self.dim = dim
self.ff_dim = ff_dim
self.n_layers = n_layers
self.n_heads = n_heads
self.n_kv_heads = n_kv_heads
self.prefix_attn = prefix_attn
self.max_context = max_context
self.rope_theta = rope_theta
# MoE config
moe = moe or {}
self.moe_start_layer = moe.get("start_layer", 4)
self.moe_num_experts = moe.get("n_experts", 64)
self.moe_experts_per_token = moe.get("n_experts_per_tok", 8)
self.moe_expert_inner_dim = moe.get("expert_inner_dim", 1024)
# Standard HuggingFace attributes (required by vLLM)
self.hidden_size = dim
self.num_attention_heads = n_heads
self.num_key_value_heads = n_kv_heads
self.num_hidden_layers = n_layers
self.intermediate_size = ff_dim
self.vocab_size = vocab_size
self.max_position_embeddings = max_context
# Moondream3 uses token 0 (<|endoftext|>) as both BOS and EOS.
# Token 3 (<|md_reserved_2|>) is an answer delimiter that the model
# implementation suppresses during generation.
self.bos_token_id = 0
self.eos_token_id = 0
# MoE standard attributes
self.num_local_experts = self.moe_num_experts
self.num_experts_per_tok = self.moe_experts_per_token
class Moondream3Config(PreTrainedConfig):
"""Combined configuration for Moondream3 multimodal model."""
model_type = "moondream3"
is_composition = True
def __init__(
self,
config: dict | None = None,
**kwargs,
):
config = config or {}
# Parse text config
text_config = config.get("text", {})
self.text_config: Moondream3TextConfig = Moondream3TextConfig(**text_config)
# Parse vision config
vision_config = config.get("vision", {})
self.vision_config = Moondream3VisionConfig(**vision_config)
# Store the original config dict for model access
self.config = config
tokenizer_config = config.get("tokenizer", {})
self.answer_token_id = tokenizer_config.get("answer_id", 3)
super().__init__(**kwargs)
# Expose key attributes at top level for vLLM compatibility
self.hidden_size = self.text_config.hidden_size
self.num_attention_heads = self.text_config.num_attention_heads
self.num_key_value_heads = self.text_config.num_key_value_heads
self.num_hidden_layers = self.text_config.num_hidden_layers
self.vocab_size = self.text_config.vocab_size
self.intermediate_size = self.text_config.intermediate_size
# Moondream3 uses token 0 (<|endoftext|>) as both BOS and EOS.
# Token 3 (<|md_reserved_2|>) is an answer delimiter that the model
# implementation suppresses during generation.
self.bos_token_id = 0
self.eos_token_id = 0
def get_text_config(self, decoder: bool = False) -> "Moondream3TextConfig":
"""Return the text config for vLLM's text_config detection.
Args:
decoder: Ignored. Only used for encoder-decoder models.
"""
return self.text_config
@@ -1,33 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
# Adapted from https://huggingface.co/moonshotai/Kimi-VL-A3B-Instruct/blob/main/configuration_kimi_vl.py
from transformers.configuration_utils import PreTrainedConfig
class MoonViTConfig(PreTrainedConfig):
model_type = "moonvit"
def __init__(
self,
patch_size: int = 14,
init_pos_emb_height: int = 64,
init_pos_emb_width: int = 64,
num_attention_heads: int = 16,
num_hidden_layers: int = 27,
hidden_size: int = 1152,
intermediate_size: int = 4304,
merge_kernel_size: tuple[int, int] = (2, 2),
**kwargs,
):
super().__init__(**kwargs)
self.patch_size = patch_size
# Positional embedding config
self.init_pos_emb_height = init_pos_emb_height
self.init_pos_emb_width = init_pos_emb_width
# Transformer config
self.num_hidden_layers = num_hidden_layers
self.num_attention_heads = num_attention_heads
self.hidden_size = hidden_size
self.intermediate_size = intermediate_size
# Patch merger config
self.merge_kernel_size = merge_kernel_size
@@ -1,108 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from typing import Any
from transformers import PreTrainedConfig, Qwen3Config
from transformers.models.whisper.configuration_whisper import WhisperConfig
class MossTranscribeDiarizeConfig(PreTrainedConfig):
"""Configuration for MOSS-Transcribe-Diarize."""
model_type = "moss_transcribe_diarize"
sub_configs = {"text_config": Qwen3Config, "audio_config": WhisperConfig}
keys_to_ignore_at_inference = ["past_key_values"]
def __init__(
self,
text_config: dict[str, Any] | Qwen3Config | None = None,
audio_config: dict[str, Any] | WhisperConfig | None = None,
audio_token_id: int = 151671,
audio_merge_size: int = 4,
adaptor_input_dim: int | None = None,
tie_word_embeddings: bool = True,
**kwargs: Any,
) -> None:
text_config_obj: Qwen3Config
if text_config is None:
text_config_obj = Qwen3Config(
vocab_size=151936,
hidden_size=1024,
intermediate_size=3072,
num_hidden_layers=28,
num_attention_heads=16,
num_key_value_heads=8,
head_dim=128,
max_position_embeddings=40960,
tie_word_embeddings=tie_word_embeddings,
rope_theta=1_000_000.0,
layer_types=["full_attention"] * 28,
)
elif isinstance(text_config, dict):
text_config_obj = Qwen3Config(**text_config)
else:
text_config_obj = text_config
audio_config_obj: WhisperConfig
if audio_config is None:
audio_config_obj = WhisperConfig(
num_mel_bins=80,
d_model=1024,
encoder_layers=24,
encoder_attention_heads=16,
encoder_ffn_dim=4096,
max_source_positions=1500,
dropout=0.0,
attention_dropout=0.0,
activation_dropout=0.0,
activation_function="gelu",
encoder_layerdrop=0.0,
scale_embedding=False,
)
elif isinstance(audio_config, dict):
audio_config_obj = WhisperConfig(**audio_config)
else:
audio_config_obj = audio_config
text_config_obj.tie_word_embeddings = tie_word_embeddings
if not getattr(text_config_obj, "layer_types", None):
text_config_obj.layer_types = [
"full_attention"
] * text_config_obj.num_hidden_layers
super().__init__(tie_word_embeddings=tie_word_embeddings, **kwargs)
self.text_config = text_config_obj
self.audio_config = audio_config_obj
self.audio_token_id = int(audio_token_id)
self.audio_merge_size = int(audio_merge_size)
self.adaptor_input_dim = (
int(adaptor_input_dim)
if adaptor_input_dim is not None
else int(audio_config_obj.d_model) * int(audio_merge_size)
)
self.vocab_size = int(text_config_obj.vocab_size)
self.hidden_size = int(text_config_obj.hidden_size)
self.intermediate_size = int(text_config_obj.intermediate_size)
self.num_hidden_layers = int(text_config_obj.num_hidden_layers)
self.num_attention_heads = int(text_config_obj.num_attention_heads)
self.num_key_value_heads = int(text_config_obj.num_key_value_heads)
self.head_dim = int(text_config_obj.head_dim)
self.hidden_act = text_config_obj.hidden_act
self.max_position_embeddings = int(text_config_obj.max_position_embeddings)
self.rms_norm_eps = float(text_config_obj.rms_norm_eps)
rope_parameters = getattr(text_config_obj, "rope_parameters", None)
rope_theta = float(getattr(text_config_obj, "rope_theta", 1_000_000.0))
if rope_parameters is None:
rope_parameters = {
"rope_type": "default",
"rope_theta": rope_theta,
}
text_config_obj.rope_parameters = rope_parameters
self.rope_parameters = rope_parameters
self.rope_theta = float(rope_parameters.get("rope_theta", rope_theta))
self.attention_bias = bool(text_config_obj.attention_bias)
self.attention_dropout = float(text_config_obj.attention_dropout)
self.is_causal = True
@@ -1,389 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""MuseGlimmer model configuration for vLLM.
Native vLLM copy of the MuseGlimmer HuggingFace configs
(``configuration_muse_glimmer.py``). MuseGlimmer's
``model_type`` is not yet registered in released transformers, so vLLM ships
this config so partners can serve MuseGlimmer checkpoints *without* trust_remote_code.
The field set and defaults are kept byte-for-byte in sync with the HF reference
so a checkpoint's ``config.json`` deserializes identically here. Both the text
and vision configs are consumed by the native multimodal serving path.
"""
from __future__ import annotations
from transformers import Qwen3Config
from transformers.configuration_utils import PreTrainedConfig
def _default_no_rope_layers(num_hidden_layers: int) -> list[int]:
# iRoPE mask: NoPE every 4 layers, counted backward from the last layer.
stride = 4
return [
0 if (num_hidden_layers - 1 - i) % stride == 0 else 1
for i in range(num_hidden_layers)
]
class MuseGlimmerTextConfig(PreTrainedConfig):
model_type = "muse_glimmer_text"
keys_to_ignore_at_inference = ["past_key_values"]
def __init__(
self,
vocab_size: int = 202_048,
hidden_size: int = 6656,
intermediate_size: int = 19968,
num_hidden_layers: int = 52,
num_attention_heads: int = 32,
num_key_value_heads: int = 2,
head_dim: int = 128,
hidden_activation: str = "silu",
max_position_embeddings: int = 16_384,
initializer_range: float = 0.02,
rms_norm_eps: float = 1e-5,
use_cache: bool = True,
pad_token_id: int | None = None,
eos_token_id: int | list[int] | None = 200_001,
bos_token_id: int | None = 200_000,
tie_word_embeddings: bool = False,
rope_parameters: dict | None = None,
rope_theta: float | None = None,
attention_bias: bool = False,
attention_dropout: float = 0.0,
query_pre_attn_scalar: int = 256,
sliding_window: int | None = 2048,
layer_types: list[str] | None = None,
final_logit_softcapping: float | None = 20.0,
attn_logit_softcapping: float | None = None,
use_bidirectional_attention: bool | None = None,
# MuseGlimmer-specific
qk_scale_factor: float = 43.7840518911,
use_qk_norm: bool = True,
use_attn_output_gate: bool = True,
output_multiplier: float = 0.19611613513818404,
normalize_tok_embeddings: bool = True,
post_norm_eps: float = 1e-8,
no_rope_layers: list[int] | None = None,
**kwargs,
) -> None:
self.vocab_size = vocab_size
self.hidden_size = hidden_size
self.intermediate_size = intermediate_size
self.num_hidden_layers = num_hidden_layers
self.num_attention_heads = num_attention_heads
self.num_key_value_heads = num_key_value_heads
self.head_dim = head_dim
self.hidden_activation = hidden_activation
self.max_position_embeddings = max_position_embeddings
self.initializer_range = initializer_range
self.rms_norm_eps = rms_norm_eps
self.use_cache = use_cache
self.attention_bias = attention_bias
self.attention_dropout = attention_dropout
self.query_pre_attn_scalar = query_pre_attn_scalar
self.sliding_window = sliding_window
self.final_logit_softcapping = final_logit_softcapping
self.attn_logit_softcapping = attn_logit_softcapping
self.use_bidirectional_attention = use_bidirectional_attention
# RoPE: accept either an explicit rope_parameters dict (HF 5.x) or a
# bare rope_theta; normalize to rope_parameters for vLLM's get_rope.
if rope_parameters is None:
theta = rope_theta if rope_theta is not None else 500_000.0
rope_parameters = {"rope_type": "default", "rope_theta": theta}
self.rope_parameters = rope_parameters
# vLLM reads rope_theta off the config in some codepaths.
self.rope_theta = rope_parameters.get("rope_theta", 500_000.0)
# MuseGlimmer-specific fields
self.qk_scale_factor = qk_scale_factor
self.use_qk_norm = use_qk_norm
self.use_attn_output_gate = use_attn_output_gate
self.output_multiplier = output_multiplier
self.normalize_tok_embeddings = normalize_tok_embeddings
self.post_norm_eps = post_norm_eps
self.no_rope_layers = (
no_rope_layers
if no_rope_layers is not None
else _default_no_rope_layers(num_hidden_layers)
)
if layer_types is None:
layer_types = [
"full_attention" if self.no_rope_layers[i] == 0 else "sliding_attention"
for i in range(num_hidden_layers)
]
self.layer_types = layer_types
super().__init__(
pad_token_id=pad_token_id,
bos_token_id=bos_token_id,
eos_token_id=eos_token_id,
tie_word_embeddings=tie_word_embeddings,
**kwargs,
)
class MuseGlimmerVisionConfig(PreTrainedConfig):
model_type = "muse_glimmer_vision"
def __init__(
self,
patch_size: int = 14,
pos_emb_height: int = 32,
pos_emb_width: int = 32,
num_attention_heads: int = 16,
num_hidden_layers: int = 50,
hidden_size: int = 1536,
intermediate_size: int = 8960,
hidden_act: str = "gelu",
merge_kernel_size: int = 2,
rope_parameters: dict | None = None,
max_position_embeddings: int = 32 * 32,
output_dim: int = 6144,
patch_temporal: int = 2,
adapter_dim: int = 4096,
layer_norm_eps: float = 1e-5,
layer_types: list[str] | None = None,
**kwargs,
) -> None:
super().__init__(**kwargs)
self.patch_size = patch_size
self.pos_emb_height = pos_emb_height
self.pos_emb_width = pos_emb_width
self.num_attention_heads = num_attention_heads
self.num_hidden_layers = num_hidden_layers
self.hidden_size = hidden_size
self.intermediate_size = intermediate_size
self.hidden_act = hidden_act
self.merge_kernel_size = merge_kernel_size
self.rope_parameters = rope_parameters
self.max_position_embeddings = max_position_embeddings
self.output_dim = output_dim
self.patch_temporal = patch_temporal
self.adapter_dim = adapter_dim
self.layer_norm_eps = layer_norm_eps
if layer_types is None:
stride = 4
layer_types = [
"full_attention"
if (i + 1) % stride == 0 or i == num_hidden_layers - 1
else "sliding_attention"
for i in range(num_hidden_layers)
]
self.layer_types = layer_types
class MuseGlimmerConfig(PreTrainedConfig):
model_type = "muse_glimmer"
sub_configs = {
"text_config": MuseGlimmerTextConfig,
"vision_config": MuseGlimmerVisionConfig,
}
# --- Flat (legacy-converter) -> canonical normalization -------------------
# MuseGlimmer checkpoints exist in two config layouts in the wild:
#
# * CANONICAL (current HF converter, transformers 5.15): nested
# ``text_config`` / ``vision_config`` sub-dicts with canonical field
# names (``hidden_activation``, ``final_logit_softcapping``,
# ``rope_parameters``, ``vision_config.hidden_size`` ...).
#
# * FLAT (older converter, e.g. Ruan's ``rl_v1/hf``, transformers 5.9):
# every field is a top-level key, with different names
# (``hidden_act``, ``output_soft_cap_temp``, ``rope_theta``,
# ``vision_latent_dim`` ...) and NO ``text_config`` nesting.
#
# Without normalization a flat config silently deserializes to an
# ALL-DEFAULT text config (every checkpoint value ignored) — a dangerous
# correctness bug: a checkpoint whose arch differs from the defaults would
# load into a wrong-shaped model with no error. So when we detect a flat
# config we hoist the flat fields into ``text_config`` / ``vision_config``
# and rename them to canonical names.
# flat text field name -> canonical MuseGlimmerTextConfig field name
_FLAT_TEXT_RENAMES = {
"hidden_act": "hidden_activation",
"output_soft_cap_temp": "final_logit_softcapping",
}
# canonical MuseGlimmerTextConfig constructor params that may appear flat
_FLAT_TEXT_KEYS = frozenset(
{
"vocab_size",
"hidden_size",
"intermediate_size",
"num_hidden_layers",
"num_attention_heads",
"num_key_value_heads",
"head_dim",
"hidden_activation",
"hidden_act",
"max_position_embeddings",
"initializer_range",
"rms_norm_eps",
"use_cache",
"tie_word_embeddings",
"rope_parameters",
"rope_theta",
"attention_bias",
"attention_dropout",
"query_pre_attn_scalar",
"sliding_window",
"layer_types",
"final_logit_softcapping",
"output_soft_cap_temp",
"attn_logit_softcapping",
"use_bidirectional_attention",
"qk_scale_factor",
"use_qk_norm",
"use_attn_output_gate",
"output_multiplier",
"normalize_tok_embeddings",
"post_norm_eps",
"no_rope_layers",
}
)
# flat vision field name -> canonical MuseGlimmerVisionConfig field name
_FLAT_VISION_RENAMES = {
"vision_latent_dim": "hidden_size",
"vision_heads": "num_attention_heads",
"vision_layers": "num_hidden_layers",
"vision_output_dim": "output_dim",
"vision_patch_size": "patch_size",
"vision_patch_temporal": "patch_temporal",
"vision_adapter_dim": "adapter_dim",
"vision_pos_emb_grid_h": "pos_emb_height",
"vision_pos_emb_grid_w": "pos_emb_width",
"vision_downsample_factor": "merge_kernel_size",
}
@classmethod
def _looks_flat(cls, kwargs: dict) -> bool:
# Flat if there is no explicit text_config but there ARE text-level
# fields at the top level (e.g. hidden_size / num_hidden_layers).
if kwargs.get("text_config") is not None:
return False
return any(k in kwargs for k in ("hidden_size", "num_hidden_layers"))
@classmethod
def _normalize_flat(cls, kwargs: dict) -> dict:
kwargs = dict(kwargs)
text: dict = {}
for key in list(kwargs.keys()):
if key in cls._FLAT_TEXT_KEYS:
canon = cls._FLAT_TEXT_RENAMES.get(key, key)
text.setdefault(canon, kwargs.pop(key))
vision_mlp_ratio = kwargs.pop("vision_mlp_ratio", None)
sparse_factor = kwargs.pop("vision_sparse_attention_factor", None)
vision: dict = {}
for key in list(kwargs.keys()):
if key in cls._FLAT_VISION_RENAMES:
vision.setdefault(cls._FLAT_VISION_RENAMES[key], kwargs.pop(key))
if vision_mlp_ratio is not None:
hidden_size = vision.get("hidden_size", 1536)
vision["intermediate_size"] = int(vision_mlp_ratio * hidden_size)
if sparse_factor is not None:
sparse_factor = int(sparse_factor)
if sparse_factor <= 0:
raise ValueError("vision_sparse_attention_factor must be positive")
num_layers = vision.get("num_hidden_layers", 50)
vision["layer_types"] = [
"full_attention"
if (layer_idx + 1) % sparse_factor == 0 or layer_idx == num_layers - 1
else "sliding_attention"
for layer_idx in range(num_layers)
]
if text:
kwargs["text_config"] = text
if vision:
kwargs["vision_config"] = vision
return kwargs
def __init__(
self,
text_config: dict | MuseGlimmerTextConfig | None = None,
vision_config: dict | MuseGlimmerVisionConfig | None = None,
image_token_id: int = 200092,
video_token_id: int = 200091,
**kwargs,
) -> None:
# Detect + fold a flat config into nested text/vision before building
# the sub-configs. (image/video token ids are handled below; a flat
# ``patch_token_id`` alias is mapped too.)
if text_config is None and self._looks_flat(kwargs):
kwargs = self._normalize_flat(kwargs)
text_config = kwargs.pop("text_config", None)
vision_config = kwargs.pop("vision_config", vision_config)
# flat image-token alias
if "patch_token_id" in kwargs and "image_token_id" not in kwargs:
image_token_id = kwargs.pop("patch_token_id")
if text_config is None:
self.text_config = MuseGlimmerTextConfig()
elif isinstance(text_config, dict):
self.text_config = MuseGlimmerTextConfig(**text_config)
else:
self.text_config = text_config
if vision_config is None:
self.vision_config = MuseGlimmerVisionConfig()
elif isinstance(vision_config, dict):
self.vision_config = MuseGlimmerVisionConfig(**vision_config)
else:
self.vision_config = vision_config
self.image_token_id = image_token_id
self.video_token_id = video_token_id
self.patch_token_id = image_token_id
super().__init__(**kwargs)
class MuseGlimmerAssistantConfig(Qwen3Config):
"""Config for the Muse Glimmer DFlash draft head.
The head is Qwen3-shaped and runs on vLLM's generic ``qwen3_dflash``
implementation, so this derives from ``Qwen3Config``. It cannot BE
``Qwen3Config``, because two of the checkpoint's values do not survive that
class:
* ``sliding_window`` is gated behind ``use_sliding_window``, which defaults
to False -- ``Qwen3Config(sliding_window=2048).sliding_window`` is None.
The checkpoint declares ``sliding_window: 2048`` and five
``sliding_attention`` layers, so the window silently disappears and the
DFlash path then raises "sliding attention requires a window size".
* ``vocab_size`` is absent from the checkpoint, so Qwen3's default of
151936 applies instead of Muse Glimmer's 202048. That one is *silent*: it
builds an all-zero ``draft_id_to_target_id`` remap, and also puts
pad/bos/eos/mask token ids out of range.
Both defaults are set here so an unmodified checkpoint loads correctly.
"""
model_type = "muse_glimmer_assistant"
def __init__(
self,
vocab_size: int = 202048,
use_sliding_window: bool = True,
dflash_config: dict | None = None,
**kwargs,
) -> None:
super().__init__(
vocab_size=vocab_size, use_sliding_window=use_sliding_window, **kwargs
)
# muse_glimmer_assistant uses non-causal attention.
self.dflash_config = {"causal": False} | (dflash_config or {})
__all__ = [
"MuseGlimmerTextConfig",
"MuseGlimmerVisionConfig",
"MuseGlimmerConfig",
"MuseGlimmerAssistantConfig",
]
@@ -1,50 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""OpenVLA configuration support.
OpenVLA checkpoints use a custom ``model_type`` and nest the language model
configuration under ``text_config``. This shim lets vLLM load the checkpoint
configuration without executing Hugging Face remote code.
"""
from typing import Any
from transformers import LlamaConfig, PreTrainedConfig
class OpenVLAConfig(PreTrainedConfig):
"""Configuration class for OpenVLA models."""
model_type = "openvla"
def __init__(
self,
timm_model_ids: list[str] | None = None,
timm_override_act_layers: list[str | None] | None = None,
image_sizes: list[int] | None = None,
use_fused_vision_backbone: bool = True,
image_token_index: int = 32000,
n_action_bins: int = 256,
text_config: dict[str, Any] | LlamaConfig | None = None,
**kwargs: Any,
) -> None:
kwargs.setdefault("architectures", ["OpenVLAForActionPrediction"])
super().__init__(**kwargs)
self.timm_model_ids = timm_model_ids or [
"vit_large_patch14_reg4_dinov2.lvd142m",
"vit_so400m_patch14_siglip_224",
]
self.timm_override_act_layers = timm_override_act_layers or [None, None]
self.image_sizes = image_sizes or [224, 224]
self.use_fused_vision_backbone = use_fused_vision_backbone
self.image_token_index = image_token_index
self.n_action_bins = n_action_bins
if text_config is None:
text_config = LlamaConfig(architectures=["LlamaForCausalLM"])
elif isinstance(text_config, dict):
text_config = text_config.copy()
text_config.setdefault("architectures", ["LlamaForCausalLM"])
text_config = LlamaConfig(**text_config)
self.text_config = text_config
-124
View File
@@ -1,124 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
# ruff: noqa: E501
# adapted from https://huggingface.co/AIDC-AI/Ovis2-1B/blob/main/configuration_ovis.py
from typing import Any
from transformers import Aimv2VisionConfig, AutoConfig, PreTrainedConfig
# ----------------------------------------------------------------------
# Visual Tokenizer Configuration
# ----------------------------------------------------------------------
class BaseVisualTokenizerConfig(PreTrainedConfig):
def __init__(
self,
vocab_size=16384,
tokenize_function="softmax",
tau=1.0,
depths=None,
drop_cls_token=False,
backbone_config: PreTrainedConfig | dict | None = None,
hidden_stride: int = 1,
**kwargs,
):
super().__init__(**kwargs)
self.vocab_size = vocab_size
self.tokenize_function = tokenize_function
self.tau = tau
if isinstance(depths, str):
depths = [int(x) for x in depths.split("|")]
self.depths = depths
self.backbone_kwargs = dict[str, Any]()
self.drop_cls_token = drop_cls_token
if backbone_config is not None:
assert isinstance(backbone_config, (PreTrainedConfig, dict)), (
f"expect `backbone_config` to be instance of PreTrainedConfig or dict, but got {type(backbone_config)} type"
)
if not isinstance(backbone_config, PreTrainedConfig):
model_type = backbone_config.pop("model_type")
if model_type == "aimv2":
backbone_config = Aimv2VisionConfig(**backbone_config)
else:
backbone_config = AutoConfig.for_model(
model_type, **backbone_config
)
self.backbone_config = backbone_config
self.hidden_stride = hidden_stride
class Aimv2VisualTokenizerConfig(BaseVisualTokenizerConfig):
model_type = "aimv2_visual_tokenizer"
def __init__(self, **kwargs):
super().__init__(**kwargs)
if self.drop_cls_token:
self.drop_cls_token = False
if self.depths:
assert len(self.depths) == 1
self.backbone_kwargs["num_hidden_layers"] = self.depths[0]
class SiglipVisualTokenizerConfig(BaseVisualTokenizerConfig):
model_type = "siglip_visual_tokenizer"
def __init__(self, **kwargs):
super().__init__(**kwargs)
if self.drop_cls_token:
self.drop_cls_token = False
if self.depths:
assert len(self.depths) == 1
self.backbone_kwargs["num_hidden_layers"] = self.depths[0]
AutoConfig.register("siglip_visual_tokenizer", SiglipVisualTokenizerConfig)
AutoConfig.register("aimv2_visual_tokenizer", Aimv2VisualTokenizerConfig)
# ----------------------------------------------------------------------
# Ovis Configuration
# ----------------------------------------------------------------------
class OvisConfig(PreTrainedConfig):
model_type = "ovis"
def __init__(
self,
llm_config: PreTrainedConfig | dict | None = None,
visual_tokenizer_config: PreTrainedConfig | dict | None = None,
multimodal_max_length=8192,
hidden_size=None,
conversation_formatter_class=None,
llm_attn_implementation=None,
disable_tie_weight=False,
**kwargs,
):
super().__init__(**kwargs)
if llm_config is not None:
assert isinstance(llm_config, (PreTrainedConfig, dict)), (
f"expect `llm_config` to be instance of PreTrainedConfig or dict, but got {type(llm_config)} type"
)
if not isinstance(llm_config, PreTrainedConfig):
model_type = llm_config["model_type"]
llm_config.pop("model_type")
llm_config = AutoConfig.for_model(model_type, **llm_config)
# map llm_config to text_config
self.text_config = llm_config
if visual_tokenizer_config is not None:
assert isinstance(visual_tokenizer_config, (PreTrainedConfig, dict)), (
f"expect `visual_tokenizer_config` to be instance of PreTrainedConfig or dict, but got {type(visual_tokenizer_config)} type"
)
if not isinstance(visual_tokenizer_config, PreTrainedConfig):
model_type = visual_tokenizer_config["model_type"]
visual_tokenizer_config.pop("model_type")
visual_tokenizer_config = AutoConfig.for_model(
model_type, **visual_tokenizer_config
)
self.visual_tokenizer_config = visual_tokenizer_config
self.multimodal_max_length = multimodal_max_length
self.hidden_size = hidden_size
self.conversation_formatter_class = conversation_formatter_class
self.llm_attn_implementation = llm_attn_implementation
self.disable_tie_weight = disable_tie_weight
@@ -1,105 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from typing import Any
from transformers import PreTrainedConfig
from transformers.models.auto import CONFIG_MAPPING
class QianfanOCRVisionConfig(PreTrainedConfig):
model_type = "qianfan_ocr_vision"
def __init__(
self,
hidden_size: int = 1024,
intermediate_size: int = 4096,
num_hidden_layers: int = 24,
num_attention_heads: int = 16,
num_channels: int = 3,
image_size: int = 448,
patch_size: int = 14,
hidden_act: str = "gelu",
layer_norm_eps: float = 1e-6,
attention_dropout: float = 0.0,
drop_path_rate: float = 0.1,
qkv_bias: bool = True,
qk_normalization: bool = False,
norm_type: str = "layer_norm",
initializer_range: float = 0.02,
initializer_factor: float = 0.1,
use_mask_token: bool = False,
use_mean_pooling: bool = True,
**kwargs: Any,
):
super().__init__(**kwargs)
self.hidden_size = hidden_size
self.intermediate_size = intermediate_size
self.num_hidden_layers = num_hidden_layers
self.num_attention_heads = num_attention_heads
self.num_channels = num_channels
self.image_size = image_size
self.patch_size = patch_size
self.hidden_act = hidden_act
self.layer_norm_eps = layer_norm_eps
self.attention_dropout = attention_dropout
self.drop_path_rate = drop_path_rate
self.qkv_bias = qkv_bias
self.qk_normalization = qk_normalization
self.norm_type = norm_type
self.initializer_range = initializer_range
self.initializer_factor = initializer_factor
self.use_mask_token = use_mask_token
self.use_mean_pooling = use_mean_pooling
class QianfanOCRConfig(PreTrainedConfig):
model_type = "qianfan_ocr"
def __init__(
self,
vision_config: dict | None = None,
text_config: dict | None = None,
downsample_ratio: float = 0.5,
dynamic_image_size: bool = True,
force_image_size: int = 448,
image_token_id: int = 151671,
max_dynamic_patch: int = 12,
min_dynamic_patch: int = 1,
pad2square: bool = False,
ps_version: str = "v2",
select_layer: int = -1,
template: str = "internvl2_5",
use_thumbnail: bool = True,
tie_word_embeddings: bool = False,
**kwargs: Any,
):
super().__init__(**kwargs)
if isinstance(vision_config, dict):
self.vision_config = QianfanOCRVisionConfig(**vision_config)
elif vision_config is None:
self.vision_config = QianfanOCRVisionConfig()
else:
self.vision_config = vision_config
if isinstance(text_config, dict):
model_type = text_config.get("model_type", "qwen3")
self.text_config = CONFIG_MAPPING[model_type](**text_config)
elif text_config is None:
self.text_config = CONFIG_MAPPING["qwen3"]()
else:
self.text_config = text_config
self.downsample_ratio = downsample_ratio
self.dynamic_image_size = dynamic_image_size
self.force_image_size = force_image_size
self.image_token_id = image_token_id
self.max_dynamic_patch = max_dynamic_patch
self.min_dynamic_patch = min_dynamic_patch
self.pad2square = pad2square
self.ps_version = ps_version
self.select_layer = select_layer
self.template = template
self.use_thumbnail = use_thumbnail
self.tie_word_embeddings = tie_word_embeddings
@@ -1,455 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
# ruff: noqa
# mypy: ignore-errors
# coding=utf-8
# Copyright 2026 The Qwen team, Alibaba Group and the HuggingFace Inc. team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from transformers.configuration_utils import PreTrainedConfig
from transformers.modeling_rope_utils import rope_config_validation
from transformers.utils import logging
logger = logging.get_logger(__name__)
class Qwen3ASRAudioEncoderConfig(PreTrainedConfig):
r"""
This is the configuration class to store the configuration of a [`Qwen3ASRAudioEncoder`]. It is used to instantiate a
Qwen3-ASR audio encoder according to the specified arguments, defining the model architecture. Instantiating a
configuration with the defaults will yield a similar configuration to that of the audio encoder of the Qwen2-Audio
architecture.
e.g. [Qwen/Qwen3-ASR-1.7B](https://huggingface.co/Qwen/Qwen3-ASR-1.7B)
Configuration objects inherit from [`PreTrainedConfig`] and can be used to control the model outputs. Read the
documentation from [`PreTrainedConfig`] for more information.
Args:
num_mel_bins (`int`, *optional*, defaults to 128):
Number of mel features used per input features. Should correspond to the value used in the
`Qwen3ASRProcessor` class.
encoder_layers (`int`, *optional*, defaults to 32):
Number of encoder layers.
encoder_attention_heads (`int`, *optional*, defaults to 20):
Number of attention heads for each attention layer in the Transformer encoder.
encoder_ffn_dim (`int`, *optional*, defaults to 5120):
Dimensionality of the "intermediate" (often named feed-forward) layer in encoder.
d_model (`int`, *optional*, defaults to 1280):
Dimensionality of the layers.
dropout (`float`, *optional*, defaults to 0.0):
The dropout probability for all fully connected layers in the embeddings, encoder, and pooler.
attention_dropout (`float`, *optional*, defaults to 0.0):
The dropout ratio for the attention probabilities.
activation_function (`str`, *optional*, defaults to `"gelu"`):
The non-linear activation function (function or string) in the encoder and pooler. If string, `"gelu"`,
`"relu"`, `"silu"` and `"gelu_new"` are supported.
activation_dropout (`float`, *optional*, defaults to 0.0):
The dropout ratio for activations inside the fully connected layer.
scale_embedding (`bool`, *optional*, defaults to `False`):
Scale embeddings by diving by sqrt(d_model).
initializer_range (`float`, *optional*, defaults to 0.02):
The standard deviation of the truncated_normal_initializer for initializing all weight matrices.
max_source_positions (`int`, *optional*, defaults to 1500):
The maximum sequence length of log-mel filter-bank features that this model might ever be used with.
n_window (`int`, *optional*, defaults to 100):
The chunk for conv and flash attn in AudioEncoder.
output_dim (`int`, *optional*, defaults to 3584):
The output dimension of AudioEncoder.
Example:
```python
>>> from transformers import Qwen3ASRAudioEncoderConfig, Qwen3ASRAudioEncoder
>>> # Initializing a Qwen3ASRAudioEncoderConfig
>>> configuration = Qwen3ASRAudioEncoderConfig()
>>> # Initializing a Qwen3ASRAudioEncoder (with random weights)
>>> model = Qwen3ASRAudioEncoder(configuration)
>>> # Accessing the model configuration
>>> configuration = model.config
```"""
model_type = "qwen3_asr_audio_encoder"
def __init__(
self,
num_mel_bins=128,
encoder_layers=32,
encoder_attention_heads=20,
encoder_ffn_dim=5120,
d_model=1280,
dropout=0,
attention_dropout=0,
activation_function="gelu",
activation_dropout=0,
scale_embedding=False,
initializer_range=0.02,
max_source_positions=1500,
n_window=100,
output_dim=3584,
n_window_infer=400,
conv_chunksize=500,
downsample_hidden_size=480,
**kwargs,
):
super().__init__(**kwargs)
self.num_mel_bins = num_mel_bins
self.d_model = d_model
self.encoder_layers = encoder_layers
self.encoder_attention_heads = encoder_attention_heads
self.encoder_ffn_dim = encoder_ffn_dim
self.dropout = dropout
self.attention_dropout = attention_dropout
self.activation_function = activation_function
self.activation_dropout = activation_dropout
self.num_hidden_layers = encoder_layers
self.initializer_range = initializer_range
self.scale_embedding = (
scale_embedding # scale factor will be sqrt(d_model) if True
)
self.max_source_positions = max_source_positions
self.n_window = n_window
self.output_dim = output_dim
self.n_window_infer = n_window_infer
self.conv_chunksize = conv_chunksize
self.downsample_hidden_size = downsample_hidden_size
class Qwen3ASRTextConfig(PreTrainedConfig):
r"""
This is the configuration class to store the configuration of a [`Qwen3ASRTextModel`]. It is used to instantiate a
Qwen3-ASR model according to the specified arguments, defining the model architecture. Instantiating a configuration
with the defaults will yield a similar configuration to that of
Qwen3-ASR-1.7B [Qwen/Qwen3-ASR-1.7B](https://huggingface.co/Qwen/Qwen3-ASR-1.7B)
Configuration objects inherit from [`PreTrainedConfig`] and can be used to control the model outputs. Read the
documentation from [`PreTrainedConfig`] for more information.
Args:
vocab_size (`int`, *optional*, defaults to 151936):
Vocabulary size of the Qwen3ASR model. Defines the number of different tokens that can be represented by the
`inputs_ids` passed when calling [`Qwen3ASRModel`]
hidden_size (`int`, *optional*, defaults to 4096):
Dimension of the hidden representations.
intermediate_size (`int`, *optional*, defaults to 22016):
Dimension of the MLP representations.
num_hidden_layers (`int`, *optional*, defaults to 32):
Number of hidden layers in the Transformer encoder.
num_attention_heads (`int`, *optional*, defaults to 32):
Number of attention heads for each attention layer in the Transformer encoder.
num_key_value_heads (`int`, *optional*, defaults to 32):
This is the number of key_value heads that should be used to implement Grouped Query Attention. If
`num_key_value_heads=num_attention_heads`, the model will use Multi Head Attention (MHA), if
`num_key_value_heads=1` the model will use Multi Query Attention (MQA) otherwise GQA is used. When
converting a multi-head checkpoint to a GQA checkpoint, each group key and value head should be constructed
by meanpooling all the original heads within that group. For more details, check out [this
paper](https://huggingface.co/papers/2305.13245). If it is not specified, will default to `32`.
head_dim (`int`, *optional*, defaults to 128):
The dimension of the head. If not specified, will default to `hidden_size // num_attention_heads`.
hidden_act (`str` or `function`, *optional*, defaults to `"silu"`):
The non-linear activation function (function or string) in the decoder.
max_position_embeddings (`int`, *optional*, defaults to 128000):
The maximum sequence length that this model might ever be used with.
initializer_range (`float`, *optional*, defaults to 0.02):
The standard deviation of the truncated_normal_initializer for initializing all weight matrices.
rms_norm_eps (`float`, *optional*, defaults to 1e-06):
The epsilon used by the rms normalization layers.
use_cache (`bool`, *optional*, defaults to `True`):
Whether or not the model should return the last key/values attentions (not used by all models). Only
relevant if `config.is_decoder=True`.
tie_word_embeddings (`bool`, *optional*, defaults to `False`):
Whether the model's input and output word embeddings should be tied.
rope_theta (`float`, *optional*, defaults to 5000000.0):
The base period of the RoPE embeddings.
rope_scaling (`Dict`, *optional*):
Dictionary containing the scaling configuration for the RoPE embeddings. NOTE: if you apply new rope type
and you expect the model to work on longer `max_position_embeddings`, we recommend you to update this value
accordingly.
Expected contents:
`rope_type` (`str`):
The sub-variant of RoPE to use. Can be one of ['default', 'linear', 'dynamic', 'yarn', 'longrope',
'llama3'], with 'default' being the original RoPE implementation.
`factor` (`float`, *optional*):
Used with all rope types except 'default'. The scaling factor to apply to the RoPE embeddings. In
most scaling types, a `factor` of x will enable the model to handle sequences of length x *
original maximum pre-trained length.
`original_max_position_embeddings` (`int`, *optional*):
Used with 'dynamic', 'longrope' and 'llama3'. The original max position embeddings used during
pretraining.
`attention_factor` (`float`, *optional*):
Used with 'yarn' and 'longrope'. The scaling factor to be applied on the attention
computation. If unspecified, it defaults to value recommended by the implementation, using the
`factor` field to infer the suggested value.
`beta_fast` (`float`, *optional*):
Only used with 'yarn'. Parameter to set the boundary for extrapolation (only) in the linear
ramp function. If unspecified, it defaults to 32.
`beta_slow` (`float`, *optional*):
Only used with 'yarn'. Parameter to set the boundary for interpolation (only) in the linear
ramp function. If unspecified, it defaults to 1.
`short_factor` (`list[float]`, *optional*):
Only used with 'longrope'. The scaling factor to be applied to short contexts (<
`original_max_position_embeddings`). Must be a list of numbers with the same length as the hidden
size divided by the number of attention heads divided by 2
`long_factor` (`list[float]`, *optional*):
Only used with 'longrope'. The scaling factor to be applied to long contexts (<
`original_max_position_embeddings`). Must be a list of numbers with the same length as the hidden
size divided by the number of attention heads divided by 2
`low_freq_factor` (`float`, *optional*):
Only used with 'llama3'. Scaling factor applied to low frequency components of the RoPE
`high_freq_factor` (`float`, *optional*):
Only used with 'llama3'. Scaling factor applied to high frequency components of the RoPE
attention_bias (`bool`, defaults to `False`, *optional*, defaults to `False`):
Whether to use a bias in the query, key, value and output projection layers during self-attention.
attention_dropout (`float`, *optional*, defaults to 0.0):
The dropout ratio for the attention probabilities.
```python
>>> from transformers import Qwen3ASRTextModel, Qwen3ASRTextConfig
>>> # Initializing a Qwen3ASR style configuration
>>> configuration = Qwen3ASRTextConfig()
>>> # Initializing a model from the Qwen3-VL-7B style configuration
>>> model = Qwen3ASRTextModel(configuration)
>>> # Accessing the model configuration
>>> configuration = model.config
```"""
model_type = "qwen3_asr_text"
base_config_key = "text_config"
def __init__(
self,
vocab_size=151936,
hidden_size=4096,
intermediate_size=22016,
num_hidden_layers=32,
num_attention_heads=32,
num_key_value_heads=32,
head_dim=128,
hidden_act="silu",
max_position_embeddings=128000,
initializer_range=0.02,
rms_norm_eps=1e-6,
use_cache=True,
tie_word_embeddings=False,
rope_theta=5000000.0,
rope_scaling=None,
attention_bias=False,
attention_dropout=0.0,
**kwargs,
):
self.vocab_size = vocab_size
self.max_position_embeddings = max_position_embeddings
self.hidden_size = hidden_size
self.intermediate_size = intermediate_size
self.num_hidden_layers = num_hidden_layers
self.num_attention_heads = num_attention_heads
# for backward compatibility
if num_key_value_heads is None:
num_key_value_heads = num_attention_heads
self.num_key_value_heads = num_key_value_heads
self.head_dim = head_dim
self.hidden_act = hidden_act
self.initializer_range = initializer_range
self.rms_norm_eps = rms_norm_eps
self.use_cache = use_cache
self.rope_theta = rope_theta
self.rope_scaling = rope_scaling
self.attention_bias = attention_bias
self.attention_dropout = attention_dropout
# Validate the correctness of rotary position embeddings parameters
# BC: if there is a 'type' field, move it to 'rope_type'.
if self.rope_scaling is not None and "type" in self.rope_scaling:
self.rope_scaling["rope_type"] = self.rope_scaling["type"]
rope_config_validation(self)
super().__init__(tie_word_embeddings=tie_word_embeddings, **kwargs)
class Qwen3ASRThinkerConfig(PreTrainedConfig):
r"""
This is the configuration class to store the configuration of a [`Qwen3ASRThinker`]. It is used to instantiate a
Qwen3-ASR-Thinker model according to the specified arguments, defining the model architecture. Instantiating a
configuration with the defaults will yield a similar configuration to that of the thinker component of the Qwen3-Omni
architecture.
e.g. [Qwen/Qwen3-ASR-1.7B](https://huggingface.co/Qwen/Qwen3-ASR-1.7B)
Configuration objects inherit from [`PreTrainedConfig`] and can be used to control the model outputs. Read the
documentation from [`PreTrainedConfig`] for more information.
Args:
audio_config (`dict`, *optional*):
The config dictionary of the audio backbone.
text_config (`dict`, *optional*):
The config dictionary of the text backbone.
audio_token_id (`int`, *optional*, defaults to 151646):
The audio token id to encode the audio prompt.
audio_start_token_id (`int`, *optional*, defaults to 151647):
The audio start token id to encode the audio prompt.
user_token_id (`int`, *optional*, defaults to 872):
The user token id to encode the user token.
initializer_range (`float`, *optional*, defaults to 0.02):
The standard deviation of the truncated_normal_initializer for initializing all weight matrices.
Example:
```python
>>> from transformers import Qwen3ASRThinkerModel, Qwen3ASRThinkerConfig
>>> # Initializing a default Qwen3ASRThinkerConfig
>>> configuration = Qwen3ASRThinkerConfig()
>>> # Initializing a model (with random weights) from the default configuration
>>> model = Qwen3ASRThinkerModel(configuration)
>>> # Accessing the model configuration
>>> configuration = model.config
```"""
model_type = "qwen3_asr_thinker"
attribute_map = {}
sub_configs = {
"audio_config": Qwen3ASRAudioEncoderConfig,
"text_config": Qwen3ASRTextConfig,
}
def __init__(
self,
audio_config=None,
text_config=None,
audio_token_id=151646,
audio_start_token_id=151647,
user_token_id=872,
initializer_range=0.02,
classify_num=None,
**kwargs,
):
super().__init__(**kwargs)
self.user_token_id = user_token_id
self.audio_start_token_id = audio_start_token_id
self.initializer_range = initializer_range
self.classify_num = classify_num
if isinstance(audio_config, dict):
audio_config = Qwen3ASRAudioEncoderConfig(**audio_config)
elif audio_config is None:
audio_config = Qwen3ASRAudioEncoderConfig()
self.audio_config = audio_config
if isinstance(text_config, dict):
text_config = Qwen3ASRTextConfig(**text_config)
elif text_config is None:
text_config = Qwen3ASRTextConfig()
self.text_config = text_config
self.audio_token_id = audio_token_id
class Qwen3ASRConfig(PreTrainedConfig):
"""
This is the configuration class to store the configuration of a [`Qwen3ASRForConditionalGeneration`]. It is used to instantiate a Qwen3ASR
model according to the specified sub-models configurations, defining the model architecture.
Instantiating a configuration with the defaults will yield a similar configuration to that of the
[Qwen/Qwen3-ASR-1.7B](https://huggingface.co/Qwen/Qwen3-ASR-1.7B) architecture.
Configuration objects inherit from [`PreTrainedConfig`] and can be used to control the model outputs. Read the
documentation from [`PreTrainedConfig`] for more information.
Args:
thinker_config (`dict`, *optional*): Configuration of the underlying thinker sub-model.
support_languages (`List[str]`, *optional*): The languages supported by the model.
Example:
```python
>>> from transformers import (
... Qwen3ASRThinkerConfig,
... Qwen3ASRForConditionalGeneration,
... Qwen3ASRConfig,
... )
>>> # Initializing a Qwen3ASR style configuration
>>> configuration = Qwen3ASRConfig()
>>> # Initializing a model from the configuration
>>> model = Qwen3ASRForConditionalGeneration(configuration)
>>> # Accessing the model configuration
>>> configuration = model.config
```"""
model_type = "qwen3_asr"
sub_configs = {
"thinker_config": Qwen3ASRThinkerConfig,
}
def __init__(
self,
thinker_config=None,
audio_config=None,
text_config=None,
audio_token_id=None,
support_languages=None,
timestamp_token_id=None,
timestamp_segment_time=None,
**kwargs,
):
if thinker_config is None:
thinker_config = {
key: value
for key, value in (
("audio_config", audio_config),
("text_config", text_config),
("audio_token_id", audio_token_id),
)
if value is not None
}
if not thinker_config:
logger.info(
"thinker_config is None. Initializing thinker model with "
"default values"
)
self.thinker_config = Qwen3ASRThinkerConfig(**thinker_config)
self.support_languages = support_languages
self.timestamp_token_id = timestamp_token_id
self.timestamp_segment_time = timestamp_segment_time
super().__init__(**kwargs)
def get_text_config(self, decoder=False) -> "PreTrainedConfig":
"""
Returns the config that is meant to be used with text IO. On most models, it is the original config instance
itself. On specific composite models, it is under a set of valid names.
Args:
decoder (`Optional[bool]`, *optional*, defaults to `False`):
If set to `True`, then only search for decoder config names.
"""
# Overridden for deeply nested config like Qwen2.5-Omni. We don't have any omni model
# except for Qwen yet. This has to be generalized if more deeply nested configs are
# added. NOTE: currently method used only by vLLM
return self.thinker_config.get_text_config()
__all__ = ["Qwen3ASRConfig", "Qwen3ASRThinkerConfig", "Qwen3ASRAudioEncoderConfig"]
-110
View File
@@ -1,110 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Radio vision model configuration."""
from typing import Any
from transformers.configuration_utils import PreTrainedConfig
from transformers.utils import logging
logger = logging.get_logger(__name__)
VIT_TIMM_DIM_BY_NAME: dict[str, tuple[int, int, int, int]] = {
"vit_small_patch16_224": (384, 12, 6, 1536),
"vit_base_patch16_224": (768, 12, 12, 3072),
"vit_large_patch16_224": (1024, 24, 16, 4096),
"vit_huge_patch16_224": (1280, 32, 16, 5120),
}
OPENAI_CLIP_MEAN = (0.48145466, 0.4578275, 0.40821073)
OPENAI_CLIP_STD = (0.26862954, 0.26130258, 0.27577711)
class RadioConfig(PreTrainedConfig):
r"""This is the configuration class to store the configuration of a Radio
vision model. It is used to instantiate a Radio model according to the
specified arguments, defining the model architecture.
Args:
model_name: Name of the vision transformer model
(e.g., "vit_base_patch16_224"). Used to determine architecture
dimensions from `VIT_TIMM_DIM_BY_NAME`.
image_size: The size (resolution) of each image.
patch_size: The size (resolution) of each patch.
qkv_bias: Whether to add a bias to the queries, keys and values.
qk_normalization: Whether to apply normalization to queries and keys.
norm_type: The normalization type to use.
layer_norm_eps: The epsilon used by the layer normalization layers.
initializer_factor: A factor for initializing all weight matrices.
hidden_act: The non-linear activation function in the encoder.
cpe_max_size: Maximum image size for position embeddings.
norm_mean: Mean values for image normalization (RGB channels).
Defaults to (0.48145466, 0.4578275, 0.40821073)).
norm_std: Standard deviation values for image normalization
(RGB channels). Defaults to (0.26862954, 0.26130258, 0.27577711)).
register_multiple: Number of register tokens to use.
teachers: A list of teacher model configurations. Each teacher configuration is
a dict with keys like "name" and some may have "use_summary".
cls_token_per_teacher: Whether to use a separate CLS token for each teacher.
video_temporal_patch_size: Number of consecutive video frames grouped into
a single tubelet for temporal compression. Default 1 (no compression).
When > 1, a dedicated video_embedder (3*T*P*P -> hidden) is created
alongside the image embedder (3*P*P -> hidden).
separate_video_embedder: When True and video_temporal_patch_size > 1, use a
dedicated video patch embedder (3*T*P*P -> hidden) separate from the
image embedder (3*P*P -> hidden). When False, a single embedder with
input size 3*T*P*P is used for both (images are duplicated T times).
"""
model_type = "radio"
def __init__(
self,
model_name: str,
image_size: int = 224,
patch_size: int = 16,
qkv_bias: bool = True,
qk_normalization: bool = False,
norm_type: str = "layer_norm",
layer_norm_eps: float = 1e-6,
initializer_factor: float = 1.0,
hidden_act: str = "gelu",
cpe_max_size: int = 2048,
norm_mean: tuple[float, float, float] | list = OPENAI_CLIP_MEAN,
norm_std: tuple[float, float, float] | list = OPENAI_CLIP_STD,
register_multiple: int | None = None,
teachers: list[dict[str, Any]] | None = None,
cls_token_per_teacher: bool = False,
video_temporal_patch_size: int = 1,
separate_video_embedder: bool = True,
**kwargs,
):
self.model_name = model_name
(
self.hidden_size,
self.num_hidden_layers,
self.num_attention_heads,
self.intermediate_size,
) = VIT_TIMM_DIM_BY_NAME[model_name]
self.image_size = image_size
self.patch_size = patch_size
self.qkv_bias = qkv_bias
self.qk_normalization = qk_normalization
self.norm_type = norm_type
self.layer_norm_eps = layer_norm_eps
self.initializer_factor = initializer_factor
self.hidden_act = hidden_act
self.cpe_max_size = cpe_max_size
self.norm_mean = (
list(norm_mean) if isinstance(norm_mean, (tuple, list)) else norm_mean
)
self.norm_std = (
list(norm_std) if isinstance(norm_std, (tuple, list)) else norm_std
)
self.register_multiple = register_multiple
self.teachers = teachers if teachers is not None else []
self.cls_token_per_teacher = cls_token_per_teacher
self.video_temporal_patch_size = video_temporal_patch_size
self.separate_video_embedder = separate_video_embedder
super().__init__(**kwargs)
-178
View File
@@ -1,178 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from typing import Any
from transformers.configuration_utils import PreTrainedConfig
class Step3VisionEncoderConfig(PreTrainedConfig):
model_type = "step3_vision_encoder"
def __init__(
self,
hidden_size=1792,
intermediate_size=3072,
output_hidden_size=4096,
num_hidden_layers=63,
num_attention_heads=16,
num_channels=3,
image_size=728,
patch_size=14,
hidden_act="quick_gelu",
layer_norm_eps=1e-5,
**kwargs,
):
self.hidden_size = hidden_size
self.intermediate_size = intermediate_size
self.output_hidden_size = output_hidden_size
self.num_hidden_layers = num_hidden_layers
self.num_attention_heads = num_attention_heads
self.num_channels = num_channels
self.patch_size = patch_size
self.image_size = image_size
self.layer_norm_eps = layer_norm_eps
self.hidden_act = hidden_act
super().__init__(**kwargs)
class Step3TextConfig(PreTrainedConfig):
model_type = "step3_text"
architectures = ["Step3TextForCausalLM"]
def __init__(
self,
hidden_size: int = 7168,
intermediate_size: int = 18432,
num_attention_heads: int = 64,
num_attention_groups: int = 1,
num_hidden_layers: int = 61,
max_seq_len: int = 65536,
vocab_size: int = 128815,
rms_norm_eps: float = 1e-5,
moe_intermediate_size: int = 5120,
moe_num_experts: int = 48,
moe_top_k: int = 3,
rope_parameters: dict[str, Any] | None = None,
max_position_embedding: int = 65536,
share_expert_dim: int = 5120,
share_q_dim: int = 2048,
head_dim: int = 256,
norm_expert_weight: bool = False,
moe_layers_enum: tuple[int, ...] = (
4,
5,
6,
7,
8,
9,
10,
11,
12,
13,
14,
15,
16,
17,
18,
19,
20,
21,
22,
23,
24,
25,
26,
27,
28,
29,
30,
31,
32,
33,
34,
35,
36,
37,
38,
39,
40,
41,
42,
43,
44,
45,
46,
47,
48,
49,
50,
51,
52,
53,
54,
55,
56,
57,
58,
59,
),
**kwargs,
) -> None:
self.hidden_size = hidden_size
self.intermediate_size = intermediate_size
self.num_attention_heads = num_attention_heads
self.num_attention_groups = num_attention_groups
self.num_hidden_layers = num_hidden_layers
self.max_seq_len = max_seq_len
self.vocab_size = vocab_size
self.rms_norm_eps = rms_norm_eps
self.moe_intermediate_size = moe_intermediate_size
self.moe_num_experts = moe_num_experts
self.moe_top_k = moe_top_k
# Try to set `rope_scaling` if available, otherwise use `rope_parameters`
rope_scaling = kwargs.pop("rope_scaling", None)
rope_parameters = rope_scaling or rope_parameters or {"rope_type": "default"}
rope_theta = kwargs.pop("rope_theta", 500000.0)
if "rope_theta" not in rope_parameters:
rope_parameters["rope_theta"] = rope_theta
self.rope_parameters = rope_parameters
self.max_position_embedding = max_position_embedding
self.share_expert_dim = share_expert_dim
self.share_q_dim = share_q_dim
self.head_dim = head_dim
self.norm_expert_weight = norm_expert_weight
self.moe_layers_enum = moe_layers_enum
super().__init__(**kwargs)
class Step3VLConfig(PreTrainedConfig):
model_type = "step3_vl"
def __init__(
self,
vision_config: dict | Step3VisionEncoderConfig | None = None,
text_config: dict | Step3TextConfig | None = None,
understand_projector_stride: int = 1,
projector_bias: bool = True,
image_token_id: int = 128001,
**kwargs,
) -> None:
if vision_config is None:
vision_config = Step3VisionEncoderConfig()
elif isinstance(vision_config, dict):
vision_config = Step3VisionEncoderConfig(**vision_config)
self.vision_config = vision_config
if text_config is None:
text_config = Step3TextConfig()
elif isinstance(text_config, dict):
text_config = Step3TextConfig(**text_config)
self.text_config = text_config
self.understand_projector_stride = understand_projector_stride
self.projector_bias = projector_bias
self.hidden_size = text_config.hidden_size
self.image_token_id = image_token_id
super().__init__(**kwargs)
-103
View File
@@ -1,103 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from typing import Any
from transformers.configuration_utils import PreTrainedConfig
class Step3p5Config(PreTrainedConfig):
model_type = "step3p5"
def __init__(
self,
hidden_size: int = 5120,
intermediate_size: int = 13312,
num_attention_heads: int = 40,
num_attention_groups: int = 8,
num_hidden_layers: int = 48,
max_seq_len: int = 4096,
vocab_size: int = 65536,
rms_norm_eps: float = 1e-5,
moe_every_n_layer: int = 2,
use_moe: bool = False,
moe_intermediate_size: int = 10240,
moe_num_experts: int = 16,
moe_top_k: int = 4,
moe_layer_offset: int = 0,
rope_theta: float | list[float] | None = 500000,
rope_scaling: dict[str, Any] | None = None,
head_dim: int | None = None,
share_expert_dim: int | None = None,
norm_expert_weight: bool = True,
bos_token_id: list[int] | int | None = None,
eos_token_id: list[int] | int | None = None,
moe_router_activation: str = "softmax",
moe_router_scaling_factor: float = 1.0,
att_impl_type: str = "GQA",
use_head_wise_attn_gate: bool = False,
use_moe_router_bias: bool = True,
need_fp32_gate: bool = True,
layer_types: list[str] | None = None,
use_rope_layers: list[bool] | None = None,
yarn_only_types: list[str] | None = None,
attention_other_setting: dict[str, Any] | None = None,
num_nextn_predict_layers: int = 0,
swiglu_limits: list[float] | None = None,
swiglu_limits_shared: list[float] | None = None,
max_position_embeddings: int | None = None,
**kwargs,
):
self.hidden_size = hidden_size
self.intermediate_size = intermediate_size
self.num_attention_heads = num_attention_heads
self.num_attention_groups = num_attention_groups
self.num_hidden_layers = num_hidden_layers
self.max_seq_len = max_seq_len
self.vocab_size = vocab_size
self.rms_norm_eps = rms_norm_eps
self.use_moe = use_moe
self.moe_intermediate_size = moe_intermediate_size
self.moe_every_n_layer = moe_every_n_layer
self.moe_num_experts = moe_num_experts
self.num_experts_per_tok = moe_top_k
self.moe_top_k = moe_top_k
self.moe_layer_offset = moe_layer_offset
self.rope_theta = rope_theta
self.rope_scaling = rope_scaling
self.head_dim = head_dim
if share_expert_dim is None:
self.share_expert_dim = self.moe_intermediate_size * self.moe_top_k
else:
self.share_expert_dim = share_expert_dim
self.norm_expert_weight = norm_expert_weight
self.max_position_embeddings = max_position_embeddings
self.moe_router_activation = moe_router_activation
self.moe_router_scaling_factor = moe_router_scaling_factor
self.use_moe_router_bias = use_moe_router_bias
self.need_fp32_gate = need_fp32_gate
self.att_impl_type = att_impl_type
self.use_head_wise_attn_gate = use_head_wise_attn_gate
# For some reason the checkpoint has longer layer_types than num_hidden_layers
if layer_types is not None:
layer_types = layer_types[: self.num_hidden_layers]
self.layer_types = layer_types
self.use_rope_layers = use_rope_layers
self.yarn_only_types = yarn_only_types
self.attention_other_setting = attention_other_setting
self.num_nextn_predict_layers = num_nextn_predict_layers
self.swiglu_limits = swiglu_limits
self.swiglu_limits_shared = swiglu_limits_shared
resolved_bos_token_id = 1 if bos_token_id is None else bos_token_id
resolved_eos_token_id = [2, 3] if eos_token_id is None else eos_token_id
self.bos_token_id = resolved_bos_token_id
self.eos_token_id = resolved_eos_token_id
super().__init__(
bos_token_id=resolved_bos_token_id,
eos_token_id=resolved_eos_token_id,
**kwargs,
)
-120
View File
@@ -1,120 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
# Adapted from https://github.com/fixie-ai/ultravox/blob/ecd58c4041030bae2ad15aa6bcf04ab43199ea02/ultravox/model/ultravox_config.py
from typing import Any
import transformers
class UltravoxConfig(transformers.PreTrainedConfig):
r"""This is the configuration class to store the configuration of a
[`UltravoxForConditionalGeneration`]. It is used to instantiate an
Ultravox model according to the specified arguments, defining the model
architecture.
Configuration objects inherit from [`PreTrainedConfig`] and can be used to
control the model outputs. Read the documentation from [`PreTrainedConfig`]
for more information.
Args:
audio_config (`Union[AutoConfig, dict]`, *optional*):
Custom audio config or dict.
text_config (`Union[AutoConfig, dict]`, *optional*):
The config object of the text backbone.
audio_model_id (`str`, *optional*):
The model ID of the audio backbone.
text_model_id (`str`, *optional*):
The model ID of the text backbone.
ignore_index (`int`, *optional*, defaults to -100):
The ignore index for the loss function.
audio_token_index (`int`, *optional*, defaults to 32000):
The audio token index to encode the audio prompt.
stack_factor (`int`, *optional*, defaults to 8):
Audio downsampling factor for the multimodal projector.
norm_init (`float`, *optional*, defaults to 0.4):
The initialization value for the layer normalization.
projector_act (`str`, *optional*, defaults to `"swiglu"`):
The activation function used by the multimodal projector.
projector_ln_mid (`bool`, *optional*, defaults to `False`):
Whether to apply layer normalization at the middle of the
projector or at the end. Versions v0.4.1 and below
use `False`, but v0.5 and above use `True`.
"""
model_type = "ultravox"
audio_token = "<|audio|>"
is_composition = False
def __init__(
self,
audio_config: dict[str, Any] | None = None,
text_config: dict[str, Any] | None = None,
audio_model_id: str | None = None,
text_model_id: str | None = None,
ignore_index: int = -100,
audio_token_index: int = 32000,
hidden_size: int = 4096,
stack_factor: int = 8,
norm_init: float = 0.4,
projector_act: str = "swiglu",
projector_ln_mid: bool = False,
num_projector_layers: int = 0,
**kwargs,
):
self.ignore_index = ignore_index
self.audio_token_index = audio_token_index
self.hidden_size = hidden_size
self.stack_factor = stack_factor
self.norm_init = norm_init
self.projector_act = projector_act
self.projector_ln_mid = projector_ln_mid
self.num_projector_layers = num_projector_layers
# N.B. May set the wrapped_model_config below.
self.wrapped_model_config: transformers.PreTrainedConfig
self.text_model_id = text_model_id
if text_model_id is None:
text_config = text_config or {}
self.wrapped_model_config = transformers.CONFIG_MAPPING[
text_config.get("model_type", "llama")
](**text_config)
# N.B. May set the audio_config below.
self.audio_model_id = audio_model_id
if audio_model_id is None:
self.audio_model_id = None
audio_config = audio_config or {}
self.audio_config = transformers.CONFIG_MAPPING[
audio_config.get("model_type", "whisper")
](**audio_config)
super().__init__(**kwargs)
def __setattr__(self, key, value):
# Since --hf-overrides are applied _after_ the UltravoxConfig is
# instantiated, load the configs implicitly when assigning text_model_id
# or audio_model_id. This allows:
#
# --hf-overrides.text_model_id=<quantized variant>
#
# to behave as intended.
if key == "text_model_id" and value is not None:
from vllm.transformers_utils.config import get_config
self.wrapped_model_config = get_config(value, trust_remote_code=False)
elif key == "audio_model_id" and value is not None:
from vllm.transformers_utils.config import get_config
self.audio_config = get_config(value, trust_remote_code=False)
return super().__setattr__(key, value)
@property
def text_config(self) -> transformers.PreTrainedConfig:
# When Ultravox wraps a multi-modal model (e.g. Gemma), we instantiate
# the full model, but the text config is the text config of the inner
# model.
return self.wrapped_model_config.get_text_config()
@@ -1,35 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
# Unlimited-OCR (baidu/Unlimited-OCR) reuses
# the DeepSeek-OCR multimodal layout (DeepEncoder = SAM-ViT-B + CLIP-L, a linear
# MLP projector and a DeepSeek-V2 text backbone). The only architectural
# difference is the language model, which is a DeepSeek-V2 *MoE* with plain
# multi-head attention (``use_mla=False``) instead of the dense MLA backbone.
# We therefore reuse ``DeepseekVLV2Config`` for parsing the nested config.
from vllm.transformers_utils.configs.deepseek_vl2 import DeepseekVLV2Config
class UnlimitedOCRConfig(DeepseekVLV2Config):
model_type = "unlimited-ocr"
# An explicit ``__init__`` is required: Transformers v5 processes each
# concrete config class' ``__init__`` signature to build nested sub-configs,
# and an empty subclass (only overriding ``model_type``) would skip
# ``DeepseekVLV2Config.__init__``, leaving ``text_config`` unset.
def __init__(
self,
tile_tag: str = "2D",
global_view_pos: str = "head",
candidate_resolutions: tuple[tuple[int, int]] = ((384, 384),),
rswa_window: int = 128,
**kwargs,
):
super().__init__(
tile_tag=tile_tag,
global_view_pos=global_view_pos,
candidate_resolutions=candidate_resolutions,
**kwargs,
)
self.rswa_window = rswa_window
+1 -31
View File
@@ -41,37 +41,7 @@ __all__ = [
"Step3VLProcessor",
]
_CLASS_TO_MODULE: dict[str, str] = {
"BagelProcessor": "vllm.transformers_utils.processors.bagel",
"CohereASRProcessor": "vllm.transformers_utils.processors.cohere_asr",
"Cosmos3EdgeProcessor": "vllm.transformers_utils.processors.cosmos3_edge",
"DeepseekVLV2Processor": "vllm.transformers_utils.processors.deepseek_vl2",
"FireRedASR2Processor": "vllm.transformers_utils.processors.fireredasr2",
"FunASRProcessor": "vllm.transformers_utils.processors.funasr",
"GLM4VProcessor": "vllm.transformers_utils.processors.glm4v",
"H2OVLProcessor": "vllm.transformers_utils.processors.h2ovl",
"IsaacProcessor": "vllm.transformers_utils.processors.isaac",
"KimiAudioProcessor": "vllm.transformers_utils.processors.kimi_audio",
"KimiK25Processor": "vllm.transformers_utils.processors.kimi_k25",
"KimiK3Processor": "vllm.transformers_utils.processors.kimi_k3",
"MiMoOmniProcessor": "vllm.transformers_utils.processors.mimo_v2_omni",
"MiniCPMOProcessor": "vllm.transformers_utils.processors.minicpmo",
"MiniCPMVProcessor": "vllm.transformers_utils.processors.minicpmv",
"MiniMaxM3VLImageProcessor": "vllm.transformers_utils.processors.minimax_m3",
"MiniMaxM3VLVideoProcessor": "vllm.transformers_utils.processors.minimax_m3",
"MiniMaxVLProcessor": "vllm.transformers_utils.processors.minimax_m3",
"MistralCommonPixtralProcessor": "vllm.transformers_utils.processors.pixtral",
"MistralCommonVoxtralProcessor": "vllm.transformers_utils.processors.voxtral",
"Moondream3Processor": "vllm.transformers_utils.processors.moondream3",
"NanoNemotronVLProcessor": "vllm.transformers_utils.processors.nano_nemotron_vl",
"NemotronVLProcessor": "vllm.transformers_utils.processors.nemotron_vl",
"LlamaNemotronVLEmbedProcessor": "vllm.transformers_utils.processors.nemotron_vl",
"NVLMProcessor": "vllm.transformers_utils.processors.nvlm_d",
"OpenVLAProcessor": "vllm.transformers_utils.processors.openvla",
"OvisProcessor": "vllm.transformers_utils.processors.ovis",
"Ovis2_5Processor": "vllm.transformers_utils.processors.ovis2_5",
"Step3VLProcessor": "vllm.transformers_utils.processors.step3_vl",
}
_CLASS_TO_MODULE: dict[str, str] = {}
def __getattr__(name: str):
@@ -1,73 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
# Copyright 2025 Bytedance Ltd. and/or its affiliates.
"""BAGEL processor for image and text inputs."""
from transformers.feature_extraction_utils import BatchFeature
from transformers.image_utils import ImageInput
from transformers.processing_utils import ProcessingKwargs, ProcessorMixin, Unpack
from transformers.tokenization_utils_base import PreTokenizedInput, TextInput
class BagelProcessorKwargs(ProcessingKwargs, total=False): # type: ignore[call-arg]
_defaults = {
"images_kwargs": {
"return_tensors": "pt",
},
}
class BagelProcessor(ProcessorMixin):
"""Constructs a BAGEL processor which wraps a
SigLIP image processor and a Qwen2 tokenizer.
"""
attributes = ["image_processor", "tokenizer"]
image_processor_class = "SiglipImageProcessor"
tokenizer_class = "AutoTokenizer"
def __call__(
self,
text: TextInput
| PreTokenizedInput
| list[TextInput]
| list[PreTokenizedInput] = None,
images: ImageInput = None,
**kwargs: Unpack[BagelProcessorKwargs],
):
"""Prepare one or more sequences and images for the model."""
output_kwargs = self._merge_kwargs(
BagelProcessorKwargs,
tokenizer_init_kwargs=self.tokenizer.init_kwargs,
**kwargs,
)
if images is not None:
# Process images with the image processor
pixel_values = self.image_processor(
images, **output_kwargs["images_kwargs"]
)
else:
pixel_values = {}
text_inputs = (
self.tokenizer(text, **output_kwargs["text_kwargs"])
if text is not None
else {}
)
return BatchFeature(data={**pixel_values, **text_inputs})
def batch_decode(self, *args, **kwargs):
"""Forward all arguments to `Qwen2TokenizerFast.batch_decode`."""
return self.tokenizer.batch_decode(*args, **kwargs)
def decode(self, *args, **kwargs):
"""This method forwards all its arguments to Qwen2TokenizerFast's decode."""
return self.tokenizer.decode(*args, **kwargs)
@property
def model_input_names(self):
tokenizer_input_names = self.tokenizer.model_input_names
image_processor_input_names = self.image_processor.model_input_names
return list(dict.fromkeys(tokenizer_input_names + image_processor_input_names))
@@ -1,573 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import logging
import math
import random
import numpy as np
import torch
import torch.nn.functional as F
from torch import nn
from torchaudio.functional import melscale_fbanks
from transformers import AutoFeatureExtractor, AutoProcessor, BatchFeature
from transformers.feature_extraction_sequence_utils import (
SequenceFeatureExtractor,
)
from transformers.processing_utils import ProcessorMixin
logger = logging.getLogger(__name__)
CONSTANT = 1e-5
INF_VAL = 10000.0
class FilterbankFeatures(nn.Module):
"""Featurizer that converts wavs to Mel Spectrograms.
See AudioToMelSpectrogramPreprocessor for args.
"""
window: torch.Tensor
fb: torch.Tensor
def __init__(
self,
sample_rate=16000,
n_window_size=320,
n_window_stride=160,
window="hann",
normalize="per_feature",
n_fft=None,
preemph=0.97,
nfilt=64,
lowfreq=0,
highfreq=None,
log=True,
log_zero_guard_type="add",
log_zero_guard_value=2**-24,
dither=CONSTANT,
pad_to=16,
max_duration=30,
frame_splicing=1,
exact_pad=False,
pad_value=0,
mag_power=2.0,
use_grads=False,
rng=None,
nb_augmentation_prob=0.0,
nb_max_freq=4000,
mel_norm="slaney",
stft_exact_pad=False,
stft_conv=False,
device="cpu",
):
super().__init__()
if stft_conv or stft_exact_pad:
logger.warning(
"Using torch_stft is deprecated and has been removed. "
"The values have been forcibly set to False for "
"FilterbankFeatures and AudioToMelSpectrogramPreprocessor. "
"Please set exact_pad to True as needed."
)
if exact_pad and n_window_stride % 2 == 1:
raise NotImplementedError(
f"{self} received exact_pad == True, but hop_size was odd. "
"If audio_length % hop_size == 0, the returned spectrogram "
"would not be of length audio_length // hop_size. "
"Please use an even hop_size."
)
self.log_zero_guard_value = log_zero_guard_value
if (
n_window_size is None
or n_window_stride is None
or not isinstance(n_window_size, int)
or not isinstance(n_window_stride, int)
or n_window_size <= 0
or n_window_stride <= 0
):
raise ValueError(
f"{self} got an invalid value for either n_window_size or "
f"n_window_stride. Both must be positive ints."
)
self.sample_rate = sample_rate
self.win_length = n_window_size
self.hop_length = n_window_stride
self.n_fft = n_fft or 2 ** math.ceil(math.log2(self.win_length))
self.stft_pad_amount = (
(self.n_fft - self.hop_length) // 2 if exact_pad else None
)
self.exact_pad = exact_pad
self.sample_rate = sample_rate
self.max_duration = max_duration
if exact_pad:
logger.info("STFT using exact pad")
torch_windows = {
"hann": torch.hann_window,
"hamming": torch.hamming_window,
"blackman": torch.blackman_window,
"bartlett": torch.bartlett_window,
"none": None,
}
window_fn = torch_windows.get(window)
window_tensor = (
window_fn(self.win_length, periodic=False) if window_fn else None
)
self.register_buffer("window", window_tensor)
self.normalize = normalize
self.log = log
self.dither = dither
self.frame_splicing = frame_splicing
self.nfilt = nfilt
self.preemph = preemph
self.pad_to = pad_to
highfreq = highfreq or sample_rate / 2
self.sample_rate = sample_rate
# disable pad min duration
# self.pad_min_duration = 1.0
self.pad_min_duration = 0.0
self.pad_direction = "both"
filterbanks = melscale_fbanks(
n_freqs=self.n_fft // 2 + 1,
f_min=lowfreq,
f_max=highfreq,
n_mels=nfilt,
sample_rate=sample_rate,
norm=mel_norm,
mel_scale="slaney",
).T.unsqueeze(0)
self.register_buffer("fb", filterbanks)
# Calculate maximum sequence length
max_length = self.get_seq_len(
torch.tensor(max_duration * sample_rate, dtype=torch.float)
)
max_pad = pad_to - (max_length % pad_to) if pad_to > 0 else 0
self.max_length = max_length + max_pad
self.pad_value = pad_value
self.mag_power = mag_power
# We want to avoid taking the log of zero
# There are two options: either adding or clamping to a small value
if log_zero_guard_type not in ["add", "clamp"]:
raise ValueError(
f"{self} received {log_zero_guard_type} for the "
f"log_zero_guard_type parameter. It must be either 'add' or "
f"'clamp'."
)
self.use_grads = use_grads
if not use_grads:
self.forward = torch.no_grad()(self.forward)
self._rng = random.Random() if rng is None else rng
self.nb_augmentation_prob = nb_augmentation_prob
if self.nb_augmentation_prob > 0.0:
if nb_max_freq >= sample_rate / 2:
self.nb_augmentation_prob = 0.0
else:
self._nb_max_fft_bin = int((nb_max_freq / sample_rate) * n_fft)
# log_zero_guard_value is the the small we want to use, we support
# an actual number, or "tiny", or "eps"
self.log_zero_guard_type = log_zero_guard_type
assert self.window is not None
assert self.fb is not None
self.window = self.window.to(dtype=torch.bfloat16)
self.fb = self.fb.to(dtype=torch.bfloat16)
self.generator = torch.Generator(device=device)
self.generator.manual_seed(0)
@torch._dynamo.disable
def stft(self, x):
# disable autocast to get full range of stft values
with torch.amp.autocast(x.device.type, enabled=False):
return torch.stft(
x,
n_fft=self.n_fft,
hop_length=self.hop_length,
win_length=self.win_length,
center=not self.exact_pad,
window=self.window.to(dtype=torch.float, device=x.device),
return_complex=True,
pad_mode="constant",
)
def log_zero_guard_value_fn(self, x):
if isinstance(self.log_zero_guard_value, str):
if self.log_zero_guard_value == "tiny":
return torch.finfo(x.dtype).tiny
elif self.log_zero_guard_value == "eps":
return torch.finfo(x.dtype).eps
else:
raise ValueError(
f"{self} received {self.log_zero_guard_value} for the "
f"log_zero_guard_type parameter. It must be either a "
f"number, 'tiny', or 'eps'"
)
else:
return self.log_zero_guard_value
def get_seq_len(self, seq_len):
# Assuming that center is True is stft_pad_amount = 0
pad_amount = (
self.stft_pad_amount * 2
if self.stft_pad_amount is not None
else self.n_fft // 2 * 2
)
seq_len = torch.floor_divide(
(seq_len + pad_amount - self.n_fft), self.hop_length
)
return seq_len.to(dtype=torch.long)
@property
def filter_banks(self):
return self.fb
def splice_frames(self, x, frame_splicing):
"""Stacks frames together across feature dim.
input is batch_size, feature_dim, num_frames
output is batch_size, feature_dim*frame_splicing, num_frames
"""
seq = [x]
for n in range(1, frame_splicing):
seq.append(torch.cat([x[:, :, :n], x[:, :, n:]], dim=2))
return torch.cat(seq, dim=1)
def normalize_batch(self, x, seq_len, normalize_type):
x_mean = None
x_std = None
if normalize_type == "per_feature":
batch_size = x.shape[0]
max_time = x.shape[2]
# When doing stream capture to a graph, item() is not allowed
# because it calls cudaStreamSynchronize(). Therefore, we are
# sacrificing some error checking when running with cuda graphs.
# if (
# torch.cuda.is_available()
# and not torch.cuda.is_current_stream_capturing()
# and torch.any(seq_len == 1).item()
# ):
# raise ValueError(
# "normalize_batch with `per_feature` normalize_type "
# "received a tensor of length 1. This will result in "
# "torch.std() returning nan. Make sure your audio length "
# "has enough samples for a single feature (ex. at least "
# "`hop_length` for Mel Spectrograms)."
# )
time_steps = (
torch.arange(max_time, device=x.device)
.unsqueeze(0)
.expand(batch_size, max_time)
)
valid_mask = time_steps < seq_len.unsqueeze(1)
x_mean_numerator = torch.where(valid_mask.unsqueeze(1), x, 0.0).sum(axis=2)
x_mean_denominator = valid_mask.sum(axis=1)
x_mean = x_mean_numerator / x_mean_denominator.unsqueeze(1)
# Subtract 1 in the denominator to correct for the bias.
x_std = torch.sqrt(
torch.sum(
torch.where(valid_mask.unsqueeze(1), x - x_mean.unsqueeze(2), 0.0)
** 2,
axis=2,
)
/ (x_mean_denominator.unsqueeze(1) - 1.0)
)
x_std = x_std.masked_fill(
x_std.isnan(), 0.0
) # edge case: only 1 frame in denominator
# make sure x_std is not zero
x_std += CONSTANT
return (x - x_mean.unsqueeze(2)) / x_std.unsqueeze(2), x_mean, x_std
elif normalize_type == "all_features":
x_mean = torch.zeros(seq_len.shape, dtype=x.dtype, device=x.device)
x_std = torch.zeros(seq_len.shape, dtype=x.dtype, device=x.device)
for i in range(x.shape[0]):
x_mean[i] = x[i, :, : seq_len[i].item()].mean()
x_std[i] = x[i, :, : seq_len[i].item()].std()
# make sure x_std is not zero
x_std += CONSTANT
return (x - x_mean.view(-1, 1, 1)) / x_std.view(-1, 1, 1), x_mean, x_std
elif "fixed_mean" in normalize_type and "fixed_std" in normalize_type:
x_mean = torch.tensor(normalize_type["fixed_mean"], device=x.device)
x_std = torch.tensor(normalize_type["fixed_std"], device=x.device)
return (
(x - x_mean.view(x.shape[0], x.shape[1]).unsqueeze(2))
/ x_std.view(x.shape[0], x.shape[1]).unsqueeze(2),
x_mean,
x_std,
)
else:
return x, x_mean, x_std
@torch.compile
def forward(self, x, seq_len, linear_spec=False):
if x.shape[1] < self.sample_rate * self.pad_min_duration:
pad_amount = int(self.sample_rate * self.pad_min_duration) - x.shape[1]
if self.pad_direction == "right":
x = F.pad(x, (0, pad_amount), value=self.pad_value)
elif self.pad_direction == "left":
x = F.pad(x, (pad_amount, 0), value=self.pad_value)
elif self.pad_direction == "both":
left_pad = pad_amount // 2
right_pad = pad_amount - left_pad
x = F.pad(x, (left_pad, right_pad), value=self.pad_value)
else:
raise ValueError(
f"{self} received an invalid pad_direction: {self.pad_direction}. "
f"It must be one of 'left', 'right', or 'both'."
)
seq_len = torch.tensor([x.shape[1]], dtype=torch.float, device=x.device)
seq_len_time = seq_len
seq_len_unfixed = self.get_seq_len(seq_len)
# fix for seq_len = 0 for streaming; if size was 0, it is always padded
# to 1, and normalizer fails
seq_len = torch.where(
seq_len == 0, torch.zeros_like(seq_len_unfixed), seq_len_unfixed
)
if self.stft_pad_amount is not None:
x = torch.nn.functional.pad(
x.unsqueeze(1), (self.stft_pad_amount, self.stft_pad_amount), "constant"
).squeeze(1)
# use dither for inference as well
if self.dither > 0:
x += self.dither * torch.randn(
x.shape, dtype=x.dtype, device=x.device, generator=self.generator
)
# do preemphasis
if self.preemph is not None:
timemask = torch.arange(x.shape[1], device=x.device).unsqueeze(
0
) < seq_len_time.unsqueeze(1)
x = torch.cat(
(x[:, 0].unsqueeze(1), x[:, 1:] - self.preemph * x[:, :-1]), dim=1
)
x = x.masked_fill(~timemask, 0.0)
x = self.stft(x)
# torch stft returns complex tensor (of shape [B,N,T]); so convert to magnitude
# guard is needed for sqrt if grads are passed through
guard = 0 if not self.use_grads else CONSTANT
x = torch.view_as_real(x)
x = torch.sqrt(x.pow(2).sum(-1) + guard)
# get power spectrum
if self.mag_power != 1.0:
x = x.pow(self.mag_power)
# return plain spectrogram if required
if linear_spec:
return x, seq_len
# disable autocast, otherwise it might be automatically casted to fp16
# on fp16 compatible GPUs and get NaN values for input value of 65520
with torch.amp.autocast(x.device.type, enabled=False):
# dot with filterbank energies
x = torch.matmul(self.fb.to(x.dtype), x)
# log features if required
if self.log:
if self.log_zero_guard_type == "add":
x = torch.log(x + self.log_zero_guard_value_fn(x))
elif self.log_zero_guard_type == "clamp":
x = torch.log(torch.clamp(x, min=self.log_zero_guard_value_fn(x)))
else:
raise ValueError("log_zero_guard_type was not understood")
# frame splicing if required
if self.frame_splicing > 1:
x = self.splice_frames(x, self.frame_splicing)
# normalize if required
if self.normalize:
x, _, _ = self.normalize_batch(x, seq_len, normalize_type=self.normalize)
# mask to zero any values beyond seq_len in batch, pad to multiple of
# `pad_to` (for efficiency)
max_len = x.size(-1)
mask = torch.arange(max_len, device=x.device)
mask = mask.repeat(x.size(0), 1) >= seq_len.unsqueeze(1)
x = x.masked_fill(
mask.unsqueeze(1).type(torch.bool).to(device=x.device), self.pad_value
)
del mask
pad_to = self.pad_to
if pad_to == "max":
x = nn.functional.pad(
x, (0, self.max_length - x.size(-1)), value=self.pad_value
)
elif pad_to > 0:
pad_amt = x.size(-1) % pad_to
if pad_amt != 0:
x = nn.functional.pad(x, (0, pad_to - pad_amt), value=self.pad_value)
return x, seq_len
class CohereASRFeatureExtractor(SequenceFeatureExtractor):
"""HF-compatible feature extractor wrapping FilterbankFeatures."""
model_input_names = ["input_features"]
def __init__(
self,
feature_size=64,
sampling_rate=16000,
padding_value=0.0,
max_duration=30,
n_window_size=320,
n_window_stride=160,
window="hann",
normalize="per_feature",
n_fft=None,
preemph=0.97,
lowfreq=0,
highfreq=None,
log=True,
log_zero_guard_type="add",
log_zero_guard_value=2**-24,
dither=CONSTANT,
pad_to=16,
frame_splicing=1,
exact_pad=False,
mag_power=2.0,
nb_augmentation_prob=0.0,
nb_max_freq=4000,
mel_norm="slaney",
stft_exact_pad=False,
stft_conv=False,
device="cpu",
**kwargs,
):
super().__init__(
feature_size=feature_size,
sampling_rate=sampling_rate,
padding_value=padding_value,
**kwargs,
)
self.max_duration = max_duration
self.hop_length = n_window_stride
self._device = torch.device(device)
self._fb_config = dict(
sample_rate=sampling_rate,
n_window_size=n_window_size,
n_window_stride=n_window_stride,
window=window,
normalize=normalize,
n_fft=n_fft,
preemph=preemph,
nfilt=feature_size,
lowfreq=lowfreq,
highfreq=highfreq,
log=log,
log_zero_guard_type=log_zero_guard_type,
log_zero_guard_value=log_zero_guard_value,
dither=dither,
pad_to=pad_to,
max_duration=max_duration,
frame_splicing=frame_splicing,
exact_pad=exact_pad,
pad_value=padding_value,
mag_power=mag_power,
nb_augmentation_prob=nb_augmentation_prob,
nb_max_freq=nb_max_freq,
mel_norm=mel_norm,
stft_exact_pad=stft_exact_pad,
stft_conv=stft_conv,
device=device,
)
self._filterbank: FilterbankFeatures | None = None
@property
def filterbank(self) -> FilterbankFeatures:
if self._filterbank is None:
fb = FilterbankFeatures(**self._fb_config)
fb.eval()
self._filterbank = fb.to(self._device)
return self._filterbank
def get_seq_len(self, seq_len):
return self.filterbank.get_seq_len(seq_len)
def __call__(
self,
raw_speech,
sampling_rate=None,
return_tensors=None,
**kwargs,
) -> BatchFeature:
if isinstance(raw_speech, np.ndarray):
raw_speech = [raw_speech]
seq_len = torch.tensor([s.shape[0] for s in raw_speech])
max_len = max(s.shape[0] for s in raw_speech)
padded = np.zeros((len(raw_speech), max_len), dtype=np.float32)
for i, s in enumerate(raw_speech):
padded[i, : s.shape[0]] = s
audio_tensor = torch.from_numpy(padded).to(self._device)
seq_len = seq_len.to(self._device)
with torch.no_grad():
input_features, length = self.filterbank(audio_tensor, seq_len)
result = BatchFeature(
{"input_features": input_features.cpu(), "length": length.cpu()}
)
if return_tensors is not None:
result = result.convert_to_tensors(return_tensors)
return result
class CohereASRProcessor(ProcessorMixin):
"""HF-compatible processor combining CohereASRFeatureExtractor and a
tokenizer."""
feature_extractor_class = "CohereASRFeatureExtractor"
tokenizer_class = "AutoTokenizer"
def __init__(self, feature_extractor, tokenizer):
super().__init__(feature_extractor, tokenizer)
def __call__(
self,
text=None,
audio=None,
sampling_rate=None,
return_tensors=None,
**kwargs,
):
if audio is not None:
result = self.feature_extractor(
audio,
sampling_rate=sampling_rate,
return_tensors=return_tensors,
)
else:
result = BatchFeature()
if text is not None:
text_inputs = self.tokenizer(text, return_tensors=return_tensors, **kwargs)
result["input_ids"] = text_inputs["input_ids"]
return result
AutoFeatureExtractor.register("CohereASRFeatureExtractor", CohereASRFeatureExtractor)
AutoProcessor.register("CohereASRProcessor", CohereASRProcessor)
@@ -1,404 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
# ruff: noqa: E501
# coding=utf-8
# adapted from https://github.com/deepseek-ai/DeepSeek-VL2/blob/ff23960c5cf9e6874b44be38af930cfb0ccbb620/deepseek_vl2/models/processing_deepseek_vl_v2.py
# Copyright (c) 2023-2024 DeepSeek.
#
# Permission is hereby granted, free of charge, to any person obtaining a copy of
# this software and associated documentation files (the "Software"), to deal in
# the Software without restriction, including without limitation the rights to
# use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
# the Software, and to permit persons to whom the Software is furnished to do so,
# subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
# FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
# COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
# IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
# CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
import math
from typing import Any
import torch
import torchvision.transforms as T
from PIL import Image, ImageOps
from transformers import BatchFeature, LlamaTokenizerFast
from transformers.processing_utils import ProcessorMixin
class ImageTransform:
def __init__(
self,
mean: tuple[float, float, float] = (0.5, 0.5, 0.5),
std: tuple[float, float, float] = (0.5, 0.5, 0.5),
normalize: bool = True,
):
self.mean = mean
self.std = std
self.normalize = normalize
transform_pipelines = [T.ToTensor()]
if normalize:
transform_pipelines.append(T.Normalize(mean, std))
self.transform = T.Compose(transform_pipelines)
def __call__(self, pil_img: Image.Image):
x = self.transform(pil_img)
return x
class DeepseekVLV2Processor(ProcessorMixin):
tokenizer_class = ("LlamaTokenizer", "LlamaTokenizerFast")
attributes = ["tokenizer"]
def __init__(
self,
tokenizer: LlamaTokenizerFast,
candidate_resolutions: tuple[tuple[int, int]],
patch_size: int,
downsample_ratio: int,
image_mean: tuple[float, float, float] = (0.5, 0.5, 0.5),
image_std: tuple[float, float, float] = (0.5, 0.5, 0.5),
normalize: bool = True,
image_token: str = "<image>",
pad_token: str = "<|▁pad▁|>",
add_special_token: bool = False,
sft_format: str = "deepseek",
mask_prompt: bool = True,
ignore_id: int = -100,
**kwargs,
):
self.candidate_resolutions = candidate_resolutions
self.image_size = candidate_resolutions[0][0]
self.patch_size = patch_size
self.image_mean = image_mean
self.image_std = image_std
self.normalize = normalize
self.downsample_ratio = downsample_ratio
self.image_transform = ImageTransform(
mean=image_mean, std=image_std, normalize=normalize
)
self.tokenizer = tokenizer
self.tokenizer.padding_side = "left" # must set this,padding side with make a difference in batch inference
# add the pad_token as special token to use 'tokenizer.pad_token' and 'tokenizer.pad_token_id'
if tokenizer.pad_token is None:
self.tokenizer.add_special_tokens({"pad_token": pad_token})
# add image token
image_token_id = self.tokenizer.vocab.get(image_token)
if image_token_id is None:
special_tokens = [image_token]
special_tokens_dict = {"additional_special_tokens": special_tokens}
self.tokenizer.add_special_tokens(special_tokens_dict)
self.image_token_id = self.tokenizer.vocab.get(image_token)
# add five special tokens for grounding-related tasks
# <|ref|>, <|/ref|>, <|det|>, <|/det|>, <|grounding|>
special_tokens = ["<|ref|>", "<|/ref|>", "<|det|>", "<|/det|>", "<|grounding|>"]
special_tokens_dict = {"additional_special_tokens": special_tokens}
self.tokenizer.add_special_tokens(special_tokens_dict)
# add special tokens for SFT data
special_tokens = ["<|User|>", "<|Assistant|>"]
special_tokens_dict = {"additional_special_tokens": special_tokens}
self.tokenizer.add_special_tokens(special_tokens_dict)
self.image_token = image_token
self.pad_token = pad_token
self.add_special_token = add_special_token
self.sft_format = sft_format
self.mask_prompt = mask_prompt
self.ignore_id = ignore_id
super().__init__(
tokenizer,
**kwargs,
)
def select_best_resolution(self, image_size):
# used for cropping
original_width, original_height = image_size
best_fit = None
max_effective_resolution = 0
min_wasted_resolution = float("inf")
for width, height in self.candidate_resolutions:
scale = min(width / original_width, height / original_height)
downscaled_width, downscaled_height = (
int(original_width * scale),
int(original_height * scale),
)
effective_resolution = min(
downscaled_width * downscaled_height, original_width * original_height
)
wasted_resolution = (width * height) - effective_resolution
if effective_resolution > max_effective_resolution or (
effective_resolution == max_effective_resolution
and wasted_resolution < min_wasted_resolution
):
max_effective_resolution = effective_resolution
min_wasted_resolution = wasted_resolution
best_fit = (width, height)
return best_fit
@property
def bos_id(self):
return self.tokenizer.bos_token_id
@property
def eos_id(self):
return self.tokenizer.eos_token_id
@property
def pad_id(self):
return self.tokenizer.pad_token_id
def encode(self, text: str, bos: bool = True, eos: bool = False):
t = self.tokenizer.encode(text, add_special_tokens=False)
if bos:
t = [self.bos_id] + t
if eos:
t = t + [self.eos_id]
return t
def decode(self, t: list[int], **kwargs) -> str:
return self.tokenizer.decode(t, **kwargs)
def process_one(
self,
prompt: str,
images: list[Image.Image],
inference_mode: bool = True,
**kwargs: Any,
):
"""Args:
prompt (str): the formatted prompt;
images (list[ImageType]): the list of images;
inference_mode (bool): if True, then remove the last eos token;
**kwargs: Additional keyword arguments.
Returns:
outputs (BaseProcessorOutput): the output of the processor,
- input_ids (torch.LongTensor): [N + image tokens]
- target_ids (torch.LongTensor): [N + image tokens]
- pixel_values (torch.FloatTensor): [n_patches, 3, H, W]
- image_id (int): the id of the image token
- num_image_tokens (list[int]): the number of image tokens
"""
if prompt is None or images is None:
raise ValueError("prompt and images must be used at the same time.")
sft_format = prompt
(
tokenized_str,
images_list,
images_seq_mask,
images_spatial_crop,
num_image_tokens,
) = self.tokenize_with_images(
sft_format, images, bos=True, eos=True, cropping=len(images) <= 2
)
masked_tokenized_str = []
for token_index in tokenized_str:
if token_index != self.image_token_id:
masked_tokenized_str.append(token_index)
else:
masked_tokenized_str.append(self.ignore_id)
assert (
len(tokenized_str) == len(images_seq_mask) == len(masked_tokenized_str)
), (
f"tokenized_str's length {len(tokenized_str)}, input_ids' length {len(masked_tokenized_str)}, "
f"imags_seq_mask's length {len(images_seq_mask)}, are not equal"
)
input_ids = torch.LongTensor(tokenized_str)
target_ids = torch.LongTensor(masked_tokenized_str)
images_seq_mask = torch.tensor(images_seq_mask, dtype=torch.bool)
# set input_ids < 0 | input_ids == self.image_token_id as ignore_id
target_ids[(input_ids < 0) | (input_ids == self.image_token_id)] = (
self.ignore_id
)
input_ids[input_ids < 0] = self.pad_id
if inference_mode:
# Remove the ending eos token
assert input_ids[-1] == self.eos_id
input_ids = input_ids[:-1]
target_ids = target_ids[:-1]
images_seq_mask = images_seq_mask[:-1]
if len(images_list) == 0:
pixel_values = torch.zeros((1, 3, self.image_size, self.image_size))
images_spatial_crop = torch.zeros((1, 2), dtype=torch.long)
else:
pixel_values = torch.stack(images_list, dim=0)
images_spatial_crop = torch.tensor(images_spatial_crop, dtype=torch.long)
input_ids = input_ids.unsqueeze(0)
prepare = BatchFeature(
data=dict(
input_ids=input_ids,
pixel_values=pixel_values,
images_seq_mask=images_seq_mask,
images_spatial_crop=images_spatial_crop,
num_image_tokens=num_image_tokens,
),
tensor_type="pt",
)
return prepare
def __call__(
self,
*,
text: str,
images: list[Image.Image],
inference_mode: bool = True,
**kwargs: Any,
):
"""Args:
text (str): the formatted prompt;
images (list[ImageType]): the list of images;
inference_mode (bool): if True, then remove the last eos token;
**kwargs:
Returns:
outputs (BaseProcessorOutput): the output of the processor,
- input_ids (torch.LongTensor): [N + image tokens]
- images (torch.FloatTensor): [n_images, 3, H, W]
- image_id (int): the id of the image token
- num_image_tokens (list[int]): the number of image tokens
"""
prepare = self.process_one(
prompt=text,
images=images,
inference_mode=inference_mode,
)
return prepare
def tokenize_with_images(
self,
conversation: str,
images: list[Image.Image],
bos: bool = True,
eos: bool = True,
cropping: bool = True,
):
"""Tokenize text with <image> tags."""
num_image_tags = conversation.count(self.image_token)
if num_image_tags != len(images):
raise ValueError(
f"Number of {self.image_token!r} tokens in prompt "
f"({num_image_tags}) does not match number of images "
f"({len(images)})."
)
text_splits = conversation.split(self.image_token)
images_list, images_seq_mask, images_spatial_crop = [], [], []
num_image_tokens = []
tokenized_str = []
for text_sep, image in zip(text_splits, images):
"""encode text_sep"""
tokenized_sep = self.encode(text_sep, bos=False, eos=False)
tokenized_str += tokenized_sep
images_seq_mask += [False] * len(tokenized_sep)
"""select best resolution for anyres"""
if cropping:
best_width, best_height = self.select_best_resolution(image.size)
else:
best_width, best_height = self.image_size, self.image_size
"""process the global view"""
global_view = ImageOps.pad(
image,
(self.image_size, self.image_size),
color=tuple(int(x * 255) for x in self.image_transform.mean),
)
images_list.append(self.image_transform(global_view))
"""process the local views"""
local_view = ImageOps.pad(
image,
(best_width, best_height),
color=tuple(int(x * 255) for x in self.image_transform.mean),
)
for i in range(0, best_height, self.image_size):
for j in range(0, best_width, self.image_size):
images_list.append(
self.image_transform(
local_view.crop(
(j, i, j + self.image_size, i + self.image_size)
)
)
)
"""record height / width crop num"""
num_width_tiles, num_height_tiles = (
best_width // self.image_size,
best_height // self.image_size,
)
images_spatial_crop.append([num_width_tiles, num_height_tiles])
"""add image tokens"""
h = w = math.ceil(
(self.image_size // self.patch_size) / self.downsample_ratio
)
# global views tokens h * (w + 1), 1 is for line separator
tokenized_image = [self.image_token_id] * h * (w + 1)
# add a separator between global and local views
tokenized_image += [self.image_token_id]
# local views tokens, (num_height_tiles * h) * (num_width_tiles * w + 1)
tokenized_image += (
[self.image_token_id]
* (num_height_tiles * h)
* (num_width_tiles * w + 1)
)
tokenized_str += tokenized_image
images_seq_mask += [True] * len(tokenized_image)
num_image_tokens.append(len(tokenized_image))
"""process the last text split"""
tokenized_sep = self.encode(text_splits[-1], bos=False, eos=False)
tokenized_str += tokenized_sep
images_seq_mask += [False] * len(tokenized_sep)
"""add the bos and eos tokens"""
if bos:
tokenized_str = [self.bos_id] + tokenized_str
images_seq_mask = [False] + images_seq_mask
if eos:
tokenized_str = tokenized_str + [self.eos_id]
images_seq_mask = images_seq_mask + [False]
assert len(tokenized_str) == len(images_seq_mask), (
f"tokenize_with_images func: tokenized_str's length {len(tokenized_str)} is not equal to imags_seq_mask's length {len(images_seq_mask)}"
)
return (
tokenized_str,
images_list,
images_seq_mask,
images_spatial_crop,
num_image_tokens,
)
@@ -1,345 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from typing import TYPE_CHECKING
import numpy as np
import torch
import torch.nn.functional as F
from transformers import (
AutoFeatureExtractor,
BatchFeature,
)
from transformers.feature_extraction_sequence_utils import SequenceFeatureExtractor
from transformers.processing_utils import ProcessorMixin
from transformers.utils import TensorType
from vllm.logger import init_logger
from vllm.utils.import_utils import LazyLoader
if TYPE_CHECKING:
import kaldi_native_fbank as knf
else:
knf = LazyLoader("knf", globals(), "kaldi_native_fbank")
logger = init_logger(__name__)
class CMVN:
def __init__(self, dim, means, inverse_std_variences):
self.dim, self.means, self.inverse_std_variences = (
dim,
np.array(means),
np.array(inverse_std_variences),
)
def __call__(self, x):
assert x.shape[-1] == self.dim, "CMVN dim mismatch"
out = x - self.means
out = out * self.inverse_std_variences
return out
class KaldifeatFbank:
def __init__(self, num_mel_bins=80, frame_length=25, frame_shift=10, dither=1.0):
self.dither = dither
opts = knf.FbankOptions()
opts.frame_opts.dither = dither
opts.mel_opts.num_bins = num_mel_bins
opts.frame_opts.snip_edges = True
opts.mel_opts.debug_mel = False
self.opts = opts
def __call__(self, sample_rate, wav_np, is_train=False):
dither = self.dither if is_train else 0.0
self.opts.frame_opts.dither = dither
fbank = knf.OnlineFbank(self.opts)
fbank.accept_waveform(sample_rate, wav_np.tolist())
feat = []
for i in range(fbank.num_frames_ready):
feat.append(fbank.get_frame(i))
if len(feat) == 0:
print("Check data, len(feat) == 0", wav_np, flush=True)
return np.zeros((0, self.opts.mel_opts.num_bins))
feat = np.vstack(feat)
return feat
class FireRedASR2FeatureExtractor(SequenceFeatureExtractor):
r"""Constructs a FireRedASR2 feature extractor.
This feature extractor inherits from [`~feature_extraction_sequence_
utils.SequenceFeatureExtractor`] which contains most of the main
methods. Users should refer to this superclass for more information
regarding those methods.
This class extracts mel-filter bank features from raw speech using a custom
numpy implementation of the `Short Time Fourier Transform` which should
match pytorch's `torch.stft` equivalent.
Args:
feature_size (`int`, *optional*, defaults to 80):
The feature dimension of the extracted features.
sampling_rate (`int`, *optional*, defaults to 16000):
The sampling rate at which the audio files should be digitalized
expressed in hertz (Hz).
chunk_length (`int`, *optional*, defaults to 30):
The maximum number of chunks of `sampling_rate` samples used to
trim and pad longer or shorter audio sequences.
padding_value (`float`, *optional*, defaults to 0.0):
Padding value used to pad the audio. Should correspond to silences.
dither (`float`, *optional*, defaults to 0.0):
Adds dithering. In other words, adds a small Gaussian noise to each frame.
E.g. use 0.0001 to add dithering with a normal distribution centered
around 0.0 with standard deviation 0.0001 (assuming [-1,+1] range
of raw_speech). The value 0.0 means no dithering.
Dithering has similar effect as `spectrogram(mel_floor=...)`. It reduces
the high log_mel_fbank values for signals with hard-zero sections,
when VAD cutoff is present in the signal.
"""
model_input_names = ["input_features"]
def __init__(
self,
feature_size=80,
sampling_rate=16000,
chunk_length=30,
padding_value=0.0,
return_attention_mask=False,
dim=80,
means=None,
inverse_std_variences=None,
num_mel_bins=80,
frame_length=25,
frame_shift=10,
dither=0.0,
max_length=3000,
downsample_rate=2,
left_context=3,
right_context=3,
**kwargs,
):
super().__init__(
feature_size=feature_size,
sampling_rate=sampling_rate,
padding_value=padding_value,
return_attention_mask=return_attention_mask,
**kwargs,
)
self.chunk_length = chunk_length
self.max_length = max_length
self.dim = dim
self.means = means
self.inverse_std_variences = inverse_std_variences
self.num_mel_bins = num_mel_bins
self.frame_length = frame_length
self.frame_shift = frame_shift
self.dither = dither
self.sampling_rate = sampling_rate
self.downsample_rate = downsample_rate
self.context = left_context + 1 + right_context
def __call__(
self,
raw_speech: np.ndarray | list[float] | list[np.ndarray] | list[list[float]],
truncation: bool = True,
pad_to_multiple_of: int | None = None,
return_tensors: str | TensorType | None = None,
return_attention_mask: bool | None = None,
padding: str | None = "max_length",
max_length: int | None = None,
sampling_rate: int | None = None,
do_normalize: bool | None = None,
**kwargs,
) -> BatchFeature:
if sampling_rate != self.sampling_rate:
raise ValueError(
f"The model corresponding to this feature extractor: "
f"{self.__class__.__name__} was trained using a sampling "
f"rate of {self.sampling_rate}. Please make sure that the "
f"provided `raw_speech` input was sampled with "
f"{self.sampling_rate} and not {sampling_rate}."
)
def padding_position_is_0(padded_input, input_lengths):
N, T = padded_input.size()[:2]
mask = torch.ones((N, T)).to(padded_input.device)
for i in range(N):
mask[i, input_lengths[i] :] = 0
mask = mask.unsqueeze(dim=1)
return mask.to(torch.uint8)
# initialize the CMVN and Fbank objects
self.cmvn = CMVN(self.dim, self.means, self.inverse_std_variences)
self.fbank = KaldifeatFbank(
num_mel_bins=self.num_mel_bins,
frame_length=self.frame_length,
frame_shift=self.frame_shift,
dither=self.dither,
)
feats = []
speech_lengths = []
fake_token_lengths = []
for speech in raw_speech:
"""
We must multiply by 32768 here because FireRedASR2 loads audio data
using kaldiio.load_mat, while vLLM loads audio data using pyav.
"""
speech = speech * 32768
fbank = self.fbank(sampling_rate, speech)
fbank = self.cmvn(fbank)
fbank = torch.from_numpy(fbank).float()
length = fbank.size(0)
feats.append(fbank)
speech_lengths.append(length)
padded_input2 = fbank
padded_input2 = F.pad(
padded_input2, (0, 0, 0, self.context - 1), "constant", 0.0
)
src_mask = padding_position_is_0(
padded_input2[None, :, :], torch.tensor([length], dtype=torch.int32)
)
x_mask = src_mask
mask = x_mask[:, :, :-2:2][:, :, :-2:2]
input_lengths = mask[:, -1, :].sum(dim=-1)
input_lengths = input_lengths // self.downsample_rate
fake_token_len = torch.clamp(input_lengths, min=1)
fake_token_lengths.append(fake_token_len)
feats = torch.stack(feats, dim=0)
batched_speech = self.pad(
BatchFeature({"input_features": feats}),
padding=padding,
max_length=max_length if max_length else self.max_length,
truncation=truncation,
pad_to_multiple_of=pad_to_multiple_of,
return_attention_mask=return_attention_mask or do_normalize,
)
if return_tensors is not None:
batched_speech = batched_speech.convert_to_tensors(return_tensors)
batched_speech["speech_lengths"] = torch.tensor(speech_lengths)
batched_speech["fake_token_lengths"] = torch.concat(fake_token_lengths)
return batched_speech
class FireRedASR2Processor(ProcessorMixin):
r"""Constructs a FireRedASR2 processor which wraps a FireRedASR2 feature extractor
and a FireRedASR2 tokenizer into a single processor.
[`FireRedASR2Processor`] offers all the functionalities of
[`FireRedASR2FeatureExtractor`] and [`Qwen2Tokenizer`]. See the
[`~FireRedASR2Processor.__call__`] and [`~FireRedASR2Processor.decode`] for more
information.
Args:
feature_extractor (`FireRedASR2FeatureExtractor`): An instance of
[`FireRedASR2FeatureExtractor`].
The feature extractor is a required input.
tokenizer (`Qwen2Tokenizer`):
An instance of [`Qwen2Tokenizer`]. The tokenizer is a required
input.
"""
feature_extractor_class = "FireRedASR2FeatureExtractor"
tokenizer_class = ("Qwen2Tokenizer", "Qwen2TokenizerFast")
def __init__(
self,
feature_extractor,
tokenizer,
audio_token="<|AUDIO|>",
):
super().__init__(feature_extractor, tokenizer)
self.current_processor = self.feature_extractor
self._in_target_context_manager = False
self.audio_token = (
tokenizer.audio_token if hasattr(tokenizer, "audio_token") else audio_token
)
self.audio_token_id = tokenizer.convert_tokens_to_ids(self.audio_token)
def get_decoder_prompt_ids(self, task=None, language=None, no_timestamps=True):
return self.tokenizer.get_decoder_prompt_ids(
task=task, language=language, no_timestamps=no_timestamps
)
def __call__(self, *args, **kwargs):
"""Forwards the `audio` argument to FireRedASR2FeatureExtractor's
[`~FireRedASR2FeatureExtractor.__call__`] and the `text` argument to
[`~Qwen2Tokenizer.__call__`]. Please refer to the docstring of the
above two methods for more information.
"""
if self._in_target_context_manager:
return self.current_processor(*args, **kwargs)
audio = kwargs.pop("audio", None)
sampling_rate = kwargs.pop("sampling_rate", None)
text = kwargs.pop("text", None)
if len(args) > 0:
audio = args[0]
args = args[1:]
if text is None:
raise ValueError("You need to specify `text` input to process.")
elif isinstance(text, str):
text = [text]
elif not isinstance(text, list) and not isinstance(text[0], str):
raise ValueError(
"Invalid input text. Please provide a string, or a list of strings"
)
if audio is not None:
# ensure we have as much audios as audio tokens
num_audio_tokens = sum(sample.count(self.audio_token) for sample in text)
num_audios = 1 if type(audio) is np.ndarray else len(audio)
if num_audio_tokens != num_audios:
raise ValueError(
f"Found {num_audio_tokens} {self.audio_token} token{'s' if num_audio_tokens > 1 else ''} in provided text but received {num_audios} audio{'s' if num_audios > 1 else ''}" # noqa: E501
)
inputs = self.feature_extractor(
audio, *args, sampling_rate=sampling_rate, **kwargs
)
expanded_text = []
for sample in text:
replace_str = []
while self.audio_token in sample:
num_audio_tokens = int(inputs["fake_token_lengths"].item())
expanded_audio_token = self.audio_token * num_audio_tokens
replace_str.append(expanded_audio_token)
sample = sample.replace(self.audio_token, "<placeholder>", 1)
while "<placeholder>" in sample:
sample = sample.replace("<placeholder>", replace_str.pop(0), 1)
expanded_text.append(sample)
text = expanded_text
if text is not None:
encodings = self.tokenizer(text, **kwargs)
if text is None:
return inputs
elif audio is None:
return encodings
else:
inputs["labels"] = encodings["input_ids"]
return inputs
def get_prompt_ids(self, text: str, return_tensors="np"):
return self.tokenizer.get_prompt_ids(text, return_tensors=return_tensors)
AutoFeatureExtractor.register(
"FireRedASR2FeatureExtractor", FireRedASR2FeatureExtractor
)
@@ -1,429 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import numpy as np
import torch
import torch.nn as nn
import torchaudio.compliance.kaldi as kaldi
from torch.nn.utils.rnn import pad_sequence
from transformers import (
AutoFeatureExtractor,
BatchFeature,
)
from transformers.feature_extraction_sequence_utils import SequenceFeatureExtractor
from transformers.processing_utils import ProcessorMixin
from transformers.utils import TensorType
from vllm.logger import init_logger
logger = init_logger(__name__)
def apply_cmvn(inputs, cmvn): # noqa
"""Apply CMVN with mvn data."""
device = inputs.device
# dtype = inputs.dtype
frame, dim = inputs.shape
means = cmvn[0:1, :dim]
vars = cmvn[1:2, :dim]
inputs += means.to(device)
inputs *= vars.to(device)
return inputs.type(torch.float32)
def apply_lfr(inputs, lfr_m, lfr_n):
# LFR_inputs = []
T = inputs.shape[0]
T_lfr = int(np.ceil(T / lfr_n))
left_padding = inputs[0].repeat((lfr_m - 1) // 2, 1)
inputs = torch.vstack((left_padding, inputs))
T = T + (lfr_m - 1) // 2
feat_dim = inputs.shape[-1]
strides = (lfr_n * feat_dim, 1)
sizes = (T_lfr, lfr_m * feat_dim)
last_idx = (T - lfr_m) // lfr_n + 1
num_padding = lfr_m - (T - last_idx * lfr_n)
if num_padding > 0:
num_padding = (
(2 * lfr_m - 2 * T + (T_lfr - 1 + last_idx) * lfr_n)
/ 2
* (T_lfr - last_idx)
)
inputs = torch.vstack([inputs] + [inputs[-1:]] * int(num_padding))
LFR_outputs = inputs.as_strided(sizes, strides)
return LFR_outputs.clone().type(torch.float32)
def load_cmvn(cmvn_file):
with open(cmvn_file, encoding="utf-8") as f:
lines = f.readlines()
means_list = []
vars_list = []
for i in range(len(lines)):
line_item = lines[i].split()
if line_item[0] == "<AddShift>":
line_item = lines[i + 1].split()
if line_item[0] == "<LearnRateCoef>":
add_shift_line = line_item[3 : (len(line_item) - 1)]
means_list = list(add_shift_line)
continue
elif line_item[0] == "<Rescale>":
line_item = lines[i + 1].split()
if line_item[0] == "<LearnRateCoef>":
rescale_line = line_item[3 : (len(line_item) - 1)]
vars_list = list(rescale_line)
continue
means = np.array(means_list).astype(np.float32)
vars = np.array(vars_list).astype(np.float32)
cmvn = np.array([means, vars])
cmvn = torch.as_tensor(cmvn, dtype=torch.float32)
return cmvn
class WavFrontend(nn.Module):
"""Conventional frontend structure for ASR."""
def __init__(
self,
cmvn_file: str = "null",
fs: int = 16000,
window: str = "hamming",
n_mels: int = 80,
frame_length: int = 25,
frame_shift: int = 10,
filter_length_min: int = -1,
filter_length_max: int = -1,
lfr_m: int = 1,
lfr_n: int = 1,
dither: float = 1.0,
snip_edges: bool = True,
upsacle_samples: bool = True,
**kwargs,
):
super().__init__()
self.fs = fs
self.window = window
self.n_mels = n_mels
self.frame_length = frame_length
self.frame_shift = frame_shift
self.filter_length_min = filter_length_min
self.filter_length_max = filter_length_max
self.lfr_m = lfr_m
self.lfr_n = lfr_n
self.cmvn_file = cmvn_file
self.dither = dither
self.snip_edges = snip_edges
self.upsacle_samples = upsacle_samples
self.cmvn = None if self.cmvn_file is None else load_cmvn(self.cmvn_file)
def output_size(self) -> int:
return self.n_mels * self.lfr_m
def forward(
self,
input: torch.Tensor,
input_lengths,
**kwargs,
) -> tuple[torch.Tensor, torch.Tensor]:
batch_size = input.size(0)
feats = []
feats_lens = []
for i in range(batch_size):
waveform_length = input_lengths[i]
waveform = input[i][:waveform_length]
if self.upsacle_samples:
waveform = waveform * (1 << 15)
waveform = waveform.unsqueeze(0)
mat = kaldi.fbank(
waveform,
num_mel_bins=self.n_mels,
frame_length=min(self.frame_length, waveform_length / self.fs * 1000),
frame_shift=self.frame_shift,
dither=self.dither,
energy_floor=0.0,
window_type=self.window,
sample_frequency=self.fs,
snip_edges=self.snip_edges,
)
if self.lfr_m != 1 or self.lfr_n != 1:
mat = apply_lfr(mat, self.lfr_m, self.lfr_n)
if self.cmvn is not None:
mat = apply_cmvn(mat, self.cmvn)
feat_length = mat.size(0)
feats.append(mat)
feats_lens.append(feat_length)
feats_lens = torch.as_tensor(feats_lens)
if batch_size == 1:
feats_pad = feats[0][None, :, :]
else:
feats_pad = pad_sequence(feats, batch_first=True, padding_value=0.0)
return feats_pad, feats_lens
class FunASRFeatureExtractor(SequenceFeatureExtractor):
r"""Constructs a FunASR feature extractor.
This feature extractor inherits from [`~feature_extraction_sequence_
utils.SequenceFeatureExtractor`] which contains most of the main
methods. Users should refer to this superclass for more information
regarding those methods.
This class extracts mel-filter bank features from raw speech using a custom
numpy implementation of the `Short Time Fourier Transform` which should
match pytorch's `torch.stft` equivalent.
Args:
feature_size (`int`, *optional*, defaults to 80):
The feature dimension of the extracted features.
sampling_rate (`int`, *optional*, defaults to 16000):
The sampling rate at which the audio files should be digitalized
expressed in hertz (Hz).
hop_length (`int`, *optional*, defaults to 160):
Length of the overlapping windows for the STFT used to obtain the
Mel Frequency coefficients.
chunk_length (`int`, *optional*, defaults to 30):
The maximum number of chunks of `sampling_rate` samples used to
trim and pad longer or shorter audio sequences.
n_fft (`int`, *optional*, defaults to 400):
Size of the Fourier transform.
padding_value (`float`, *optional*, defaults to 0.0):
Padding value used to pad the audio. Should correspond to silences.
dither (`float`, *optional*, defaults to 0.0):
Adds dithering. In other words, adds a small Gaussian noise to each frame.
E.g. use 0.0001 to add dithering with a normal distribution centered
around 0.0 with standard deviation 0.0001 (assuming [-1,+1] range
of raw_speech). The value 0.0 means no dithering.
Dithering has similar effect as `spectrogram(mel_floor=...)`. It reduces
the high log_mel_fbank values for signals with hard-zero sections,
when VAD cutoff is present in the signal.
"""
model_input_names = ["input_features"]
def __init__(
self,
feature_size=80,
sampling_rate=16000,
hop_length=160,
chunk_length=30,
n_fft=400,
padding_value=0.0,
dither=0.0,
max_length=1000,
return_attention_mask=False,
**kwargs,
):
super().__init__(
feature_size=feature_size,
sampling_rate=sampling_rate,
padding_value=padding_value,
return_attention_mask=return_attention_mask,
**kwargs,
)
self.frontend_conf = kwargs.get("frontend_conf", {})
self.max_length = max_length
self.n_fft = n_fft
self.hop_length = hop_length
self.chunk_length = chunk_length
self.n_samples = chunk_length * sampling_rate
self.nb_max_frames = self.n_samples // hop_length
self.sampling_rate = sampling_rate
self.dither = dither
def extract_fbank(
self, data, data_len=None, data_type: str = "sound", frontend=None, **kwargs
):
if isinstance(data, np.ndarray):
data = torch.from_numpy(data)
if len(data.shape) < 2:
data = data[None, :] # data: [batch, N]
data_len = [data.shape[1]] if data_len is None else data_len
elif isinstance(data, torch.Tensor):
if len(data.shape) < 2:
data = data[None, :] # data: [batch, N]
data_len = [data.shape[1]] if data_len is None else data_len
elif isinstance(data, (list, tuple)):
data_list, data_len = [], []
for data_i in data:
if isinstance(data_i, np.ndarray):
data_i = torch.from_numpy(data_i)
data_list.append(data_i)
data_len.append(data_i.shape[0])
data = pad_sequence(data_list, batch_first=True)
data, data_len = frontend(data, data_len, **kwargs)
if isinstance(data_len, (list, tuple)):
data_len = torch.tensor([data_len])
return data.to(torch.float32), data_len.to(torch.int32)
def __call__(
self,
raw_speech: np.ndarray | list[float] | list[np.ndarray] | list[list[float]],
truncation: bool = True,
pad_to_multiple_of: int | None = None,
return_tensors: str | TensorType | None = None,
return_attention_mask: bool | None = None,
padding: str | None = "max_length",
max_length: int | None = None,
sampling_rate: int | None = None,
do_normalize: bool | None = None,
device: str | None = "cpu",
return_token_timestamps: bool | None = None,
**kwargs,
) -> BatchFeature:
frontend = WavFrontend(**self.frontend_conf, dither=self.dither)
feats = []
speech_lengths = []
fake_token_lengths = []
for speech in raw_speech:
feature, length = self.extract_fbank(
speech,
data_type=kwargs.get("data_type", "sound"),
frontend=frontend,
is_final=True,
)
feats.append(feature)
speech_lengths.append(length)
olens = 1 + (length - 3 + 2 * 1) // 2
olens = 1 + (olens - 3 + 2 * 1) // 2
fake_token_len = (olens - 1) // 2 + 1
fake_token_len = torch.clamp(fake_token_len, min=1)
fake_token_lengths.append(fake_token_len)
feats = torch.concat(feats, dim=0)
batched_speech = self.pad(
BatchFeature({"input_features": feats}),
padding=padding,
max_length=max_length if max_length else self.max_length,
truncation=truncation,
pad_to_multiple_of=pad_to_multiple_of,
return_attention_mask=return_attention_mask or do_normalize,
)
if return_tensors is not None:
batched_speech = batched_speech.convert_to_tensors(return_tensors)
batched_speech["speech_lengths"] = torch.tensor(speech_lengths)
batched_speech["fake_token_lengths"] = torch.concat(fake_token_lengths)
return batched_speech
class FunASRProcessor(ProcessorMixin):
r"""Constructs a FunASR processor which wraps a FunASR feature extractor and
a FunASR tokenizer into a single processor.
[`FunASRProcessor`] offers all the functionalities of
[`FunASRFeatureExtractor`] and [`Qwen2Tokenizer`]. See the
[`~FunASRProcessor.__call__`] and [`~FunASRProcessor.decode`] for more
information.
Args:
feature_extractor (`FunASRFeatureExtractor`): An instance of
[`FunASRFeatureExtractor`].
The feature extractor is a required input.
tokenizer (`Qwen2Tokenizer`):
An instance of [`Qwen2Tokenizer`]. The tokenizer is a required
input.
"""
feature_extractor_class = "FunASRFeatureExtractor"
tokenizer_class = ("Qwen2Tokenizer", "Qwen2TokenizerFast")
def __init__(
self,
feature_extractor,
tokenizer,
audio_token="<|AUDIO|>",
):
super().__init__(feature_extractor, tokenizer)
self.current_processor = self.feature_extractor
self._in_target_context_manager = False
self.audio_token = (
tokenizer.audio_token if hasattr(tokenizer, "audio_token") else audio_token
)
self.audio_token_id = tokenizer.convert_tokens_to_ids(self.audio_token)
def get_decoder_prompt_ids(self, task=None, language=None, no_timestamps=True):
return self.tokenizer.get_decoder_prompt_ids(
task=task, language=language, no_timestamps=no_timestamps
)
def __call__(self, *args, **kwargs):
"""Forwards the `audio` argument to FunASRFeatureExtractor's
[`~FunASRFeatureExtractor.__call__`] and the `text` argument to
[`~Qwen2Tokenizer.__call__`]. Please refer to the docstring of the
above two methods for more information.
"""
if self._in_target_context_manager:
return self.current_processor(*args, **kwargs)
audio = kwargs.pop("audio", None)
sampling_rate = kwargs.pop("sampling_rate", None)
text = kwargs.pop("text", None)
if len(args) > 0:
audio = args[0]
args = args[1:]
if text is None:
raise ValueError("You need to specify `text` input to process.")
elif isinstance(text, str):
text = [text]
elif not isinstance(text, list) and not isinstance(text[0], str):
raise ValueError(
"Invalid input text. Please provide a string, or a list of strings"
)
if audio is not None:
# ensure we have as much audios as audio tokens
num_audio_tokens = sum(sample.count(self.audio_token) for sample in text)
num_audios = 1 if type(audio) is np.ndarray else len(audio)
if num_audio_tokens != num_audios:
raise ValueError(
f"Found {num_audio_tokens} {self.audio_token} token{'s' if num_audio_tokens > 1 else ''} in provided text but received {num_audios} audio{'s' if num_audios > 1 else ''}" # noqa: E501
)
inputs = self.feature_extractor(
audio, *args, sampling_rate=sampling_rate, **kwargs
)
expanded_text = []
for sample in text:
replace_str = []
while self.audio_token in sample:
num_audio_tokens = inputs["fake_token_lengths"].item()
expanded_audio_token = self.audio_token * num_audio_tokens
replace_str.append(expanded_audio_token)
sample = sample.replace(self.audio_token, "<placeholder>", 1)
while "<placeholder>" in sample:
sample = sample.replace("<placeholder>", replace_str.pop(0), 1)
expanded_text.append(sample)
text = expanded_text
if text is not None:
encodings = self.tokenizer(text, **kwargs)
if text is None:
return inputs
elif audio is None:
return encodings
else:
inputs["labels"] = encodings["input_ids"]
return inputs
def get_prompt_ids(self, text: str, return_tensors="np"):
return self.tokenizer.get_prompt_ids(text, return_tensors=return_tensors)
AutoFeatureExtractor.register("FunASRFeatureExtractor", FunASRFeatureExtractor)
@@ -1,35 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
# Adapted from
# https://github.com/zai-org/CogAgent
from transformers import PythonBackend
from transformers.image_processing_backends import TorchvisionBackend
from transformers.image_utils import PILImageResampling
from transformers.processing_utils import ProcessorMixin
class GLM4VImageProcessorFast(TorchvisionBackend):
"""Port of https://huggingface.co/zai-org/glm-4v-9b/blob/main/tokenization_chatglm.py#L177
to HF Transformers.
"""
resample = PILImageResampling.BICUBIC
image_mean = [0.48145466, 0.4578275, 0.40821073]
image_std = [0.26862954, 0.26130258, 0.27577711]
size = {"height": 1120, "width": 1120}
do_resize = True
do_rescale = True
do_normalize = True
class GLM4VProcessor(ProcessorMixin):
attributes = ["image_processor", "tokenizer"]
def __init__(
self,
image_processor: GLM4VImageProcessorFast,
tokenizer: PythonBackend,
) -> None:
self.image_processor = image_processor
self.tokenizer = tokenizer
-387
View File
@@ -1,387 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
# adapted from https://huggingface.co/h2oai/h2ovl-mississippi-2b/blob/main/modeling_h2ovl_chat.py
# https://huggingface.co/h2oai/h2ovl-mississippi-2b/blob/main/image_process.py
# --------------------------------------------------------
# H2OVL-Mississippi
# Copyright (c) 2024 H2O.AI
# Licensed under Apache 2.0 License [see LICENSE for details]
# --------------------------------------------------------
import torch
from PIL import Image
from vllm.tokenizers.hf import HfTokenizer
from .internvl import (
InternVLImageProcessor,
InternVLProcessor,
build_transform,
find_closest_aspect_ratio,
get_internvl_target_ratios,
)
def resolve_h2ovl_min_max_num(
*,
min_dynamic_patch: int,
max_dynamic_patch: int,
dynamic_image_size: bool,
use_thumbnail: bool,
) -> tuple[int, int]:
min_dynamic_patch = min_dynamic_patch if dynamic_image_size else 1
max_dynamic_patch = max_dynamic_patch if dynamic_image_size else 1
if use_thumbnail and max_dynamic_patch != 1:
max_dynamic_patch += 1
return min_dynamic_patch, max_dynamic_patch
def get_h2ovl_target_ratios(
min_num: int,
max_num: int,
*,
prior_aspect_ratio: tuple[int, int] | None,
) -> list[tuple[int, int]]:
target_ratios = get_internvl_target_ratios(min_num, max_num)
# if prior_aspect_ratio is provided, filter the target ratios
if prior_aspect_ratio is not None:
target_ratios = [
ratio
for ratio in target_ratios
if prior_aspect_ratio[0] % ratio[0] != 0
and prior_aspect_ratio[1] % ratio[1] != 0
]
return target_ratios
# modified to include blocks generated in second pass
def calculate_h2ovl_targets(
*,
orig_width: int,
orig_height: int,
target_ratios: list[tuple[int, int]],
image_size: int,
use_thumbnail: bool,
) -> tuple[int, int, int, tuple[int, int]]:
aspect_ratio = orig_width / orig_height
# find the closest aspect ratio to the target
target_aspect_ratio = find_closest_aspect_ratio(
aspect_ratio,
target_ratios,
width=orig_width,
height=orig_height,
image_size=image_size,
)
# calculate the target width and height
target_width = image_size * target_aspect_ratio[0]
target_height = image_size * target_aspect_ratio[1]
blocks = target_aspect_ratio[0] * target_aspect_ratio[1]
# add thumbnail image if num_blocks != 1
if use_thumbnail and blocks != 1:
blocks += 1
return blocks, target_width, target_height, target_aspect_ratio
# adapted from https://huggingface.co/OpenGVLab/InternVL2-1B
# refactored to handle prior_aspect_ratio
def dynamic_preprocess_h2ovl(
image: Image.Image,
*,
target_ratios: list[tuple[int, int]],
image_size: int,
use_thumbnail: bool,
) -> tuple[list[Image.Image], tuple[int, int]]:
orig_width, orig_height = image.size
# calculate the number of blocks without thumbnail
(
blocks,
target_width,
target_height,
target_aspect_ratio,
) = calculate_h2ovl_targets(
orig_width=orig_width,
orig_height=orig_height,
target_ratios=target_ratios,
image_size=image_size,
use_thumbnail=False,
)
# resize the image
resized_img = image.resize((target_width, target_height))
processed_images = []
for i in range(blocks):
box = (
(i % (target_width // image_size)) * image_size,
(i // (target_width // image_size)) * image_size,
((i % (target_width // image_size)) + 1) * image_size,
((i // (target_width // image_size)) + 1) * image_size,
)
# split the image
split_img = resized_img.crop(box)
processed_images.append(split_img)
assert len(processed_images) == blocks
if use_thumbnail and len(processed_images) != 1:
thumbnail_img = image.resize((image_size, image_size))
processed_images.append(thumbnail_img)
return processed_images, target_aspect_ratio
def _preprocess_image(
image: Image.Image,
*,
input_size: int,
min_num: int,
max_num: int,
use_thumbnail: bool,
prior_aspect_ratio: tuple[int, int] | None,
) -> tuple[torch.Tensor, tuple[int, int]]:
target_ratios = get_h2ovl_target_ratios(
min_num,
max_num,
prior_aspect_ratio=prior_aspect_ratio,
)
transform = build_transform(input_size=input_size)
images, target_aspect_ratio = dynamic_preprocess_h2ovl(
image,
image_size=input_size,
use_thumbnail=use_thumbnail,
target_ratios=target_ratios,
)
pixel_values = torch.stack([transform(image) for image in images])
return pixel_values, target_aspect_ratio
# refactored to use the _preprocess_image function
def image_to_pixel_values_h2ovl(
image: Image.Image,
*,
input_size: int,
min_num: int,
max_num: int,
use_thumbnail: bool,
use_msac: bool,
) -> torch.Tensor:
# when MSAC is turned on, we need to process the image twice
if use_msac:
# first pass
pixel_values1, aspect_ratio1 = _preprocess_image(
image,
input_size=input_size,
min_num=1,
max_num=max_num,
use_thumbnail=True,
prior_aspect_ratio=None,
)
# second pass
pixel_values2, _ = _preprocess_image(
image,
input_size=input_size,
min_num=3,
max_num=max_num,
use_thumbnail=True,
prior_aspect_ratio=aspect_ratio1,
)
# combine pixel values
pixel_values = torch.cat(
[pixel_values2[:-1], pixel_values1[:-1], pixel_values2[-1:]], 0
)
else:
pixel_values, _ = _preprocess_image(
image,
input_size=input_size,
min_num=min_num,
max_num=max_num,
use_thumbnail=use_thumbnail,
prior_aspect_ratio=None,
)
return pixel_values
class H2OVLImageProcessor(InternVLImageProcessor):
def __init__(
self,
image_size: int,
min_dynamic_patch: int,
max_dynamic_patch: int,
dynamic_image_size: bool,
use_thumbnail: bool,
use_msac: bool,
) -> None:
super().__init__(
image_size=image_size,
min_dynamic_patch=min_dynamic_patch,
max_dynamic_patch=max_dynamic_patch,
dynamic_image_size=dynamic_image_size,
use_thumbnail=use_thumbnail,
)
self.use_msac = use_msac
def resolve_min_max_num(
self,
*,
min_dynamic_patch: int | None = None,
max_dynamic_patch: int | None = None,
dynamic_image_size: bool | None = None,
use_thumbnail: bool | None = None,
) -> tuple[int, int]:
if min_dynamic_patch is None:
min_dynamic_patch = self.min_dynamic_patch
if max_dynamic_patch is None:
max_dynamic_patch = self.max_dynamic_patch
if dynamic_image_size is None:
dynamic_image_size = self.dynamic_image_size
if use_thumbnail is None:
use_thumbnail = self.use_thumbnail
return resolve_h2ovl_min_max_num(
min_dynamic_patch=min_dynamic_patch,
max_dynamic_patch=max_dynamic_patch,
dynamic_image_size=dynamic_image_size,
use_thumbnail=use_thumbnail,
)
def _images_to_pixel_values_lst(
self,
images: list[Image.Image],
min_dynamic_patch: int | None = None,
max_dynamic_patch: int | None = None,
dynamic_image_size: bool | None = None,
) -> list[torch.Tensor]:
use_msac = self.use_msac if len(images) == 1 else False
min_num, max_num = self.resolve_min_max_num(
min_dynamic_patch=min_dynamic_patch,
max_dynamic_patch=max_dynamic_patch,
dynamic_image_size=dynamic_image_size,
use_thumbnail=False, # Applied in image_to_pixel_values
)
return [
image_to_pixel_values_h2ovl(
image,
input_size=self.image_size,
min_num=min_num,
max_num=max_num,
use_thumbnail=self.use_thumbnail,
use_msac=use_msac,
)
for image in images
]
class H2OVLProcessor(InternVLProcessor):
def __init__(
self,
image_processor: H2OVLImageProcessor,
tokenizer: HfTokenizer,
*,
image_seq_length: int,
start_image_token: str = "<img>",
end_image_token: str = "</img>",
ctx_image_token: str = "<IMG_CONTEXT>",
) -> None:
super().__init__(
image_processor=image_processor,
tokenizer=tokenizer,
image_seq_length=image_seq_length,
start_image_token=start_image_token,
end_image_token=end_image_token,
ctx_image_token=ctx_image_token,
)
self.image_processor: H2OVLImageProcessor
def resolve_target_ratios(
self,
*,
min_dynamic_patch: int | None = None,
max_dynamic_patch: int | None = None,
dynamic_image_size: bool | None = None,
use_thumbnail: bool | None = None,
prior_aspect_ratio: tuple[int, int] | None = None,
override_min_num: int | None = None,
) -> list[tuple[int, int]]:
min_num, max_num = self.image_processor.resolve_min_max_num(
min_dynamic_patch=min_dynamic_patch,
max_dynamic_patch=max_dynamic_patch,
dynamic_image_size=dynamic_image_size,
use_thumbnail=use_thumbnail,
)
if override_min_num is not None:
min_num = override_min_num
return get_h2ovl_target_ratios(
min_num,
max_num,
prior_aspect_ratio=prior_aspect_ratio,
)
def get_num_image_tokens(
self,
*,
image_width: int,
image_height: int,
use_msac: bool | None = None,
) -> int:
image_processor = self.image_processor
use_msac = image_processor.use_msac if use_msac is None else use_msac
use_thumbnail = image_processor.use_thumbnail
if use_msac:
target_ratios_1 = self.resolve_target_ratios(
use_thumbnail=False, # Applied in calculate_targets
override_min_num=1,
)
num_patches_1, _, _, aspect_ratio_1 = calculate_h2ovl_targets(
orig_width=image_width,
orig_height=image_height,
image_size=image_processor.image_size,
target_ratios=target_ratios_1,
use_thumbnail=True,
)
target_ratios_2 = self.resolve_target_ratios(
use_thumbnail=False, # Applied in calculate_targets
prior_aspect_ratio=aspect_ratio_1,
override_min_num=3,
)
num_patches_2, _, _, _ = calculate_h2ovl_targets(
orig_width=image_width,
orig_height=image_height,
image_size=image_processor.image_size,
target_ratios=target_ratios_2,
use_thumbnail=True,
)
num_patches = num_patches_1 + num_patches_2 - 1
else:
target_ratios = self.resolve_target_ratios(
use_thumbnail=False, # Applied in calculate_targets
)
num_patches, _, _, _ = calculate_h2ovl_targets(
orig_width=image_width,
orig_height=image_height,
image_size=image_processor.image_size,
target_ratios=target_ratios,
use_thumbnail=use_thumbnail,
)
return num_patches * self.image_seq_length
-509
View File
@@ -1,509 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import math
from typing import Any, TypedDict
import numpy as np
import torch
import torch.nn.functional as F
from PIL import Image
from transformers import (
BatchFeature,
ImageProcessingMixin,
ProcessorMixin,
TensorType,
)
from transformers.processing_utils import ProcessingKwargs
from typing_extensions import Unpack
from vllm.tokenizers.hf import HfTokenizer
MAX_PIXELS = 60_000_000 # 60-megapixel ceiling ≈ 8200 × 7300 px
# Vision preprocessing constants
VISION_MEAN = (0.5, 0.5, 0.5)
VISION_STD = (0.5, 0.5, 0.5)
VISION_SCALE = 1 / 255
def _make_writeable(arr: np.ndarray) -> np.ndarray:
"""Return *arr* itself if it is already writeable, otherwise try to flip the
write flag in-place and finally fall back to `arr.copy()`.
This guarantees the buffer handed to `torch.from_numpy()` is always
writeable, silencing the PyTorch warning about undefined behaviour.
"""
if arr.flags.writeable:
return arr
# First, try the cheap path — in-place flag toggle (works for mmap'd arrays
# and some shared memory buffers):
try:
arr.setflags(write=True)
return arr # success: no data copy
except ValueError:
# Buffer is inherently read-only (e.g. backed by PyAV / PIL): make copy
return arr.copy()
def extract_image_pil(image: Image.Image) -> torch.Tensor:
if image.width * image.height > MAX_PIXELS:
raise ValueError(
f"Image (w={image.width}, h={image.height}) > MAX=`{MAX_PIXELS}`"
)
img = image if image.mode == "RGB" else image.convert("RGB")
arr = np.asarray(img)
arr = _make_writeable(arr)
return torch.from_numpy(arr)
def get_image_size_for_max_num_patches(
image_height: int,
image_width: int,
patch_size: int,
max_num_patches: int,
min_num_patches: int | None = None,
eps: float = 1e-5,
pixel_shuffle_scale: int = 1,
) -> tuple[int, int]:
r"""Compute a target resolution whose patch grid satisfies patching parametrization.
Args:
image_height (`int`):
Height in pixels of the source image prior to any resizing.
image_width (`int`):
Width in pixels of the source image prior to any resizing.
patch_size (`int`):
Size of the square patch used by the vision encoder.
max_num_patches (`int`):
Upper bound on `(height / patch_size) * (width / patch_size)` after
resizing.
min_num_patches (`int`, *optional*):
Lower bound on the number of patches. When provided the image will
be scaled up if necessary.
eps (`float`, *optional*, defaults to 1e-5):
Convergence tolerance for the internal binary search to determine
the target dimensions.
pixel_shuffle_scale (`int`, *optional*, defaults to 1):
Additional stride multiplier applied when pixel shuffle later
reduces spatial resolution.
Returns:
`tuple[int, int]`: Height and width (in pixels) that are multiples of
`patch_size * pixel_shuffle_scale` and respect both the maximum and
optional minimum patch-count constraints.
"""
def get_scaled_image_size(scale, original_size, patch_size, pixel_shuffle_scale):
scaled_size = scale * original_size
divisor = patch_size * pixel_shuffle_scale
scaled_size = math.ceil(scaled_size / divisor) * divisor
scaled_size = max(divisor, scaled_size)
return int(scaled_size)
# Ensure divisibility
divisor = patch_size * pixel_shuffle_scale
adjusted_height = math.ceil(image_height / divisor) * divisor
adjusted_height = max(divisor, adjusted_height)
adjusted_width = math.ceil(image_width / divisor) * divisor
adjusted_width = max(divisor, adjusted_width)
num_patches = (adjusted_height / patch_size) * (adjusted_width / patch_size)
if min_num_patches is not None and num_patches < min_num_patches:
# Scale up
scale_min, scale_max = 1.0, 100.0
while (scale_max - scale_min) >= eps:
scale = (scale_min + scale_max) / 2
target_height = get_scaled_image_size(
scale, image_height, patch_size, pixel_shuffle_scale
)
target_width = get_scaled_image_size(
scale, image_width, patch_size, pixel_shuffle_scale
)
num_patches = (target_height / patch_size) * (target_width / patch_size)
if num_patches >= min_num_patches:
scale_max = scale
else:
scale_min = scale
scale = scale_max
target_height = get_scaled_image_size(
scale, image_height, patch_size, pixel_shuffle_scale
)
target_width = get_scaled_image_size(
scale, image_width, patch_size, pixel_shuffle_scale
)
return target_height, target_width
elif num_patches <= max_num_patches:
return adjusted_height, adjusted_width
else:
# Scale down
scale_min, scale_max = eps / 10, 1.0
while (scale_max - scale_min) >= eps:
scale = (scale_min + scale_max) / 2
target_height = get_scaled_image_size(
scale, image_height, patch_size, pixel_shuffle_scale
)
target_width = get_scaled_image_size(
scale, image_width, patch_size, pixel_shuffle_scale
)
num_patches = (target_height / patch_size) * (target_width / patch_size)
if num_patches <= max_num_patches:
scale_min = scale
else:
scale_max = scale
scale = scale_min
target_height = get_scaled_image_size(
scale, image_height, patch_size, pixel_shuffle_scale
)
target_width = get_scaled_image_size(
scale, image_width, patch_size, pixel_shuffle_scale
)
return target_height, target_width
_MEAN_TENSOR = torch.tensor(VISION_MEAN, dtype=torch.float32).view(1, 1, 1, -1)
_STD_TENSOR = torch.tensor(VISION_STD, dtype=torch.float32).view(1, 1, 1, -1)
def prepare_image_tensor(
image: torch.Tensor,
scale: float = VISION_SCALE,
) -> torch.Tensor:
r"""Standardize RGB images prior to patch extraction via rescaling and whitening.
Args:
image (`torch.Tensor`):
Tensor with shape `(..., height, width, 3)` containing RGB values.
The tensor is converted to floating point if needed.
scale (`float`, *optional*, defaults to `VISION_SCALE`):
Scalar multiplier applied before normalization.
Returns:
`torch.Tensor`: Normalized tensor with the same shape as the input and
dtype `torch.float32`.
"""
if not torch.is_floating_point(image):
image = image.float()
rescaled = image * scale
# Use precomputed tensors and move to the correct device if needed
mean_tensor = _MEAN_TENSOR.to(image.device)
std_tensor = _STD_TENSOR.to(image.device)
normalized = (rescaled - mean_tensor) / std_tensor
return normalized
def patchify_vision(image: torch.Tensor, patch_size: int) -> torch.Tensor:
r"""Convert normalized images into flattened ViT-style patches.
Args:
image (`torch.Tensor`):
Tensor of shape `(num_images, height, width, channels)`.
patch_size (`int`):
Edge length of the square patches
Returns:
`torch.Tensor`:
Patch tensor where each position stores the flattened pixels
belonging to that patch.
Raises:
ValueError: If `height` or `width` is not divisible by `patch_size`.
"""
num_images, height, width, channels = image.shape
if height % patch_size or width % patch_size:
raise ValueError(
"Dimensions of images "
f"{image.shape} are not divisible by patch_size={patch_size}."
)
patches = image.reshape(
num_images,
height // patch_size,
patch_size,
width // patch_size,
patch_size,
channels,
)
patches = patches.permute(0, 1, 3, 2, 4, 5)
patches = patches.reshape(
num_images,
height // patch_size,
width // patch_size,
channels * patch_size * patch_size,
)
return patches
def process_vision_for_patches(
images: torch.Tensor,
patch_size: int,
max_num_patches: int,
min_num_patches: int | None = None,
pixel_shuffle_scale: int = 1,
) -> tuple[torch.Tensor, list[int]]:
r"""Resize, normalize, and patchify RGB images for the vision encoder.
Args:
images (`torch.Tensor`):
Either `(height, width, channels)` for a single image or
`(num_images, height, width, channels)` for a batch. Channels are
expected to be RGB.
patch_size (`int`):
Edge length of square patches; implicitly controls resize grid granularity.
max_num_patches (`int`):
Maximum number of patches allowed after resizing.
min_num_patches (`int`, *optional*):
Minimum number of patches. If provided, the routine upsamples images
as needed to satisfy the lower bound.
pixel_shuffle_scale (`int`, *optional*, defaults to 1):
Pixel shuffle scale factor; influences the target grid that the
function produces.
Returns:
`tuple[torch.Tensor, list[int]]`: A pair `(patches, dims_virtual)`
where `patches` has shape `(num_images, target_h / patch_size, target_w
/ patch_size, channels * patch_size**2)` and `dims_virtual` encodes
effective `(images, height, width)` dimensions after optional pixel
shuffling.
"""
# Add batch dim if single image
if images.dim() == 3:
images = images.unsqueeze(0)
# Permute to channel first for resize
images = images.permute(0, 3, 1, 2)
# Get target dimensions
_, _, orig_height, orig_width = images.shape
target_height, target_width = get_image_size_for_max_num_patches(
orig_height,
orig_width,
patch_size,
max_num_patches,
min_num_patches=min_num_patches,
pixel_shuffle_scale=pixel_shuffle_scale,
)
# Resize
images = F.interpolate(
images,
size=(target_height, target_width),
mode="bilinear",
align_corners=False,
)
# Back to channel last
images = images.permute(0, 2, 3, 1)
# Normalize
images = prepare_image_tensor(images)
# Patchify
patches = patchify_vision(images, patch_size=patch_size)
# Calculate dimensions for the patches
n_images, h_patches, w_patches, _ = patches.shape
dims_virtual = (
[1, h_patches, w_patches]
if pixel_shuffle_scale == 1
else [1, h_patches // pixel_shuffle_scale, w_patches // pixel_shuffle_scale]
)
return patches, dims_virtual
class IsaacImagesKwargs(TypedDict, total=False):
patch_size: int
max_num_patches: int
min_num_patches: int
pixel_shuffle_scale: int
class IsaacProcessorKwargs(ProcessingKwargs, total=False): # type: ignore[call-arg]
images_kwargs: IsaacImagesKwargs
_defaults = {
"text_kwargs": {"padding": False},
"images_kwargs": {},
}
class IsaacImageProcessor(ImageProcessingMixin):
model_input_names = ["pixel_values", "image_grid_thw"]
def __init__(
self,
patch_size: int = 16,
vision_max_num_patches: int = 6144,
vision_min_num_patches: int = 256,
pixel_shuffle_scale: int = 2,
) -> None:
self.patch_size = patch_size
self.vision_max_num_patches = vision_max_num_patches
self.vision_min_num_patches = vision_min_num_patches
self.pixel_shuffle_scale = pixel_shuffle_scale
def __call__(
self,
images: Image.Image | list[Image.Image],
return_tensors: str | TensorType | None = None,
**kwargs: Unpack[IsaacImagesKwargs],
) -> BatchFeature:
"""Preprocess images into format compatible with vLLM input processing."""
if not isinstance(images, list):
images = [images]
all_pixel_values: list[torch.Tensor] = []
all_image_grids: list[torch.Tensor] = []
for image in images:
image_tensor = extract_image_pil(image)
patches, dims_virtual = process_vision_for_patches(
image_tensor,
patch_size=kwargs.get("patch_size", self.patch_size),
max_num_patches=kwargs.get(
"max_num_patches", self.vision_max_num_patches
),
min_num_patches=kwargs.get(
"min_num_patches", self.vision_min_num_patches
),
pixel_shuffle_scale=kwargs.get(
"pixel_shuffle_scale", self.pixel_shuffle_scale
),
)
# Isaac packs a dummy temporal dim for images
patches = patches.unsqueeze(1) # [N, T=1, Hp, Wp, D]
hp, wp, dim = patches.shape[-3], patches.shape[-2], patches.shape[-1]
current_num_patches = hp * wp
pixel_values = patches.reshape(current_num_patches, dim) # [N_tokens, D]
# Use real patch dimensions for image_grid_thw, not virtual dimensions
# This ensures the vision model receives correct grid info for pixel shuffle
dims_real = [1, hp, wp] # Real patch dimensions
image_grid_thw = torch.tensor(dims_real).unsqueeze(0)
all_pixel_values.append(pixel_values)
all_image_grids.append(image_grid_thw)
if all_pixel_values:
final_pixel_values = torch.cat(all_pixel_values, dim=0)
final_image_grids = torch.cat(all_image_grids, dim=0)
else:
final_pixel_values = torch.empty(0, 0)
final_image_grids = torch.empty(0, 3)
return BatchFeature(
data={
"pixel_values": final_pixel_values,
"image_grid_thw": final_image_grids,
},
tensor_type=return_tensors,
)
class IsaacProcessor(ProcessorMixin):
attributes = ["image_processor", "tokenizer"]
def __init__(
self,
image_processor: IsaacImageProcessor,
tokenizer: HfTokenizer,
image_token: str = "<image>",
):
self.image_processor = image_processor
self.tokenizer = tokenizer
self.image_token = image_token
def __call__(
self,
text: str | list[str] | None = None,
images: Image.Image | list[Image.Image] | None = None,
return_tensors: str | TensorType | None = None,
**kwargs: Unpack[IsaacProcessorKwargs], # type: ignore[misc]
) -> BatchFeature:
output_kwargs = self._merge_kwargs(
IsaacProcessorKwargs,
tokenizer_init_kwargs=self.tokenizer.init_kwargs,
**kwargs,
)
if images is not None:
image_inputs = self.image_processor(
images, **output_kwargs["images_kwargs"]
)
image_grid_thw = image_inputs["image_grid_thw"]
else:
image_inputs = {}
image_grid_thw = []
if text is not None:
if not isinstance(text, list):
text = [text]
if image_inputs:
text = text.copy() # below lines change text in-place
merge_length = self.image_processor.pixel_shuffle_scale**2
index = 0
for i in range(len(text)):
while self.image_token in text[i]:
num_image_tokens = image_grid_thw[index].prod() // merge_length
text[i] = text[i].replace(
self.image_token, "<|placeholder|>" * num_image_tokens, 1
)
index += 1
text[i] = text[i].replace("<|placeholder|>", "<|image_pad|>")
text_inputs = self.tokenizer(text, **output_kwargs["text_kwargs"])
else:
text_inputs = {}
return BatchFeature(
data={**text_inputs, **image_inputs},
tensor_type=return_tensors,
)
def apply_chat_template(
self,
messages: list[dict[str, Any]],
tokenize: bool = False,
add_generation_prompt: bool = False,
**kwargs,
) -> Any:
# Convert mixed content messages to simple text format
processed_messages = []
for message in messages:
if "content" in message and isinstance(message["content"], list):
# Handle mixed content (text + image)
text_parts = []
for content_item in message["content"]:
if content_item.get("type") == "text":
text_parts.append(content_item.get("text", ""))
elif content_item.get("type") == "image":
# Replace image with vision token
text_parts.append(self.image_token)
processed_message = {
"role": message.get("role", "user"),
"content": "".join(text_parts),
}
processed_messages.append(processed_message)
else:
# Regular text message
processed_messages.append(message)
kwargs["return_dict"] = False
return self.tokenizer.apply_chat_template(
processed_messages,
tokenize=tokenize,
add_generation_prompt=add_generation_prompt,
**kwargs,
)
@@ -1,105 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
# Copyright 2026 The Moonshot AI team and the HuggingFace Inc. team.
# All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Processor for Kimi-Audio ASR model."""
import numpy as np
from transformers import BatchFeature, ProcessorMixin
from transformers.audio_utils import AudioInput
from transformers.tokenization_utils_base import PreTokenizedInput, TextInput
class KimiAudioProcessor(ProcessorMixin):
# Required for ProcessorMixin
attributes = ["feature_extractor", "tokenizer"]
feature_extractor_class = "AutoFeatureExtractor"
tokenizer_class = "AutoTokenizer"
# Special token IDs
KIMIA_MEDIA_BEGIN: int = 151661
KIMIA_MEDIA_END: int = 151663
KIMIA_TEXT_BLANK: int = 151666
# Audio processing constants
AUDIO_SEQ_LEN: int = 376
def __init__(self, feature_extractor=None, tokenizer=None, **kwargs):
self.feature_extractor = feature_extractor
self.tokenizer = tokenizer
def __call__(
self,
text: TextInput
| PreTokenizedInput
| list[TextInput]
| list[PreTokenizedInput]
| None = None,
audio: AudioInput | None = None,
return_tensors: str = "pt",
**kwargs,
) -> BatchFeature:
if text is not None:
if not isinstance(text, list):
text = [text]
text_inputs = self.tokenizer(
text, return_tensors=return_tensors, padding=True
)
else:
text_inputs = {}
if audio is not None:
# Ensure audio is a list
if isinstance(audio, np.ndarray):
audio = [audio]
# Pad audio to hop length (required by WhisperFeatureExtractor)
hop_length = self.feature_extractor.hop_length
padded_audio = []
for aud in audio:
length = aud.shape[-1]
if length % hop_length != 0:
pad_length = hop_length - (length % hop_length)
aud = np.pad(
aud, (0, pad_length), mode="constant", constant_values=0
)
padded_audio.append(aud)
# Use feature_extractor directly like Qwen3ASR does
audio_inputs = self.feature_extractor(
padded_audio,
sampling_rate=16000,
padding=True,
return_attention_mask=True,
return_tensors=return_tensors,
)
# Rename to match Kimi-Audio expectations
if "input_features" in audio_inputs:
audio_inputs["whisper_input_features"] = audio_inputs.pop(
"input_features"
)
if "attention_mask" in audio_inputs:
audio_inputs["feature_attention_mask"] = audio_inputs.pop(
"attention_mask"
)
else:
audio_inputs = {}
return BatchFeature(
data={**text_inputs, **audio_inputs},
tensor_type=return_tensors,
)
@@ -1,92 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from transformers import BaseImageProcessor, BatchFeature, TensorType
from transformers.processing_utils import ProcessorMixin
from vllm.multimodal.inputs import VisionChunk
from vllm.tokenizers.hf import HfTokenizer
class KimiK25Processor(ProcessorMixin):
attributes = ["image_processor", "tokenizer"]
def __init__(
self,
image_processor: BaseImageProcessor,
tokenizer: HfTokenizer,
media_token_id: int,
) -> None:
self.image_processor = image_processor
self.tokenizer = tokenizer
self.media_token_id = media_token_id
def __call__(
self,
text: str | list[str] | None = None,
vision_chunks: list[VisionChunk] | None = None,
return_tensors: str | TensorType | None = None,
do_rescale: bool = True,
do_normalize: bool = True,
**kwargs,
) -> BatchFeature:
"""Args:
text: The text to be field to the model.
vision_chunks: List of `VisionChunk` items to be processed.
For image: `VisionChunkImage` with
`type='image', image=PIL.Image`
For video_chunk: `VisionChunkVideo` with
`type='video_chunk', video_chunk=list[PIL.Image]`
Returns:
[`BatchFeature`]: A [`BatchFeature`] with the following fields:
- **input_ids** -- list of token ids to be fed to a model.
- **pixel_values** -- Pixel values to be fed to a model.
Returned when `vision_chunks` is not `None`.
- **grid_thws** -- list of image 3D grid in LLM.
Returned when `vision_chunks` is not `None`.
"""
if vision_chunks is not None:
mm_inputs = self.image_processor.preprocess(
vision_chunks,
return_tensors=return_tensors,
do_rescale=do_rescale,
do_normalize=do_normalize,
)
else:
mm_inputs = {}
if text is not None:
if not isinstance(text, list):
text = [text]
text_inputs = self.tokenizer(text)
# Note: Modify in-place
input_ids: list[list[int]] = text_inputs["input_ids"] # type: ignore
if vision_chunks is not None:
num_tokens_per_chunk = [
self.image_processor.media_tokens_calculator(chunk)
for chunk in vision_chunks
]
for i in range(len(input_ids)):
new_input_ids = []
for token in input_ids[i]:
if token == self.media_token_id:
new_input_ids.extend(
[self.media_token_id] * num_tokens_per_chunk.pop(0)
)
else:
new_input_ids.append(token)
input_ids[i] = new_input_ids
else:
text_inputs = {}
return BatchFeature(
data={**text_inputs, **mm_inputs},
tensor_type=return_tensors,
)
@@ -1,65 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from transformers import BaseImageProcessor, BatchFeature, TensorType
from transformers.processing_utils import ProcessorMixin
from vllm.tokenizers.hf import HfTokenizer
class KimiK3Processor(ProcessorMixin):
"""HF-style processor wrapper for the image-only Kimi-K3 model.
K3 exposes the standard ``image`` modality, so vLLM calls this processor
with ``images=[PIL, ...]``. The underlying checkpoint image processor
(``KimiK3VisionProcessor``) works on ``{"type": "image", "image": PIL}``
media dicts, so this wrapper adapts bare PIL images into that shape before
delegating to ``preprocess``.
Text is only tokenized here; the single ``<|kimi_image_placeholder|>``
token per image is expanded into the resolution-aware media block by the
model's ``_get_prompt_updates`` on the vLLM side.
"""
attributes = ["image_processor", "tokenizer"]
def __init__(
self,
image_processor: BaseImageProcessor,
tokenizer: HfTokenizer,
) -> None:
self.image_processor = image_processor
self.tokenizer = tokenizer
def __call__(
self,
text: str | list[str] | None = None,
images: object | list[object] | None = None,
return_tensors: str | TensorType | None = None,
do_rescale: bool = True,
do_normalize: bool = True,
**kwargs,
) -> BatchFeature:
if images is not None:
if not isinstance(images, list):
images = [images]
medias = [{"type": "image", "image": image} for image in images]
mm_inputs = self.image_processor.preprocess(
medias,
return_tensors=return_tensors,
do_rescale=do_rescale,
do_normalize=do_normalize,
)
else:
mm_inputs = {}
if text is not None:
if not isinstance(text, list):
text = [text]
text_inputs = self.tokenizer(text)
else:
text_inputs = {}
return BatchFeature(
data={**text_inputs, **mm_inputs},
tensor_type=return_tensors,
)
File diff suppressed because it is too large Load Diff
@@ -1,523 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
# coding=utf-8
# Copyright 2025 The OpenBMB Team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Processor class for MiniCPMO."""
import math
from typing import TypeAlias
import numpy as np
import regex
import torch
from transformers.image_processing_utils import BatchFeature
from transformers.image_utils import ImageInput
from transformers.processing_utils import ProcessorMixin
from transformers.tokenization_utils_base import PreTokenizedInput, TextInput
from transformers.utils import TensorType
MiniCPMOBatchFeature: TypeAlias = BatchFeature
class MiniCPMOProcessor(ProcessorMixin):
r"""Constructs a MiniCPMV processor which wraps a MiniCPMV image
processor and a MiniCPMV tokenizer into a single processor.
[`MiniCPMVProcessor`] offers all the functionalities of
[`MiniCPMVImageProcessor`] and [`LlamaTokenizerWrapper`]. See the
[`~MiniCPMVProcessor.__call__`] and [`~MiniCPMVProcessor.decode`]
for more information.
Args:
image_processor ([`MiniCPMVImageProcessor`], *optional*):
The image processor is a required input.
tokenizer ([`LlamaTokenizerWrapper`], *optional*):
The tokenizer is a required input.
"""
attributes = ["image_processor", "feature_extractor", "tokenizer"]
feature_extractor_class = "WhisperFeatureExtractor"
image_processor_class = "AutoImageProcessor"
tokenizer_class = "AutoTokenizer"
def __init__(
self,
image_processor=None,
feature_extractor=None,
tokenizer=None,
pool_step=2,
):
super().__init__(image_processor, feature_extractor, tokenizer)
# Mirror the MiniCPMVProcessor guard: newer (transformers v5.7+)
# MiniCPM image processors may drop the legacy `version` attribute,
# so fall back to None instead of hard-crashing. `version` only
# special-cases the 2.5 tokenization path; other values take the
# default branch.
self.version = getattr(image_processor, "version", None)
self.pool_step = pool_step
def __call__(
self,
text: TextInput | PreTokenizedInput | list[TextInput] | list[PreTokenizedInput],
images: ImageInput = None,
audios: np.ndarray | list[np.ndarray] | list[list[np.ndarray]] = None,
audio_parts: list | None = None,
max_length: int | None = None,
do_pad: bool | None = True,
max_slice_nums: int | None = None,
use_image_id: bool = True,
chunk_input: bool = False,
return_tensors: str | TensorType | None = TensorType.PYTORCH,
sampling_rate: int | None = 16000,
**kwargs,
) -> MiniCPMOBatchFeature:
if images is not None:
image_inputs = self.image_processor(
images,
do_pad=do_pad,
max_slice_nums=max_slice_nums,
return_tensors=return_tensors,
)
else:
image_inputs = None
if audios is not None:
audio_features, audio_feature_lens, audio_phs = self.audio_feature_extract(
audios, audio_parts, chunk_input, sampling_rate
)
else:
audio_features, audio_feature_lens, audio_phs = [], [], []
model_inputs = self._convert_omni_to_inputs(
image_inputs,
audio_phs,
text,
max_slice_nums=max_slice_nums,
use_image_id=use_image_id,
max_length=max_length,
**kwargs,
)
model_inputs["audio_features"] = audio_features
model_inputs["audio_feature_lens"] = audio_feature_lens
return MiniCPMOBatchFeature(data={**model_inputs})
def get_audio_placeholder(self, audio_lens, chunk_input, chunk_length):
pool_step = self.pool_step
feature_lens = math.ceil(audio_lens / self.feature_extractor.hop_length)
feature_lens = (feature_lens - 1) // 2 + 1
output_lens = (feature_lens - pool_step) // pool_step + 1
audio_start = getattr(self.tokenizer, "audio_start", "<audio>")
audio_end = getattr(self.tokenizer, "audio_end", "</audio>")
if chunk_input:
fbank_feat_in_chunk = int(chunk_length * 100)
cnn_feat_in_chunk = (fbank_feat_in_chunk - 1) // 2 + 1
audio_embeds_in_chunk = (cnn_feat_in_chunk - pool_step) // pool_step + 1
num_audio_chunks = (
output_lens + audio_embeds_in_chunk - 1
) // audio_embeds_in_chunk
place_holders = ""
total_unk_len = 0
for _ in range(num_audio_chunks):
unk_len = min(audio_embeds_in_chunk, output_lens - total_unk_len)
place_holders += audio_start + "<unk>" * unk_len + audio_end
total_unk_len += unk_len
audio_placeholder = place_holders
else:
audio_placeholder = audio_start + "<unk>" * output_lens + audio_end
return audio_placeholder
def audio_feature_extract(
self,
audios: np.ndarray | list[np.ndarray] | list[list[np.ndarray]],
audio_parts: list | None = None,
chunk_input: bool | None = False,
sampling_rate: int | None = None,
chunk_length: int | None = 1,
**kwargs,
):
if isinstance(audios, np.ndarray):
audios_list = [[audios]]
elif isinstance(audios[0], np.ndarray):
audios_list = [audios]
else:
audios_list = audios
if audio_parts is not None:
assert len(audio_parts) == len(audios_list)
for parts, audios in zip(audio_parts, audios_list):
assert len(parts) == len(audios)
audio_feature_lens_list = []
audio_ph_list = []
audio_features_all = []
# audio placeholder not dependent on audio_parts
for audios in audios_list:
if audios:
audio_ph_list.append(
[
self.get_audio_placeholder(len(a), chunk_input, chunk_length)
for a in audios
]
)
else:
audio_ph_list.append([])
for idx, audios in enumerate(audios_list):
if audio_parts is not None:
# same audio part merge
audio_part = audio_parts[idx]
merge_audio = []
cur_audio = []
for aid, (part, audio) in enumerate(zip(audio_part, audios)):
if aid == 0 or audio_part[aid] == audio_part[aid - 1]:
cur_audio.append(audio)
else:
merge_audio.append(np.hstack(cur_audio))
cur_audio = [audio]
if cur_audio:
merge_audio.append(np.hstack(cur_audio))
else:
merge_audio = audios
audio_feature_lens = []
# If the audio exceeds 30 seconds, split it into chunks every 30 seconds.
final_merge_audio = []
max_audio_inp_len = 30 * (sampling_rate or 16000)
for audio in merge_audio:
if len(audio) <= max_audio_inp_len:
final_merge_audio.append(audio)
else:
for i in range(math.ceil(len(audio) / max_audio_inp_len)):
final_merge_audio.append(
audio[i * max_audio_inp_len : (i + 1) * max_audio_inp_len]
)
if audios:
audio_inputs = self.feature_extractor(
final_merge_audio,
sampling_rate=sampling_rate,
return_attention_mask=True,
padding="max_length",
return_tensors="pt",
**kwargs,
)
audio_feature = audio_inputs["input_features"]
actual_lens = audio_inputs["attention_mask"].sum(dim=1)
for feat, lens in zip(audio_feature, actual_lens):
audio_features_all.append(feat[:, :lens])
audio_feature_lens.append(lens)
audio_feature_lens = torch.hstack(audio_feature_lens)
audio_feature_lens_list.append(audio_feature_lens)
else:
audio_feature_lens_list.append([])
if audio_features_all:
audio_features = [i.permute(1, 0) for i in audio_features_all]
audio_features = torch.nn.utils.rnn.pad_sequence(
audio_features, batch_first=True, padding_value=0.0
).permute(0, 2, 1)
else:
audio_features = []
return audio_features, audio_feature_lens_list, audio_ph_list
# Copied from transformers.models.clip.processing_clip.CLIPProcessor.batch_decode
# with CLIP->Llama
def batch_decode(self, *args, **kwargs):
"""This method forwards all its arguments to LlamaTokenizerFast's
[`~PythonBackend.batch_decode`]. Please refer to the
docstring of this method for more information.
"""
output_ids = args[0]
result_text = []
for result in output_ids:
result = result[result != 0]
if len(result) > 0 and result[0] == self.tokenizer.bos_id:
result = result[1:]
if len(result) > 0 and result[-1] == self.tokenizer.eos_id:
result = result[:-1]
result_text.append(
self.tokenizer.decode(result, *args[1:], **kwargs).strip()
)
return result_text
# Copied from transformers.models.clip.processing_clip.CLIPProcessor.decode
# with CLIP->Llama
def decode(self, *args, **kwargs):
"""This method forwards all its arguments to LlamaTokenizerFast's
[`~PythonBackend.decode`]. Please refer to the docstring
of this method for more information.
"""
result = args[0]
result = result[result != 0]
if len(result) > 0 and result[0] == self.tokenizer.bos_id:
result = result[1:]
if len(result) > 0 and (
result[-1] == self.tokenizer.eos_id
or (
hasattr(self.tokenizer, "eot_id")
and result[-1] == self.tokenizer.eot_id
)
):
result = result[:-1]
return self.tokenizer.decode(result, *args[1:], **kwargs).strip()
def _convert(self, input_str, max_inp_length: int | None = None, **kwargs):
input_ids = self.tokenizer.encode(input_str, **kwargs)
if max_inp_length is not None:
input_ids = input_ids[:max_inp_length]
input_ids = torch.tensor(input_ids, dtype=torch.int32)
## image bound
start_cond = (input_ids == self.tokenizer.im_start_id) | (
input_ids == self.tokenizer.slice_start_id
)
end_cond = (input_ids == self.tokenizer.im_end_id) | (
input_ids == self.tokenizer.slice_end_id
)
image_start_idx = torch.where(start_cond)[0]
image_start_idx += 1
image_end_idx = torch.where(end_cond)[0]
assert len(image_start_idx) == len(image_end_idx), (
f"The number of image start tokens ({len(image_start_idx)}) "
f"and end tokens ({len(image_end_idx)}) must match."
)
image_bounds = torch.hstack(
[
image_start_idx.unsqueeze(-1),
image_end_idx.unsqueeze(-1),
]
)
## audio bound
audio_start_idx = torch.where(input_ids == self.tokenizer.audio_start_id)[0]
audio_end_idx = torch.where(input_ids == self.tokenizer.audio_end_id)[0]
assert len(audio_start_idx) == len(audio_end_idx)
audio_bounds = torch.hstack(
[(audio_start_idx + 1).unsqueeze(-1), audio_end_idx.unsqueeze(-1)]
)
spk_start_idx = torch.where(input_ids == self.tokenizer.spk_start_id)[0]
spk_end_idx = torch.where(input_ids == self.tokenizer.spk_end_id)[0]
assert len(spk_start_idx) == len(spk_end_idx)
spk_bounds = torch.hstack(
[(spk_start_idx + 1).unsqueeze(-1), spk_end_idx.unsqueeze(-1)]
)
return input_ids, image_bounds, audio_bounds, spk_bounds
def _convert_omni_to_inputs(
self,
images,
audio_phs,
texts: str | list[str],
truncation=None,
max_length=None,
max_slice_nums=None,
use_image_id=None,
return_tensors=None,
**kwargs,
):
if images is None and audio_phs is None:
model_inputs = self.tokenizer(
texts,
return_tensors=return_tensors,
truncation=truncation,
max_length=max_length,
**kwargs,
)
return MiniCPMOBatchFeature(data={**model_inputs})
image_tag = "(<image>./</image>)"
image_pattern = r"\(<image>./</image>\)"
audio_tag = "(<audio>./</audio>)"
audio_pattern = r"\(<audio>./</audio>\)"
split_pattern = rf"({image_pattern}|{audio_pattern})"
if isinstance(texts, str):
texts = [texts]
bs = len(texts)
if images is not None:
images, image_sizes, tgt_sizes = (
images["pixel_values"],
images["image_sizes"],
images["tgt_sizes"],
)
else:
images, image_sizes, tgt_sizes = [[]] * bs, [[]] * bs, [[]] * bs
input_ids_list = []
image_bounds_list = []
audio_bounds_list = []
spk_bounds_list = []
for index, text in enumerate(texts):
text_chunks = regex.split(split_pattern, text)
image_tags = regex.findall(image_pattern, text)
audio_tags = regex.findall(audio_pattern, text)
if image_tags:
assert images is not None
assert len(image_tags) == len(image_sizes[index])
if audio_tags:
assert audio_phs is not None
assert len(audio_tags) == len(audio_phs[index])
image_id = 0
audio_id = 0
for i, chunk in enumerate(text_chunks):
if chunk == image_tag:
image_placeholder = (
self.image_processor.get_slice_image_placeholder(
image_sizes[index][image_id],
image_id,
max_slice_nums,
use_image_id,
)
)
image_id += 1
text_chunks[i] = image_placeholder
elif chunk == audio_tag:
audio_placeholder = audio_phs[index][audio_id]
audio_id += 1
text_chunks[i] = audio_placeholder
final_text = "".join(text_chunks)
input_ids, image_bounds, audio_bounds, spk_bounds = self._convert(
final_text, max_length, **kwargs
)
input_ids_list.append(input_ids)
image_bounds_list.append(image_bounds)
audio_bounds_list.append(audio_bounds)
spk_bounds_list.append(spk_bounds)
padded_input_ids, padding_lengths = self.pad(
input_ids_list, padding_side="left"
)
attention_mask = torch.ones_like(padded_input_ids, dtype=torch.bool)
for i, length in enumerate(padding_lengths):
image_bounds_list[i] = image_bounds_list[i] + length
audio_bounds_list[i] = audio_bounds_list[i] + length
spk_bounds_list[i] = spk_bounds_list[i] + length
attention_mask[i, :length] = False
data = {
"input_ids": padded_input_ids,
"attention_mask": attention_mask,
"pixel_values": images,
"image_sizes": image_sizes,
"image_bound": image_bounds_list,
"tgt_sizes": tgt_sizes,
"audio_bounds": audio_bounds_list,
"spk_bounds": spk_bounds_list,
}
return data
@property
# Copied from
# transformers.models.clip.processing_clip.CLIPProcessor.model_input_names
def model_input_names(self):
tokenizer_input_names = self.tokenizer.model_input_names
image_processor_input_names = self.image_processor.model_input_names
feature_extractor_input_names = self.feature_extractor.model_input_names
return list(
dict.fromkeys(
tokenizer_input_names
+ image_processor_input_names
+ feature_extractor_input_names
)
)
def pad(
self,
inputs,
max_length=None,
padding_value=0,
padding_side="left",
):
if not inputs:
return torch.empty(0), []
items = []
if isinstance(inputs[0], list):
assert isinstance(inputs[0][0], torch.Tensor)
for it in inputs:
for tr in it:
items.append(tr)
else:
assert isinstance(inputs[0], torch.Tensor)
items = inputs
batch_size = len(items)
shape = items[0].shape
dim = len(shape)
assert dim <= 2
if max_length is None:
max_length = 0
max_length = max(max_length, max(item.shape[-1] for item in items))
min_length = min(item.shape[-1] for item in items)
dtype = items[0].dtype
if dim == 0:
return torch.stack([item for item in items], dim=0), [0]
elif dim == 1:
if max_length == min_length:
return (
torch.stack([item for item in items], dim=0),
[0] * batch_size,
)
tensor = torch.zeros((batch_size, max_length), dtype=dtype) + padding_value
else:
tensor = (
torch.zeros((batch_size, max_length, shape[-1]), dtype=dtype)
+ padding_value
)
padding_length = []
for i, item in enumerate(items):
if dim == 1:
if padding_side == "left":
tensor[i, -len(item) :] = item.clone()
else:
tensor[i, : len(item)] = item.clone()
elif dim == 2:
if padding_side == "left":
tensor[i, -len(item) :, :] = item.clone()
else:
tensor[i, : len(item), :] = item.clone()
padding_length.append(tensor.shape[-1] - len(item))
return tensor, padding_length
@@ -1,317 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
# Copyright 2024 The HuggingFace Inc. team.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Processor class for MiniCPMV."""
from typing import TypeAlias
import regex
import torch
from transformers.image_processing_utils import BatchFeature
from transformers.image_utils import ImageInput
from transformers.processing_utils import ProcessorMixin
from transformers.tokenization_utils_base import (
PaddingStrategy,
PreTokenizedInput,
TextInput,
TruncationStrategy,
)
from transformers.utils import TensorType
MiniCPMVBatchFeature: TypeAlias = BatchFeature
class MiniCPMVProcessor(ProcessorMixin):
r"""Constructs a MiniCPMV processor which wraps a MiniCPMV image
processor and a MiniCPMV tokenizer into a single processor.
[`MiniCPMVProcessor`] offers all the functionalities of
[`MiniCPMVImageProcessor`] and [`LlamaTokenizerWrapper`]. See the
[`~MiniCPMVProcessor.__call__`] and [`~MiniCPMVProcessor.decode`]
for more information.
Args:
image_processor ([`MiniCPMVImageProcessor`], *optional*):
The image processor is a required input.
tokenizer ([`LlamaTokenizerWrapper`], *optional*):
The tokenizer is a required input.
"""
attributes = ["image_processor", "tokenizer"]
image_processor_class = "AutoImageProcessor"
tokenizer_class = "AutoTokenizer"
def __init__(self, image_processor=None, tokenizer=None, version=None):
super().__init__(image_processor, tokenizer)
self.version = version
def __call__(
self,
text: TextInput | PreTokenizedInput | list[TextInput] | list[PreTokenizedInput],
images: ImageInput = None,
padding: bool | str | PaddingStrategy = False,
truncation: bool | str | TruncationStrategy = None,
max_length: int | None = None,
do_pad: bool | None = True,
return_tensors: str | TensorType | None = TensorType.PYTORCH,
) -> MiniCPMVBatchFeature:
"""Run the vendored MiniCPMV processor on a (text, images) pair.
Batched inputs are supported following the upstream MiniCPM-V
processor flow. ``images`` is forwarded to the underlying image
processor and ``text`` is tokenized with image placeholders
replaced by the appropriate slice tokens. Returns a
``MiniCPMVBatchFeature`` with at minimum ``input_ids`` and (when
images are provided) ``pixel_values``, ``image_sizes``,
``image_bound`` and ``tgt_sizes``.
"""
if images is not None:
image_inputs = self.image_processor(
images, do_pad=do_pad, return_tensors=return_tensors
)
else:
image_inputs = {}
return self._convert_images_texts_to_inputs(
image_inputs, text, max_length=max_length
)
# Copied from transformers.models.clip.processing_clip.CLIPProcessor
# .batch_decode with CLIP->Llama
def batch_decode(self, *args, **kwargs):
"""This method forwards all its arguments to LlamaTokenizerFast's
[`~PythonBackend.batch_decode`]. Please refer to the
docstring of this method for more information.
"""
output_ids = args[0]
result_text = []
bos_id = getattr(
self.tokenizer,
"bos_token_id",
getattr(self.tokenizer, "bos_id", 1),
)
eos_id = getattr(
self.tokenizer,
"eos_token_id",
getattr(self.tokenizer, "eos_id", 2),
)
for result in output_ids:
result = result[result != 0]
if len(result) > 0 and result[0] == bos_id:
result = result[1:]
if len(result) > 0 and result[-1] == eos_id:
result = result[:-1]
result_text.append(
self.tokenizer.decode(result, *args[1:], **kwargs).strip()
)
return result_text
# Copied from transformers.models.clip.processing_clip.CLIPProcessor
# .decode with CLIP->Llama
def decode(self, *args, **kwargs):
"""This method forwards all its arguments to LlamaTokenizerFast's
[`~PythonBackend.decode`]. Please refer to the docstring
of this method for more information.
"""
result = args[0]
result = result[result != 0]
bos_id = getattr(
self.tokenizer,
"bos_token_id",
getattr(self.tokenizer, "bos_id", 1),
)
eos_id = getattr(
self.tokenizer,
"eos_token_id",
getattr(self.tokenizer, "eos_id", 2),
)
eot_id = getattr(self.tokenizer, "eot_id", None)
if len(result) > 0 and result[0] == bos_id:
result = result[1:]
if len(result) > 0 and (
result[-1] == eos_id or (eot_id is not None and result[-1] == eot_id)
):
result = result[:-1]
return self.tokenizer.decode(result, *args[1:], **kwargs).strip()
def _convert(self, input_str, max_inp_length: int | None = None):
add_bos = getattr(self.tokenizer, "add_bos_token", False)
if self.version == (2, 5) or add_bos:
input_ids = self.tokenizer.encode(input_str)
else:
bos_id = getattr(
self.tokenizer,
"bos_token_id",
getattr(self.tokenizer, "bos_id", 1),
)
input_ids = [bos_id] + self.tokenizer.encode(input_str)
if max_inp_length is not None:
input_ids = input_ids[:max_inp_length]
input_ids = torch.tensor(input_ids, dtype=torch.int32)
im_start_id = getattr(
self.tokenizer,
"im_start_id",
self.tokenizer.convert_tokens_to_ids("<im_start>"),
)
im_end_id = getattr(
self.tokenizer,
"im_end_id",
self.tokenizer.convert_tokens_to_ids("<im_end>"),
)
image_start_tokens = torch.where(input_ids == im_start_id)[0]
image_start_tokens += 1
image_end_tokens = torch.where(input_ids == im_end_id)[0]
assert len(image_start_tokens) == len(image_end_tokens), (
f"The number of image start tokens ({len(image_start_tokens)}) "
f"and end tokens ({len(image_end_tokens)}) must match."
)
image_bounds = torch.hstack(
[
image_start_tokens.unsqueeze(-1),
image_end_tokens.unsqueeze(-1),
]
)
return input_ids, image_bounds
def _convert_images_texts_to_inputs(
self,
images,
texts,
do_pad=False,
truncation=None,
max_length=None,
return_tensors=None,
):
if not len(images):
model_inputs = self.tokenizer(
texts,
return_tensors=return_tensors,
padding=do_pad,
truncation=truncation,
max_length=max_length,
)
return MiniCPMVBatchFeature(data={**model_inputs})
pattern = "(<image>./</image>)"
images_val = images["pixel_values"]
image_sizes = images["image_sizes"]
tgt_sizes = images["tgt_sizes"]
if isinstance(texts, str):
texts = [texts]
input_ids_list = []
image_bounds_list = []
for index, text in enumerate(texts):
image_tags = regex.findall(pattern, text)
assert len(image_tags) == len(image_sizes[index])
text_chunks = text.split(pattern)
final_text = ""
for i in range(len(image_tags)):
placeholder = self.image_processor.get_slice_image_placeholder(
image_sizes[index][i]
)
final_text = final_text + text_chunks[i] + placeholder
final_text += text_chunks[-1]
input_ids, image_bounds = self._convert(final_text, max_length)
input_ids_list.append(input_ids)
image_bounds_list.append(image_bounds)
padded_input_ids, padding_lengths = self.pad(
input_ids_list,
padding_side="left",
)
for i, length in enumerate(padding_lengths):
image_bounds_list[i] = image_bounds_list[i] + length
return MiniCPMVBatchFeature(
data={
"input_ids": padded_input_ids,
"attention_mask": padded_input_ids.ne(0),
"pixel_values": images_val,
"image_sizes": image_sizes,
"image_bound": image_bounds_list,
"tgt_sizes": tgt_sizes,
}
)
@property
# Copied from
# transformers.models.clip.processing_clip.CLIPProcessor.model_input_names
def model_input_names(self):
tokenizer_input_names = self.tokenizer.model_input_names
image_processor_input_names = self.image_processor.model_input_names
return list(dict.fromkeys(tokenizer_input_names + image_processor_input_names))
# Copied from openbmb/MiniCPM-V-4_5 processing_minicpmv.py.
def pad(self, inputs, max_length=None, padding_value=0, padding_side="left"):
if not inputs:
return torch.empty(0), []
items = []
if isinstance(inputs[0], list):
assert isinstance(inputs[0][0], torch.Tensor)
for it in inputs:
for tr in it:
items.append(tr)
else:
assert isinstance(inputs[0], torch.Tensor)
items = inputs
batch_size = len(items)
shape = items[0].shape
dim = len(shape)
assert dim <= 2
if max_length is None:
max_length = 0
max_length = max(max_length, max(item.shape[-1] for item in items))
min_length = min(item.shape[-1] for item in items)
dtype = items[0].dtype
if dim == 0:
return torch.stack([item for item in items], dim=0), [0]
elif dim == 1:
if max_length == min_length:
return torch.stack([item for item in items], dim=0), [0] * batch_size
tensor = torch.zeros((batch_size, max_length), dtype=dtype) + padding_value
else:
tensor = (
torch.zeros((batch_size, max_length, shape[-1]), dtype=dtype)
+ padding_value
)
padding_lengths = []
for i, item in enumerate(items):
if dim == 1:
if padding_side == "left":
tensor[i, -len(item) :] = item.clone()
else:
tensor[i, : len(item)] = item.clone()
elif dim == 2:
if padding_side == "left":
tensor[i, -len(item) :, :] = item.clone()
else:
tensor[i, : len(item), :] = item.clone()
padding_lengths.append(tensor.shape[-1] - len(item))
return tensor, padding_lengths
@@ -1,660 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""MiniMax M3 VL HuggingFace-compatible Processor / ImageProcessor /
VideoProcessor, vendored into vLLM so the model loads without
``--trust-remote-code`` (the released checkpoint only ships these classes as
remote code via ``auto_map``).
Adapted verbatim from the ``MiniMaxAI/Minimax-M3-preview`` repository files
``image_processor.py``, ``video_processor.py`` and ``processing_minimax.py``
(revision ``db01c0fe``). Both image and video processors use Qwen-style
``smart_resize`` (bound by total pixels). The original async frame-sampling
helpers are intentionally omitted: vLLM performs its own frame loading and
feeds decoded frames to the processor.
"""
import math
import torch
from torchvision.transforms import InterpolationMode
from transformers import AutoTokenizer, BatchFeature
from transformers.image_processing_backends import (
TorchvisionBackend,
group_images_by_shape,
reorder_images,
)
from transformers.image_utils import PILImageResampling, SizeDict
from transformers.processing_utils import (
ImagesKwargs,
ProcessingKwargs,
ProcessorMixin,
Unpack,
VideosKwargs,
)
from transformers.utils import TensorType
from transformers.video_processing_utils import BaseVideoProcessor
from transformers.video_utils import group_videos_by_shape, reorder_videos
# Maximum allowed aspect ratio before smart_resize rejects the input.
MAX_RATIO = 200
# Fixed (non-configurable) bounds for the long-side resize logic, per the
# MiniMax-M3 size spec. ``min_short_side_pixel`` is the floor the short edge is
# enlarged to; ``*_MAX_TOTAL_PIXELS`` is the hard area cap that, once exceeded,
# aborts processing instead of downscaling.
MIN_SHORT_SIDE_PIXEL = 112
IMAGE_MAX_TOTAL_PIXELS = 12_845_056 # 3584 ** 2 (width * height)
VIDEO_MAX_TOTAL_PIXELS = 301_056_000 # width * height * frames
def round_by_factor(number: int | float, factor: int) -> int:
return round(number / factor) * factor
def ceil_by_factor(number: int | float, factor: int) -> int:
return math.ceil(number / factor) * factor
def floor_by_factor(number: int | float, factor: int) -> int:
return math.floor(number / factor) * factor
def _smart_resize_by_long_side(
height: int,
width: int,
factor: int,
max_long_side_pixel: int,
min_short_side_pixel: int,
max_total_pixels: int | None,
) -> tuple[int, int]:
"""Long-side based resize (MiniMax-M3 size spec).
(a) if the long side exceeds ``max_long_side_pixel`` → shrink so the long
side equals ``max_long_side_pixel``;
(b) else if the short side is below ``min_short_side_pixel`` → enlarge so the
short side equals ``min_short_side_pixel``;
(c) if the resulting area still exceeds ``max_total_pixels`` → raise.
(a) and (b) are mutually exclusive (they branch on the *original* long side).
Both sides are then rounded to a multiple of ``factor``. For videos the
``max_total_pixels`` cap is volumetric (width * height * frames) and is
enforced by the caller, so pass ``max_total_pixels=None`` here.
"""
long_side = max(height, width)
short_side = min(height, width)
scaled_height: float = height
scaled_width: float = width
if long_side > max_long_side_pixel:
beta = max_long_side_pixel / long_side
scaled_height = height * beta
scaled_width = width * beta
elif short_side < min_short_side_pixel:
beta = min_short_side_pixel / short_side
scaled_height = height * beta
scaled_width = width * beta
h_bar = max(factor, round_by_factor(scaled_height, factor))
w_bar = max(factor, round_by_factor(scaled_width, factor))
if max_total_pixels is not None and h_bar * w_bar > max_total_pixels:
raise ValueError(
f"image area {h_bar * w_bar} exceeds max_total_pixels "
f"{max_total_pixels} after resizing"
)
return h_bar, w_bar
def smart_resize(
height: int,
width: int,
factor: int = 28,
min_pixels: int = 4 * 28 * 28,
max_pixels: int = 451584,
max_long_side_pixel: int | None = None,
min_short_side_pixel: int = MIN_SHORT_SIDE_PIXEL,
max_total_pixels: int | None = None,
) -> tuple[int, int]:
"""Rescale (height, width) so each side is a multiple of ``factor``.
When ``max_long_side_pixel`` is set, use the MiniMax-M3 long-side resize
spec (see :func:`_smart_resize_by_long_side`). Otherwise fall back to the
Qwen-VL area bound, keeping the total area within ``[min_pixels, max_pixels]``.
"""
if max(height, width) / min(height, width) > MAX_RATIO:
raise ValueError(
f"absolute aspect ratio must be smaller than {MAX_RATIO}, "
f"got {max(height, width) / min(height, width)}"
)
if max_long_side_pixel is not None:
return _smart_resize_by_long_side(
height,
width,
factor=factor,
max_long_side_pixel=max_long_side_pixel,
min_short_side_pixel=min_short_side_pixel,
max_total_pixels=max_total_pixels,
)
h_bar = max(factor, round_by_factor(height, factor))
w_bar = max(factor, round_by_factor(width, factor))
if h_bar * w_bar > max_pixels:
beta = math.sqrt((height * width) / max_pixels)
h_bar = floor_by_factor(height / beta, factor)
w_bar = floor_by_factor(width / beta, factor)
elif h_bar * w_bar < min_pixels:
beta = math.sqrt(min_pixels / (height * width))
h_bar = ceil_by_factor(height * beta, factor)
w_bar = ceil_by_factor(width * beta, factor)
return h_bar, w_bar
class MiniMaxM3VLImageProcessorKwargs(ImagesKwargs, total=False): # type: ignore[call-arg]
patch_size: int
temporal_patch_size: int
merge_size: int
max_pixels: int
max_long_side_pixel: int
class MiniMaxM3VLImageProcessor(TorchvisionBackend):
do_resize = True
resample = PILImageResampling.BICUBIC
# required by base-class validation, not used as the resize bound
size = {"height": 672, "width": 672}
default_to_square = False
do_rescale = True
rescale_factor = 1 / 255
do_normalize = True
image_mean = [0.48145466, 0.4578275, 0.40821073]
image_std = [0.26862954, 0.26130258, 0.27577711]
do_convert_rgb = True
patch_size = 14
temporal_patch_size = 2
merge_size = 2
max_pixels = 451584 # 672 * 672
# Long-side resize spec (opt-in via ``max_long_side_pixel``). The latter two
# are fixed per the spec and are not exposed as configurable kwargs.
max_long_side_pixel = None
min_short_side_pixel = MIN_SHORT_SIDE_PIXEL
max_total_pixels = IMAGE_MAX_TOTAL_PIXELS
valid_kwargs = MiniMaxM3VLImageProcessorKwargs
model_input_names = ["pixel_values", "image_grid_thw"]
def __init__(self, **kwargs: Unpack[MiniMaxM3VLImageProcessorKwargs]):
super().__init__(**kwargs)
def preprocess(
self, images, **kwargs: Unpack[MiniMaxM3VLImageProcessorKwargs]
) -> BatchFeature:
return super().preprocess(images, **kwargs)
def _preprocess(
self,
images: list[torch.Tensor],
do_resize: bool,
size: SizeDict,
resample: "PILImageResampling | InterpolationMode | int | None",
do_rescale: bool,
rescale_factor: float,
do_normalize: bool,
image_mean: "float | list[float] | None",
image_std: "float | list[float] | None",
patch_size: int,
temporal_patch_size: int,
merge_size: int,
max_pixels: int,
max_long_side_pixel: "int | None",
disable_grouping: "bool | None",
return_tensors: "str | TensorType | None",
**kwargs,
) -> BatchFeature:
grouped_images, grouped_images_index = group_images_by_shape(
images, disable_grouping=disable_grouping
)
resized_images_grouped = {}
factor = patch_size * merge_size
for shape, stacked_images in grouped_images.items():
height, width = stacked_images.shape[-2:]
if do_resize:
resized_height, resized_width = smart_resize(
height,
width,
factor=factor,
max_pixels=max_pixels,
max_long_side_pixel=max_long_side_pixel,
min_short_side_pixel=self.min_short_side_pixel,
max_total_pixels=self.max_total_pixels,
)
stacked_images = self.resize(
stacked_images,
size=SizeDict(height=resized_height, width=resized_width),
resample=resample,
)
resized_images_grouped[shape] = stacked_images
resized_images = reorder_images(resized_images_grouped, grouped_images_index)
grouped_images, grouped_images_index = group_images_by_shape(
resized_images, disable_grouping=disable_grouping
)
processed_images_grouped = {}
processed_grids = {}
for shape, stacked_images in grouped_images.items():
resized_height, resized_width = stacked_images.shape[-2:]
patches = self.rescale_and_normalize(
stacked_images,
do_rescale,
rescale_factor,
do_normalize,
image_mean,
image_std,
)
if patches.ndim == 4:
patches = patches.unsqueeze(1)
if patches.shape[1] % temporal_patch_size != 0:
repeats = patches[:, -1:].repeat(
1,
temporal_patch_size - (patches.shape[1] % temporal_patch_size),
1,
1,
1,
)
patches = torch.cat([patches, repeats], dim=1)
batch_size, grid_t, channel = patches.shape[:3]
grid_t = grid_t // temporal_patch_size
grid_h, grid_w = resized_height // patch_size, resized_width // patch_size
patches = patches.view(
batch_size,
grid_t,
temporal_patch_size,
channel,
grid_h // merge_size,
merge_size,
patch_size,
grid_w // merge_size,
merge_size,
patch_size,
)
patches = patches.permute(0, 1, 4, 7, 5, 8, 3, 2, 6, 9)
flatten_patches = patches.reshape(
batch_size,
grid_t * grid_h * grid_w,
channel * temporal_patch_size * patch_size * patch_size,
)
processed_images_grouped[shape] = flatten_patches
processed_grids[shape] = [[grid_t, grid_h, grid_w]] * batch_size
processed_images = reorder_images(
processed_images_grouped, grouped_images_index
)
processed_grids = reorder_images(processed_grids, grouped_images_index)
pixel_values = torch.cat(processed_images, dim=0)
image_grid_thw = torch.tensor(processed_grids, dtype=torch.long)
return BatchFeature(
data={"pixel_values": pixel_values, "image_grid_thw": image_grid_thw},
tensor_type=return_tensors,
)
def get_number_of_image_patches(self, height: int, width: int, images_kwargs=None):
images_kwargs = images_kwargs or {}
patch_size = images_kwargs.get("patch_size", self.patch_size)
merge_size = images_kwargs.get("merge_size", self.merge_size)
max_pixels = images_kwargs.get("max_pixels", self.max_pixels)
max_long_side_pixel = images_kwargs.get(
"max_long_side_pixel", self.max_long_side_pixel
)
resized_height, resized_width = smart_resize(
height,
width,
factor=patch_size * merge_size,
max_pixels=max_pixels,
max_long_side_pixel=max_long_side_pixel,
min_short_side_pixel=self.min_short_side_pixel,
max_total_pixels=self.max_total_pixels,
)
grid_h, grid_w = resized_height // patch_size, resized_width // patch_size
return grid_h * grid_w
class MiniMaxM3VLVideoProcessorKwargs(VideosKwargs, total=False): # type: ignore[call-arg]
patch_size: int
temporal_patch_size: int
merge_size: int
min_pixels: int
max_pixels: int
max_long_side_pixel: int
total_pixels: int
min_frames: int
max_frames: int
fps: "float | int"
class MiniMaxM3VLVideoProcessor(BaseVideoProcessor):
do_resize = True
resample = PILImageResampling.BICUBIC
size = {"height": 672, "width": 672}
default_to_square = False
do_rescale = True
rescale_factor = 1 / 255
do_normalize = True
image_mean = [0.48145466, 0.4578275, 0.40821073]
image_std = [0.26862954, 0.26130258, 0.27577711]
do_convert_rgb = True
do_sample_frames = False
patch_size = 14
temporal_patch_size = 2
merge_size = 2
min_pixels = 4 * 28 * 28
max_pixels = 768 * 28 * 28 # 602,112
total_pixels = int(64000 * 28 * 28 * 0.9) # ~45M, ~64k tokens budget
# Long-side resize spec (opt-in via ``max_long_side_pixel``). The video
# ``max_total_pixels`` cap is volumetric (width * height * frames) and is
# enforced in ``_preprocess`` once the frame count is known.
max_long_side_pixel = None
min_short_side_pixel = MIN_SHORT_SIDE_PIXEL
max_total_pixels = VIDEO_MAX_TOTAL_PIXELS
fps = 1.0
min_frames = 4
max_frames = 768
valid_kwargs = MiniMaxM3VLVideoProcessorKwargs
model_input_names = ["pixel_values_videos", "video_grid_thw"]
def __init__(self, **kwargs: Unpack[MiniMaxM3VLVideoProcessorKwargs]):
super().__init__(**kwargs)
def _preprocess(
self,
videos: list[torch.Tensor],
do_resize: bool,
size: SizeDict,
resample: "PILImageResampling | InterpolationMode | int | None",
do_rescale: bool,
rescale_factor: float,
do_normalize: bool,
image_mean: "float | list[float] | None",
image_std: "float | list[float] | None",
patch_size: int,
temporal_patch_size: int,
merge_size: int,
min_pixels: int,
max_pixels: int,
max_long_side_pixel: "int | None" = None,
return_tensors: "str | TensorType | None" = None,
**kwargs,
) -> BatchFeature:
grouped_videos, grouped_videos_index = group_videos_by_shape(videos)
resized_videos_grouped = {}
factor = patch_size * merge_size
for shape, stacked_videos in grouped_videos.items():
batch_size, num_frames, channels, height, width = stacked_videos.shape
resized_height, resized_width = height, width
if do_resize:
resized_height, resized_width = smart_resize(
height,
width,
factor=factor,
min_pixels=min_pixels,
max_pixels=max_pixels,
max_long_side_pixel=max_long_side_pixel,
min_short_side_pixel=self.min_short_side_pixel,
# Per-frame raise disabled; the video cap is volumetric and
# is enforced below once num_frames is known.
max_total_pixels=None,
)
if (
max_long_side_pixel is not None
and resized_height * resized_width * num_frames
> self.max_total_pixels
):
raise ValueError(
f"video area {resized_height * resized_width * num_frames} "
f"(width * height * frames) exceeds max_total_pixels "
f"{self.max_total_pixels} after resizing"
)
stacked_videos = stacked_videos.view(
batch_size * num_frames, channels, height, width
)
stacked_videos = self.resize(
stacked_videos,
size=SizeDict(height=resized_height, width=resized_width),
resample=resample,
)
stacked_videos = stacked_videos.view(
batch_size,
num_frames,
channels,
resized_height,
resized_width,
)
resized_videos_grouped[shape] = stacked_videos
resized_videos = reorder_videos(resized_videos_grouped, grouped_videos_index)
grouped_videos, grouped_videos_index = group_videos_by_shape(resized_videos)
processed_videos_grouped = {}
processed_grids = {}
for shape, stacked_videos in grouped_videos.items():
resized_height, resized_width = stacked_videos.shape[-2:]
patches = self.rescale_and_normalize(
stacked_videos,
do_rescale,
rescale_factor,
do_normalize,
image_mean,
image_std,
)
if pad := -patches.shape[1] % temporal_patch_size:
repeats = patches[:, -1:].expand(-1, pad, -1, -1, -1)
patches = torch.cat([patches, repeats], dim=1)
batch_size, grid_t, channels = patches.shape[:3]
grid_t = grid_t // temporal_patch_size
grid_h, grid_w = resized_height // patch_size, resized_width // patch_size
patches = patches.view(
batch_size,
grid_t,
temporal_patch_size,
channels,
grid_h // merge_size,
merge_size,
patch_size,
grid_w // merge_size,
merge_size,
patch_size,
)
patches = patches.permute(0, 1, 4, 7, 5, 8, 3, 2, 6, 9)
flatten_patches = patches.reshape(
batch_size,
grid_t * grid_h * grid_w,
channels * temporal_patch_size * patch_size * patch_size,
)
processed_videos_grouped[shape] = flatten_patches
processed_grids[shape] = [[grid_t, grid_h, grid_w]] * batch_size
processed_videos = reorder_videos(
processed_videos_grouped, grouped_videos_index
)
processed_grids = reorder_videos(processed_grids, grouped_videos_index)
pixel_values_videos = torch.cat(processed_videos, dim=0)
video_grid_thw = torch.tensor(processed_grids, dtype=torch.long)
return BatchFeature(
data={
"pixel_values_videos": pixel_values_videos,
"video_grid_thw": video_grid_thw,
},
tensor_type=return_tensors,
)
class MiniMaxVLProcessorKwargs(ProcessingKwargs, total=False): # type: ignore[call-arg]
_defaults = {
"videos_kwargs": {
"do_resize": False,
"return_metadata": True,
},
}
class MiniMaxVLProcessor(ProcessorMixin):
IMAGE_TOKEN = "]<]image[>["
VIDEO_TOKEN = "]<]video[>["
VISION_START_TOKEN = "]<]start of image[>["
VISION_END_TOKEN = "]<]end of image[>["
@classmethod
def from_pretrained(cls, pretrained_model_name_or_path, **kwargs):
# Bypass ProcessorMixin's dynamic module lookup, which breaks in
# transformers >= 5.9 when image_processor_class is a string: the
# register() API now stores classes as {"pil": cls} dicts in
# _extra_content, but get_possibly_dynamic_module() still calls
# .__name__ on the raw value, crashing with AttributeError on dicts.
#
# Reuse the tokenizer passed by vLLM instead of loading another one,
# and keep it out of the kwargs forwarded to the sub-processor loaders.
tokenizer = kwargs.pop("tokenizer", None)
if tokenizer is None:
tokenizer = AutoTokenizer.from_pretrained(
pretrained_model_name_or_path, **kwargs
)
image_processor = MiniMaxM3VLImageProcessor.from_pretrained(
pretrained_model_name_or_path, **kwargs
)
video_processor = MiniMaxM3VLVideoProcessor.from_pretrained(
pretrained_model_name_or_path, **kwargs
)
return cls(
image_processor=image_processor,
tokenizer=tokenizer,
video_processor=video_processor,
)
def __init__(
self, image_processor=None, tokenizer=None, video_processor=None, **kwargs
):
self.image_token_id = tokenizer.convert_tokens_to_ids(self.IMAGE_TOKEN)
self.video_token_id = tokenizer.convert_tokens_to_ids(self.VIDEO_TOKEN)
super().__init__(image_processor, tokenizer, video_processor)
# Video expansion also uses image start/end tokens. Separate video
# start/end tokens exist in the tokenizer, but the original MiniMax
# serving path did not use them; keep that behavior for compatibility.
self.vision_start_token_id = tokenizer.convert_tokens_to_ids(
self.VISION_START_TOKEN
)
self.vision_end_token_id = tokenizer.convert_tokens_to_ids(
self.VISION_END_TOKEN
)
def __call__(
self,
images=None,
text=None,
videos=None,
**kwargs: Unpack[MiniMaxVLProcessorKwargs],
) -> BatchFeature:
output_kwargs = self._merge_kwargs(
MiniMaxVLProcessorKwargs,
tokenizer_init_kwargs=self.tokenizer.init_kwargs,
**kwargs,
)
if images is not None:
images_kwargs = output_kwargs["images_kwargs"]
image_inputs = self.image_processor(images=images, **images_kwargs)
image_grid_thw = image_inputs["image_grid_thw"]
else:
image_inputs = {}
image_grid_thw = None
if videos is not None:
videos_kwargs = output_kwargs["videos_kwargs"]
video_inputs = self.video_processor(videos=videos, **videos_kwargs)
video_grid_thw = video_inputs["video_grid_thw"]
if not kwargs.get("return_metadata"):
video_metadata = video_inputs.pop("video_metadata")
else:
video_metadata = video_inputs["video_metadata"]
else:
video_inputs = {}
video_grid_thw = None
if not isinstance(text, list):
text = [text]
text = text.copy()
# Expand image tokens
if image_grid_thw is not None:
merge_length = self.image_processor.merge_size**2
placeholder = "]<]placeholder[>["
index = 0
for i in range(len(text)):
while self.IMAGE_TOKEN in text[i]:
num_tokens = image_grid_thw[index].prod() // merge_length
text[i] = text[i].replace(
self.IMAGE_TOKEN,
self.VISION_START_TOKEN
+ placeholder * num_tokens
+ self.VISION_END_TOKEN,
1,
)
index += 1
text[i] = text[i].replace(placeholder, self.IMAGE_TOKEN)
# Expand video tokens
if video_grid_thw is not None:
merge_length = self.image_processor.merge_size**2
placeholder = "]<]placeholder[>["
index = 0
for i in range(len(text)):
while self.VIDEO_TOKEN in text[i]:
metadata = video_metadata[index]
grid_t = video_grid_thw[index][0]
frame_seqlen = video_grid_thw[index][1:].prod() // merge_length
video_placeholder = ""
for frame_idx in range(grid_t):
if (
metadata.fps is not None
and metadata.frames_indices is not None
):
ts = (
metadata.frames_indices[
min(
frame_idx
* self.video_processor.temporal_patch_size,
len(metadata.frames_indices) - 1,
)
]
/ metadata.fps
)
video_placeholder += f"]<]{ts:.1f} seconds[>["
video_placeholder += (
self.VISION_START_TOKEN
+ placeholder * frame_seqlen
+ self.VISION_END_TOKEN
)
text[i] = text[i].replace(self.VIDEO_TOKEN, video_placeholder, 1)
index += 1
text[i] = text[i].replace(placeholder, self.VIDEO_TOKEN)
# Tokenize
return_tensors = output_kwargs["text_kwargs"].pop("return_tensors", None)
text_inputs = self.tokenizer(text, **output_kwargs["text_kwargs"])
return BatchFeature(
data={**text_inputs, **image_inputs, **video_inputs},
tensor_type=return_tensors,
)
@@ -1,539 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Custom processor for Moondream3 model."""
import math
import numpy as np
import torch
from PIL import Image
from transformers import AutoProcessor, BatchFeature
from transformers.image_utils import ImageInput
from transformers.processing_utils import ProcessingKwargs, ProcessorMixin, Unpack
from transformers.tokenization_utils_base import (
PreTokenizedInput,
PreTrainedTokenizerBase,
TextInput,
)
from vllm.multimodal.image import convert_image_mode
__all__ = ["Moondream3Processor"]
class Moondream3ProcessorKwargs(ProcessingKwargs, total=False): # type: ignore[call-arg]
_defaults = {
"text_kwargs": {
"padding": False,
},
"images_kwargs": {
"max_crops": 12,
"overlap_margin": 4,
"crop_size": 378,
"patch_size": 14,
"convert_to_rgb": True,
"return_tensors": "pt",
},
}
def select_tiling(
height: int, width: int, crop_size: int, max_crops: int
) -> tuple[int, int]:
"""Determine the optimal number of tiles to cover an image."""
if height <= crop_size or width <= crop_size:
return (1, 1)
min_h = math.ceil(height / crop_size)
min_w = math.ceil(width / crop_size)
if min_h * min_w > max_crops:
ratio = math.sqrt(max_crops / (min_h * min_w))
return (max(1, math.floor(min_h * ratio)), max(1, math.floor(min_w * ratio)))
h_tiles = math.floor(math.sqrt(max_crops * height / width))
w_tiles = math.floor(math.sqrt(max_crops * width / height))
h_tiles = max(h_tiles, min_h)
w_tiles = max(w_tiles, min_w)
if h_tiles * w_tiles > max_crops:
if w_tiles > h_tiles:
w_tiles = math.floor(max_crops / h_tiles)
else:
h_tiles = math.floor(max_crops / w_tiles)
return (max(1, h_tiles), max(1, w_tiles))
class Moondream3Processor(ProcessorMixin):
"""Constructs a Moondream3 processor which handles image preprocessing
and tokenization for the Moondream3 multimodal model.
Args:
tokenizer: The tokenizer to use for text processing.
chat_template: Optional chat template string.
crop_size: Size of each image crop.
max_crops: Maximum number of crops per image.
overlap_margin: Margin for overlapping crops in patches.
patch_size: Size of each patch.
"""
attributes = ["tokenizer"]
valid_kwargs = [
"chat_template",
"crop_size",
"max_crops",
"overlap_margin",
"patch_size",
]
tokenizer_class = "AutoTokenizer"
# Use separate tokenizer repo
_tokenizer_repo = "moondream/starmie-v1"
# Default chat template for Moondream3
# Moondream uses special tokens for prompting:
# - Token 0 (<|endoftext|>): BOS token (ALWAYS present at position 0)
# - Token 1 (<|md_reserved_0|>): Start of instruction
# - Token 2 (<|md_reserved_1|>): Separator before question
# - Token 3 (<|md_reserved_2|>): End of question / start of answer
#
# Task routing based on text prefix:
# "caption [short|normal|long]" → describe<|md_reserved_1|>{length}
# "describe [short|normal|long]" → describe<|md_reserved_1|>{length}
# otherwise → query<|md_reserved_1|><text>
#
# Format with image:
# <|endoftext|><image><|md_reserved_0|>{task}<|md_reserved_1|>{q}<|md_reserved_2|>
# Format without image:
# <|endoftext|><|md_reserved_0|>{task}<|md_reserved_1|>{q}<|md_reserved_2|>
_default_chat_template = (
"{% for message in messages %}"
"{% if message['role'] == 'user' %}"
"{% if message['content'] is string %}"
# Simple string content (with image assumed) - route by prefix
"<|endoftext|><image><|md_reserved_0|>"
"{% if message['content'] == 'caption' %}"
"describe<|md_reserved_1|>normal<|md_reserved_2|>"
"{% elif message['content'].startswith('caption ') %}"
"describe<|md_reserved_1|>{{ message['content'][8:] }}<|md_reserved_2|>"
"{% elif message['content'] == 'describe' %}"
"describe<|md_reserved_1|>normal<|md_reserved_2|>"
"{% elif message['content'].startswith('describe ') %}"
"describe<|md_reserved_1|>{{ message['content'][9:] }}<|md_reserved_2|>"
"{% else %}"
"query<|md_reserved_1|>{{ message['content'] }}<|md_reserved_2|>"
"{% endif %}"
"{% else %}"
# List content - build Moondream's image prefix independently of
# OpenAI-style content part order, then render the text task.
"<|endoftext|>"
"{% for content in message['content'] %}"
"{% if content['type'] in ['image', 'image_url', 'input_image', 'image_pil'] %}" # noqa: E501
"<image>"
"{% endif %}"
"{% endfor %}"
"{% for content in message['content'] %}"
"{% if content['type'] == 'text' %}"
"<|md_reserved_0|>"
"{% if content['text'] == 'caption' %}"
"describe<|md_reserved_1|>normal<|md_reserved_2|>"
"{% elif content['text'].startswith('caption ') %}"
"describe<|md_reserved_1|>{{ content['text'][8:] }}<|md_reserved_2|>"
"{% elif content['text'] == 'describe' %}"
"describe<|md_reserved_1|>normal<|md_reserved_2|>"
"{% elif content['text'].startswith('describe ') %}"
"describe<|md_reserved_1|>{{ content['text'][9:] }}<|md_reserved_2|>"
"{% else %}"
"query<|md_reserved_1|>{{ content['text'] }}<|md_reserved_2|>"
"{% endif %}"
"{% endif %}"
"{% endfor %}"
"{% endif %}"
"{% elif message['role'] == 'assistant' %}"
"{{ message['content'] }}"
"{% endif %}"
"{% endfor %}"
)
def __init__(
self,
tokenizer: PreTrainedTokenizerBase | None = None,
chat_template: str | None = None,
crop_size: int = 378,
max_crops: int = 12,
overlap_margin: int = 4,
patch_size: int = 14,
**kwargs,
):
self.image_token = "<image>"
self.crop_size = crop_size
self.max_crops = max_crops
self.overlap_margin = overlap_margin
self.patch_size = patch_size
# Number of patches per crop (27x27 = 729 for 378/14)
self.patches_per_crop = (crop_size // patch_size) ** 2
# Use default chat template if none provided
if chat_template is None:
chat_template = self._default_chat_template
super().__init__(tokenizer, chat_template=chat_template)
@classmethod
def from_pretrained(
cls,
pretrained_model_name_or_path,
**kwargs,
):
"""Load the processor, using a separate tokenizer repo.
The moondream3 model uses a custom tokenizer from 'moondream/starmie-v1'
instead of having tokenizer files in the model repo.
"""
from transformers import AutoTokenizer, TokenizersBackend
from transformers.utils import cached_file
tokenizer = kwargs.pop("tokenizer", None)
tokenizer_kwargs = {
"trust_remote_code": kwargs.get("trust_remote_code", False),
}
for key in (
"cache_dir",
"force_download",
"local_files_only",
"revision",
"subfolder",
"token",
):
if key in kwargs:
tokenizer_kwargs[key] = kwargs[key]
cached_file_kwargs = {
key: tokenizer_kwargs[key]
for key in (
"cache_dir",
"force_download",
"local_files_only",
"revision",
"subfolder",
"token",
)
if key in tokenizer_kwargs
}
def load_tokenizer(repo_or_path):
try:
return AutoTokenizer.from_pretrained(repo_or_path, **tokenizer_kwargs)
except Exception:
tokenizer_file = cached_file(
repo_or_path,
"tokenizer.json",
**cached_file_kwargs,
)
return TokenizersBackend(
tokenizer_file=tokenizer_file,
clean_up_tokenization_spaces=False,
)
if isinstance(tokenizer, str):
tokenizer = load_tokenizer(tokenizer)
if tokenizer is None:
# Prefer model-local tokenizer files first. If unavailable, fall
# back to moondream's dedicated tokenizer repository.
try:
tokenizer = load_tokenizer(pretrained_model_name_or_path)
except Exception:
tokenizer = load_tokenizer(cls._tokenizer_repo)
# Configure special tokens for Moondream3
# BOS and EOS are both token 0 (<|endoftext|>), matching the native
# config (TokenizerConfig.bos_id=0, eos_id=0). This is standard for
# GPT-2 style models where <|endoftext|> signals both start and end.
# Token 1 (<|md_reserved_0|>) is a template delimiter, NOT the EOS.
tokenizer.bos_token = "<|endoftext|>"
tokenizer.bos_token_id = 0
tokenizer.eos_token = "<|endoftext|>"
tokenizer.eos_token_id = 0
# Extract processor-specific kwargs
crop_size = kwargs.pop("crop_size", 378)
max_crops = kwargs.pop("max_crops", 12)
overlap_margin = kwargs.pop("overlap_margin", 4)
patch_size = kwargs.pop("patch_size", 14)
chat_template = kwargs.pop("chat_template", None)
# Set default chat template on tokenizer if not already set
if chat_template is None:
chat_template = cls._default_chat_template
if tokenizer.chat_template is None:
tokenizer.chat_template = chat_template
return cls(
tokenizer=tokenizer,
chat_template=chat_template,
crop_size=crop_size,
max_crops=max_crops,
overlap_margin=overlap_margin,
patch_size=patch_size,
)
def __call__(
self,
images: ImageInput = None,
text: TextInput
| PreTokenizedInput
| list[TextInput]
| list[PreTokenizedInput] = None,
**kwargs: Unpack[Moondream3ProcessorKwargs],
) -> BatchFeature:
"""Process images and text for Moondream3 model.
Args:
images: Input images (PIL Image, numpy array, or list thereof).
text: Input text or list of texts.
**kwargs: Additional processing arguments.
Returns:
BatchFeature with processed inputs.
"""
output_kwargs = self._merge_kwargs(
Moondream3ProcessorKwargs,
tokenizer_init_kwargs=self.tokenizer.init_kwargs,
**kwargs,
)
# Process images
image_features = {}
if images is not None:
processed_images = []
tilings = []
images_list = images if isinstance(images, list) else [images]
for image in images_list:
pixel_values, tiling = self.preprocess_image(
image, **output_kwargs["images_kwargs"]
)
processed_images.append(pixel_values)
tilings.append(tiling)
if processed_images:
image_features["pixel_values"] = processed_images
image_features["tilings"] = tilings
# Process text
if text is not None:
if not isinstance(text, list):
text = [text]
# Get text kwargs, remove keys we set ourselves
text_kwargs = output_kwargs.get("text_kwargs", {}).copy()
text_kwargs.pop("return_tensors", None)
text_kwargs.pop("add_special_tokens", None)
# Tokenize text
tokenized = self.tokenizer(
text,
add_special_tokens=True,
return_tensors="pt",
**text_kwargs,
)
output = BatchFeature(data=dict(tokenized))
# Add image features
if image_features:
output["pixel_values"] = image_features["pixel_values"]
output["tilings"] = image_features["tilings"]
return output
# If only images were provided
return BatchFeature(data=image_features)
@staticmethod
def _image_array_to_uint8(array: np.ndarray) -> np.ndarray:
if array.dtype == np.uint8:
return np.ascontiguousarray(array)
if array.dtype == np.bool_:
return np.ascontiguousarray(array.astype(np.uint8) * 255)
if np.issubdtype(array.dtype, np.floating):
array = np.nan_to_num(array, nan=0.0, posinf=255.0, neginf=0.0)
if array.size > 0 and array.max() <= 1.0:
array = array * 255.0
array = np.rint(array)
return np.ascontiguousarray(np.clip(array, 0, 255).astype(np.uint8))
@staticmethod
def _to_pil_image(image: ImageInput) -> Image.Image:
if isinstance(image, Image.Image):
return image
if isinstance(image, torch.Tensor):
tensor = image.detach().cpu()
if tensor.dtype == torch.bfloat16:
tensor = tensor.to(torch.float32)
image_array = tensor.numpy()
elif isinstance(image, np.ndarray):
image_array = image
else:
raise TypeError(
"Moondream3 images must be PIL images, numpy arrays, "
f"or torch tensors, got {type(image)!r}."
)
if image_array.ndim == 2:
image_array = Moondream3Processor._image_array_to_uint8(image_array)
return Image.fromarray(image_array)
if image_array.ndim != 3:
raise ValueError(
"Moondream3 image arrays must have 2 or 3 dimensions, "
f"got shape {image_array.shape}."
)
channel_dims = (1, 3, 4)
if image_array.shape[-1] not in channel_dims:
if image_array.shape[0] not in channel_dims:
raise ValueError(
"Moondream3 image arrays must be HWC or CHW with 1, 3, "
f"or 4 channels, got shape {image_array.shape}."
)
image_array = np.transpose(image_array, (1, 2, 0))
image_array = Moondream3Processor._image_array_to_uint8(image_array)
if image_array.shape[-1] == 1:
image_array = image_array[..., 0]
return Image.fromarray(image_array)
def preprocess_image(
self,
image: ImageInput,
max_crops: int = 12,
overlap_margin: int = 4,
crop_size: int = 378,
patch_size: int = 14,
convert_to_rgb: bool = True,
return_tensors: str = "pt",
) -> tuple[torch.Tensor, tuple[int, int]]:
"""Preprocess an image using overlap-and-resize cropping strategy.
Args:
image: Input PIL image, numpy array, or torch tensor.
max_crops: Maximum number of crops.
overlap_margin: Margin for overlapping in patches.
crop_size: Size of each crop.
patch_size: Size of each patch.
convert_to_rgb: Whether to convert to RGB.
return_tensors: Return type ("pt" for PyTorch).
Returns:
Tuple of (pixel_values tensor, tiling tuple).
"""
image = self._to_pil_image(image)
if convert_to_rgb:
image = convert_image_mode(image, "RGB")
# Convert to numpy array
image_array = np.array(image)
original_h, original_w = image_array.shape[:2]
margin_pixels = patch_size * overlap_margin
total_margin_pixels = margin_pixels * 2
crop_patches = crop_size // patch_size
crop_window_patches = crop_patches - (2 * overlap_margin)
crop_window_size = crop_window_patches * patch_size
tiling = select_tiling(
original_h - total_margin_pixels,
original_w - total_margin_pixels,
crop_window_size,
max_crops,
)
n_crops = tiling[0] * tiling[1] + 1
crops = np.zeros((n_crops, crop_size, crop_size, 3), dtype=np.uint8)
target_size = (
tiling[0] * crop_window_size + total_margin_pixels,
tiling[1] * crop_window_size + total_margin_pixels,
)
# Resize image
pil_img = Image.fromarray(image_array)
resized = pil_img.resize(
(int(target_size[1]), int(target_size[0])),
resample=Image.Resampling.LANCZOS,
)
resized_array = np.asarray(resized)
# Create global crop
global_pil = pil_img.resize(
(crop_size, crop_size), resample=Image.Resampling.LANCZOS
)
crops[0] = np.asarray(global_pil)
# Create local crops
for i in range(tiling[0]):
for j in range(tiling[1]):
y0 = i * crop_window_size
x0 = j * crop_window_size
y_end = min(y0 + crop_size, resized_array.shape[0])
x_end = min(x0 + crop_size, resized_array.shape[1])
crop_region = resized_array[y0:y_end, x0:x_end]
crop_idx = 1 + i * tiling[1] + j
h_slice = slice(None, crop_region.shape[0])
w_slice = slice(None, crop_region.shape[1])
crops[crop_idx, h_slice, w_slice] = crop_region
# Convert to tensor: (n_crops, H, W, C) -> (n_crops, C, H, W)
pixel_values = np.transpose(crops, (0, 3, 1, 2))
if return_tensors == "pt":
# Match HF reference preprocessing exactly: convert uint8 crops to
# bfloat16 before in-place normalization.
pixel_values = (
torch.from_numpy(pixel_values)
.to(dtype=torch.bfloat16)
.div_(255.0)
.sub_(0.5)
.div_(0.5)
)
else:
pixel_values = pixel_values.astype(np.float32) / 255.0
pixel_values = (pixel_values - 0.5) / 0.5
return pixel_values, tiling
def get_num_image_tokens(self) -> int:
"""Return the number of image tokens (729 = 27x27 patches)."""
return self.patches_per_crop
def batch_decode(self, *args, **kwargs):
"""Forward to tokenizer's batch_decode."""
return self.tokenizer.batch_decode(*args, **kwargs)
def decode(self, *args, **kwargs):
"""Forward to tokenizer's decode."""
return self.tokenizer.decode(*args, **kwargs)
@property
def model_input_names(self):
tokenizer_input_names = self.tokenizer.model_input_names
return tokenizer_input_names + ["pixel_values", "tilings"]
AutoProcessor.register("Moondream3Processor", Moondream3Processor)
@@ -1,372 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import torch
import torchvision.transforms as T
from PIL import Image
from vllm.multimodal.image import convert_image_mode
from vllm.tokenizers.hf import HfTokenizer
from .internvl import InternVLImageProcessor, InternVLProcessor
def build_transform(input_size: int):
return T.Compose(
[
T.Lambda(lambda img: convert_image_mode(img, "RGB")),
T.Resize(
(input_size, input_size), interpolation=T.InterpolationMode.BICUBIC
),
T.ToTensor(),
]
)
# adapted from https://huggingface.co/nvidia/Llama-3.1-Nemotron-Nano-VL-8B-V1
def find_closest_aspect_ratio(
aspect_ratio: float,
target_ratios: list[tuple[int, int]],
*,
width: int,
height: int,
image_size: int,
) -> tuple[int, int]:
best_factor = float("-inf")
best_ratio = (1, 1)
area = width * height
for rw, rh in target_ratios:
target_aspect_ratio = rw / rh
size_factor = min((rw * rh * image_size * image_size) / area, 0.6)
ratio_closeness = min(
target_aspect_ratio / aspect_ratio, aspect_ratio / target_aspect_ratio
)
factor = size_factor * ratio_closeness
if factor > best_factor:
best_factor = factor
best_ratio = (rw, rh)
return best_ratio
def calculate_nemotron_vl_targets(
*,
orig_width: int,
orig_height: int,
target_ratios: list[tuple[int, int]],
image_size: int,
use_thumbnail: bool,
) -> tuple[int, int, int]:
aspect_ratio = orig_width / orig_height
# find the closest aspect ratio to the target
target_aspect_ratio = find_closest_aspect_ratio(
aspect_ratio,
target_ratios,
width=orig_width,
height=orig_height,
image_size=image_size,
)
# calculate the target width and height
target_width = image_size * target_aspect_ratio[0]
target_height = image_size * target_aspect_ratio[1]
blocks = target_aspect_ratio[0] * target_aspect_ratio[1]
# add thumbnail image if num_blocks != 1
if use_thumbnail and blocks != 1:
blocks += 1
return blocks, target_width, target_height
def dynamic_preprocess_nemotron_vl(
image: Image.Image,
*,
target_ratios: list[tuple[int, int]],
image_size: int,
use_thumbnail: bool,
) -> list[Image.Image]:
orig_width, orig_height = image.size
# calculate the number of blocks without thumbnail
blocks, target_width, target_height = calculate_nemotron_vl_targets(
orig_width=orig_width,
orig_height=orig_height,
target_ratios=target_ratios,
image_size=image_size,
use_thumbnail=False,
)
# resize the image
resized_img = image.resize((target_width, target_height))
processed_images = []
for i in range(blocks):
box = (
(i % (target_width // image_size)) * image_size,
(i // (target_width // image_size)) * image_size,
((i % (target_width // image_size)) + 1) * image_size,
((i // (target_width // image_size)) + 1) * image_size,
)
# split the image
split_img = resized_img.crop(box)
processed_images.append(split_img)
assert len(processed_images) == blocks
if use_thumbnail and len(processed_images) != 1:
thumbnail_img = image.resize((image_size, image_size))
processed_images.append(thumbnail_img)
return processed_images
def get_nemotron_vl_target_ratios(
min_num: int,
max_num: int,
) -> list[tuple[int, int]]:
target_ratios = {
(i, j)
for n in range(min_num, max_num + 1)
for i in range(1, n + 1)
for j in range(1, n + 1)
if min_num <= i * j <= max_num
}
return sorted(target_ratios, key=lambda x: x[0] * x[1])
def image_to_pixel_values_nemotron_vl(
image: Image.Image,
*,
input_size: int,
min_num: int,
max_num: int,
use_thumbnail: bool,
transform: T.Compose | None = None,
) -> torch.Tensor:
target_ratios = get_nemotron_vl_target_ratios(min_num, max_num)
if transform is None:
transform = build_transform(input_size=input_size)
images = dynamic_preprocess_nemotron_vl(
image,
target_ratios=target_ratios,
image_size=input_size,
use_thumbnail=use_thumbnail,
)
pixel_values = torch.stack([transform(image) for image in images])
return pixel_values
class LlamaNemotronNanoVLImageProcessor(InternVLImageProcessor):
def _images_to_pixel_values_lst(
self,
images: list[Image.Image],
min_dynamic_patch: int | None = None,
max_dynamic_patch: int | None = None,
dynamic_image_size: bool | None = None,
) -> list[torch.Tensor]:
min_num, max_num = self.resolve_min_max_num(
min_dynamic_patch=min_dynamic_patch,
max_dynamic_patch=max_dynamic_patch,
dynamic_image_size=dynamic_image_size,
use_thumbnail=False, # Applied in image_to_pixel_values
)
return [
image_to_pixel_values_nemotron_vl(
image,
input_size=self.image_size,
min_num=min_num,
max_num=max_num,
use_thumbnail=self.use_thumbnail,
transform=build_transform(self.image_size),
)
for image in images
]
class LlamaNemotronNanoVLProcessor(InternVLProcessor):
"""This model doesn't define its own HF processor,
so we implement our own one here.
The image processor is given by:
https://huggingface.co/nvidia/Llama-3.1-Nemotron-Nano-VL-8B-V1/blob/main/image_processing.py
"""
def __init__(
self,
image_processor: LlamaNemotronNanoVLImageProcessor,
tokenizer: HfTokenizer,
*,
image_seq_length: int,
start_image_token: str = "<img>",
end_image_token: str = "</img>",
ctx_image_token: str = "<image>",
) -> None:
super().__init__(
image_processor=image_processor,
tokenizer=tokenizer,
image_seq_length=image_seq_length,
start_image_token=start_image_token,
end_image_token=end_image_token,
ctx_image_token=ctx_image_token,
)
def get_num_image_tokens(
self,
*,
image_width: int,
image_height: int,
) -> int:
image_processor = self.image_processor
target_ratios = self.resolve_target_ratios(
use_thumbnail=False, # Applied in calculate_targets
)
num_patches, _, _ = calculate_nemotron_vl_targets(
orig_width=image_width,
orig_height=image_height,
image_size=image_processor.image_size,
target_ratios=target_ratios,
use_thumbnail=image_processor.use_thumbnail,
)
return num_patches * self.image_seq_length
# SigLIP normalization constants
SIGLIP_MEAN = (0.5, 0.5, 0.5)
SIGLIP_STD = (0.5, 0.5, 0.5)
def build_siglip_transform(
input_size: int,
*,
normalize_on_cpu: bool = True,
):
"""Build transform for SigLIP vision encoder with normalization.
Extends the base transform from nemotron_vl with SigLIP-specific normalization.
"""
transforms = [
T.Lambda(lambda img: convert_image_mode(img, "RGB")),
T.Resize((input_size, input_size), interpolation=T.InterpolationMode.BICUBIC),
T.ToTensor() if normalize_on_cpu else T.PILToTensor(),
]
if normalize_on_cpu:
transforms.append(T.Normalize(mean=SIGLIP_MEAN, std=SIGLIP_STD))
return T.Compose(transforms)
class LlamaNemotronVLEmbedImageProcessor(InternVLImageProcessor):
def __init__(
self,
image_size: int,
min_dynamic_patch: int,
max_dynamic_patch: int,
dynamic_image_size: bool,
use_thumbnail: bool,
*,
do_rescale: bool = True,
do_normalize: bool = True,
) -> None:
if do_rescale != do_normalize:
raise ValueError(
"LlamaNemotronVLEmbedImageProcessor requires do_rescale and "
"do_normalize to be both enabled or both disabled."
)
super().__init__(
image_size=image_size,
min_dynamic_patch=min_dynamic_patch,
max_dynamic_patch=max_dynamic_patch,
dynamic_image_size=dynamic_image_size,
use_thumbnail=use_thumbnail,
)
self.normalize_on_cpu = do_normalize
def _images_to_pixel_values_lst(
self,
images: list[Image.Image],
min_dynamic_patch: int | None = None,
max_dynamic_patch: int | None = None,
dynamic_image_size: bool | None = None,
) -> list[torch.Tensor]:
min_num, max_num = self.resolve_min_max_num(
min_dynamic_patch=min_dynamic_patch,
max_dynamic_patch=max_dynamic_patch,
dynamic_image_size=dynamic_image_size,
use_thumbnail=False, # Applied in image_to_pixel_values
)
return [
image_to_pixel_values_nemotron_vl(
image,
input_size=self.image_size,
min_num=min_num,
max_num=max_num,
use_thumbnail=self.use_thumbnail,
transform=build_siglip_transform(
self.image_size,
normalize_on_cpu=self.normalize_on_cpu,
),
)
for image in images
]
class LlamaNemotronVLEmbedProcessor(InternVLProcessor):
"""Processor for LlamaNemotronVL embedding model.
Inherits from NemotronVLProcessor and specializes it for embedding tasks:
- Uses SigLIP transform with normalization instead of base transform
- Uses different image context token (<IMG_CONTEXT> vs <image>)
"""
def __init__(
self,
image_processor: LlamaNemotronVLEmbedImageProcessor,
tokenizer: HfTokenizer,
*,
image_seq_length: int,
start_image_token: str = "<img>",
end_image_token: str = "</img>",
ctx_image_token: str = "<IMG_CONTEXT>",
) -> None:
super().__init__(
image_processor=image_processor,
tokenizer=tokenizer,
image_seq_length=image_seq_length,
start_image_token=start_image_token,
end_image_token=end_image_token,
ctx_image_token=ctx_image_token,
)
self.image_processor: LlamaNemotronVLEmbedImageProcessor
def get_num_image_tokens(
self,
*,
image_width: int,
image_height: int,
) -> int:
image_processor = self.image_processor
target_ratios = self.resolve_target_ratios(
use_thumbnail=False, # Applied in calculate_targets
)
num_patches, _, _ = calculate_nemotron_vl_targets(
orig_width=image_width,
orig_height=image_height,
image_size=image_processor.image_size,
target_ratios=target_ratios,
use_thumbnail=image_processor.use_thumbnail,
)
return num_patches * self.image_seq_length
@@ -1,63 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
# adapted from https://huggingface.co/nvidia/NVLM-D-72B/blob/main/modeling_nvlm_d.py
# --------------------------------------------------------
# NVLM-D
# Copyright (c) 2024 NVIDIA
# Licensed under Apache 2.0 License [see LICENSE for details]
# --------------------------------------------------------
from vllm.multimodal.processing import PromptUpdateDetails, cached_encode
from vllm.tokenizers.hf import HfTokenizer
from .internvl import InternVLImageProcessor, InternVLProcessor
class NVLMProcessor(InternVLProcessor):
def __init__(
self,
image_processor: InternVLImageProcessor,
tokenizer: HfTokenizer,
*,
image_seq_length: int,
start_image_token: str = "<Image>",
end_image_token: str = "</Image>",
ctx_image_token: str = "<|vision_pad|>",
) -> None:
super().__init__(
image_processor=image_processor,
tokenizer=tokenizer,
image_seq_length=image_seq_length,
start_image_token=start_image_token,
end_image_token=end_image_token,
ctx_image_token=ctx_image_token,
)
def get_image_repl(
self,
num_patches: int | None,
num_features: int | None = None,
) -> PromptUpdateDetails:
if num_patches is None:
raise NotImplementedError("Embedding inputs are not supported")
num_features = num_patches * self.image_seq_length
tile_pos_identifiers = [f"<tile_{i}>" for i in range(1, num_patches)]
if self.image_processor.use_thumbnail:
tile_pos_identifiers += ["<tile_global_thumbnail>"]
context_size = num_features // num_patches
features = "".join(
(identifier + self.ctx_image_token * context_size)
for identifier in tile_pos_identifiers
)
# We include the start and end as well because "<Image><tile" is
# tokenized as ["<Image", "><", "tile"], resulting in assertion error
# when trying to find "<tile" as a subsequence of "<Image><tile"
repl = self.start_image_token + features + self.end_image_token
full_ids = cached_encode(self.tokenizer, repl, add_special_tokens=False)
return PromptUpdateDetails.select_token_id(full_ids, self.ctx_image_token_id)
@@ -1,102 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from collections.abc import Sequence
from typing import Any
import numpy as np
import torch
from PIL import Image
from transformers import ImageProcessingMixin, ProcessorMixin
IMAGENET_MEAN = np.array([0.484375, 0.455078125, 0.40625], dtype=np.float32)
IMAGENET_STD = np.array([0.228515625, 0.2236328125, 0.224609375], dtype=np.float32)
SIGLIP_MEAN = np.array([0.5, 0.5, 0.5], dtype=np.float32)
SIGLIP_STD = np.array([0.5, 0.5, 0.5], dtype=np.float32)
def to_rgb_image(image: Any) -> Image.Image:
if isinstance(image, Image.Image):
return image.convert("RGB")
if isinstance(image, torch.Tensor):
image = image.detach().cpu().numpy()
if not isinstance(image, np.ndarray):
raise TypeError(
"OpenVLA image input must be a PIL image, numpy array, or torch tensor; "
f"got {type(image)}"
)
if image.ndim != 3:
raise ValueError(
f"OpenVLA image input must have 3 dimensions, got shape {image.shape}"
)
if image.shape[0] in (1, 3):
image = np.moveaxis(image, 0, -1)
if image.shape[-1] == 1:
image = np.repeat(image, 3, axis=-1)
elif image.shape[-1] != 3:
raise ValueError(
f"OpenVLA image input must have 1 or 3 channels, got shape {image.shape}"
)
if image.dtype != np.uint8:
image = image.astype(np.float32)
if image.max(initial=0.0) <= 1.0:
image = image * 255.0
image = np.clip(image, 0, 255).astype(np.uint8)
return Image.fromarray(image).convert("RGB")
def preprocess_openvla_image(image: Any, image_size: int) -> torch.Tensor:
rgb_image = to_rgb_image(image)
rgb_image = rgb_image.resize(
(image_size, image_size),
Image.Resampling.BICUBIC,
)
raw = np.asarray(rgb_image, dtype=np.float32) / 255.0
dinov2_pixels = ((raw - IMAGENET_MEAN) / IMAGENET_STD).transpose(2, 0, 1)
siglip_pixels = ((raw - SIGLIP_MEAN) / SIGLIP_STD).transpose(2, 0, 1)
pixel_values = np.concatenate([dinov2_pixels, siglip_pixels], axis=0)
return torch.from_numpy(pixel_values)
class OpenVLAImageProcessor(ImageProcessingMixin):
def __init__(self, *, image_size: int) -> None:
self.image_size = image_size
def __call__(
self,
images: Any | None = None,
**kwargs: object,
) -> dict[str, object]:
if images is None:
return {}
if not isinstance(images, Sequence) or isinstance(images, (str, bytes)):
images = [images]
if len(images) == 0:
return {}
pixel_values = torch.stack(
[
preprocess_openvla_image(image, image_size=self.image_size)
for image in images
],
dim=0,
)
return {"pixel_values": pixel_values}
class OpenVLAProcessor(ProcessorMixin):
def __init__(
self,
*,
image_processor: OpenVLAImageProcessor,
tokenizer: Any,
) -> None:
self.image_processor = image_processor
self.tokenizer = tokenizer
-457
View File
@@ -1,457 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
# ruff: noqa: E501
# coding=utf-8
# adapted from https://github.com/AIDC-AI/Ovis/blob/35ab51a1a1e3542fa6db260a1084cefbc8f164bb/ovis/vllm/processing_ovis.py
# Copyright 2025 The Qwen Team and The HuggingFace Inc. team. All rights reserved.
#
# This code is based on EleutherAI's GPT-NeoX library and the GPT-NeoX
# and OPT implementations in this library. It has been modified from its
# original forms to accommodate minor architectural differences compared
# to GPT-NeoX and OPT used by the Meta AI team that trained the model.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from functools import cached_property
import PIL
import torch
from transformers import BatchFeature
from transformers.image_utils import ImageInput
from transformers.processing_utils import ProcessingKwargs, ProcessorMixin, Unpack
from transformers.tokenization_utils_base import PreTokenizedInput, TextInput
from vllm.multimodal.image import convert_image_mode
__all__ = ["OvisProcessor"]
IGNORE_ID = -100
class OvisProcessorKwargs(ProcessingKwargs, total=False): # type: ignore[call-arg]
_defaults = {
"text_kwargs": {
"padding": False,
},
"images_kwargs": {
"do_convert_rgb": True,
"return_tensors": "pt",
},
}
class OvisProcessor(ProcessorMixin):
r"""Constructs an Ovis processor which wraps an Ovis image processor and a Qwen2 tokenizer into a single processor.
[`OvisProcessor`] offers all the functionalities of [`Qwen2VLImageProcessor`] and [`Qwen2TokenizerFast`]. See the
[`~OvisProcessor.__call__`] and [`~OvisProcessor.decode`] for more information.
Args:
image_processor ([`Qwen2VLImageProcessor`], *optional*):
The image processor is a required input.
tokenizer ([`Qwen2TokenizerFast`], *optional*):
The tokenizer is a required input.
chat_template (`str`, *optional*): A Jinja template which will be used to convert lists of messages
in a chat into a tokenizable string.
"""
attributes = ["image_processor", "tokenizer"]
valid_kwargs = ["chat_template", "image_pad_token", "image_segment_len"]
image_processor_class = "AutoImageProcessor"
tokenizer_class = "AutoTokenizer"
def __init__(
self,
image_processor=None,
tokenizer=None,
chat_template=None,
image_pad_token=None,
image_segment_len=255,
**kwargs,
):
self.image_token = "<image>"
self.image_pad_token = image_pad_token
self.image_segment_len = image_segment_len
super().__init__(image_processor, tokenizer, chat_template=chat_template)
@cached_property
def extra_special_tokens(self):
image_pad_token_id = self.tokenizer.get_vocab()[self.image_pad_token]
extra_special_tokens = {
"image_token": -200,
"image_atom": -300,
"image_start": -301,
"image_prefix": -302,
"image_col_sep": -303,
"image_row_sep": -304,
"image_end": -305,
"image_pad": image_pad_token_id,
}
return extra_special_tokens
def __call__(
self,
images: ImageInput = None,
text: TextInput
| PreTokenizedInput
| list[TextInput]
| list[PreTokenizedInput] = None,
**kwargs: Unpack[OvisProcessorKwargs],
) -> BatchFeature:
"""Main method to prepare for the model one or several sequences(s) and image(s). This method forwards the `text`
and `kwargs` arguments to Qwen2TokenizerFast's [`~Qwen2TokenizerFast.__call__`] if `text` is not `None` to encode
the text. To prepare the vision inputs, this method forwards the `vision_infos` and `kwargs` arguments to
Qwen2VLImageProcessor's [`~Qwen2VLImageProcessor.__call__`] if `vision_infos` is not `None`.
Args:
images (`PIL.Image.Image`, `np.ndarray`, `torch.Tensor`, `list[PIL.Image.Image]`, `list[np.ndarray]`, `list[torch.Tensor]`):
The image or batch of images to be prepared. Each image can be a PIL image, NumPy array or PyTorch
tensor. Both channels-first and channels-last formats are supported.
text (`str`, `list[str]`, `list[list[str]]`):
The sequence or batch of sequences to be encoded. Each sequence can be a string or a list of strings
(pretokenized string). If the sequences are provided as list of strings (pretokenized), you must set
`is_split_into_words=True` (to lift the ambiguity with a batch of sequences).
videos (`np.ndarray`, `torch.Tensor`, `list[np.ndarray]`, `list[torch.Tensor]`):
The image or batch of videos to be prepared. Each video can be a 4D NumPy array or PyTorch
tensor, or a nested list of 3D frames. Both channels-first and channels-last formats are supported.
return_tensors (`str` or [`~utils.TensorType`], *optional*):
If set, will return tensors of a particular framework. Acceptable values are:
- `'tf'`: Return TensorFlow `tf.constant` objects.
- `'pt'`: Return PyTorch `torch.Tensor` objects.
- `'np'`: Return NumPy `np.ndarray` objects.
- `'jax'`: Return JAX `jnp.ndarray` objects.
Returns:
[`BatchFeature`]: A [`BatchFeature`] with the following fields:
- **input_ids** -- List of token ids to be fed to a model. Returned when `text` is not `None`.
- **attention_mask** -- List of indices specifying which tokens should be attended to by the model (when
`return_attention_mask=True` or if *"attention_mask"* is in `self.model_input_names` and if `text` is not
`None`).
- **pixel_values** -- Pixel values to be fed to a model. Returned when `images` is not `None`.
- **pixel_values_videos** -- Pixel values of videos to be fed to a model. Returned when `videos` is not `None`.
- **image_grid_thw** -- List of image 3D grid in LLM. Returned when `images` is not `None`.
- **video_grid_thw** -- List of video 3D grid in LLM. Returned when `videos` is not `None`.
- **second_per_grid_ts** -- List of video seconds per time grid. Returned when `videos` is not `None`.
"""
max_partition = kwargs.pop("max_partition", 9)
covering_threshold = kwargs.pop("covering_threshold", 0.9)
output_kwargs = self._merge_kwargs(
OvisProcessorKwargs,
tokenizer_init_kwargs=self.tokenizer.init_kwargs,
**kwargs,
)
# Process all images first
image_features = {}
if images is not None:
processed_images = []
image_placeholders_list = []
grids = []
# Process each image
for image in images if isinstance(images, list) else [images]:
pixel_values, image_placeholders, grid = self.preprocess_image(
image=image,
max_partition=max_partition,
covering_threshold=covering_threshold,
**output_kwargs["images_kwargs"],
)
processed_images.append(pixel_values)
image_placeholders_list.append(image_placeholders)
grids.append(grid)
# assign all processed images
if processed_images:
image_features["image_placeholders"] = image_placeholders_list
# Process text input
if text is not None:
if not isinstance(text, list):
text = [text]
tokenized_batched_text = self._tokenize_with_image_symbol(text)
image_token_id = self.get_token_value("image_token")
replaced_ids_list = []
idx = 0
for ids_tensor in tokenized_batched_text:
if (
image_token_id in ids_tensor
and "image_placeholders" in image_features
):
if idx < len(image_features["image_placeholders"]):
# Converts in list for ease of use
ids_list = ids_tensor.tolist()
new_ids = []
# replace placeholders
for i, token_id in enumerate(ids_list):
if token_id == image_token_id:
placeholder_ids = image_features["image_placeholders"][
idx
]
new_ids.extend(placeholder_ids)
idx += 1
else:
new_ids.append(token_id)
# Converts back to tensors
ids_tensor = torch.tensor(new_ids, dtype=torch.long)
else:
raise RuntimeError(
"Mismatch between the images you provided and the number of placeholder present in the text"
)
replaced_ids_list.append(ids_tensor)
if replaced_ids_list:
replaced_and_tokenized_ids = torch.stack(replaced_ids_list)
else:
replaced_and_tokenized_ids = torch.tensor([], dtype=torch.long)
# Create the output with text features
output = BatchFeature(
data={
"input_ids": replaced_and_tokenized_ids,
}
)
# Add image features if present
if image_features:
output["pixel_values"] = processed_images
output["grids"] = grids
return output
# If only images were provided
return BatchFeature(data=image_features)
def _tokenize_with_image_symbol(self, text_list: list[str]) -> torch.LongTensor:
batch_token_ids = []
for text in text_list:
text_chunks = [
self.tokenizer(chunk, add_special_tokens=False).input_ids
for chunk in text.split(self.image_token)
]
token_ids = []
num_chuck = len(text_chunks)
for i, chunk in enumerate(text_chunks):
token_ids.extend(chunk)
if i < num_chuck - 1:
token_ids.append(self.get_token_value("image_token"))
batch_token_ids.append(token_ids)
return torch.tensor(batch_token_ids, dtype=torch.long)
def get_image_size(self):
size = self.image_processor.size
if "shortest_edge" in size:
width = height = size["shortest_edge"]
elif "height" in size and "width" in size:
width = size["width"]
height = size["height"]
else:
raise ValueError("Can't parse image size from image_processor config.")
return height, width
def get_token_value(self, tok):
return self.extra_special_tokens[tok]
def construct_image_indicators(self, grid):
image_placeholders = [
self.get_token_value("image_start"),
self.get_token_value("image_atom"),
self.get_token_value("image_prefix"),
]
if grid[0] * grid[1] > 1:
for r in range(grid[0]):
for c in range(grid[1]):
image_placeholders.append(self.get_token_value("image_atom"))
if c < grid[1] - 1:
image_placeholders.append(self.get_token_value("image_col_sep"))
if r < grid[0] - 1:
image_placeholders.append(self.get_token_value("image_row_sep"))
image_placeholders.append(self.get_token_value("image_end"))
return image_placeholders
def construct_image_placeholders(self, grid):
image_placeholders = self.construct_image_indicators(grid)
image_atom_token_id = self.get_token_value("image_atom")
# Extract the padding token ID from tokenizer
image_padding_token_id = self.get_token_value("image_pad")
# Create a new list with padding tokens inserted
padded_placeholder_tokens = []
for token in image_placeholders:
padded_placeholder_tokens.append(image_padding_token_id)
if token == image_atom_token_id:
padded_placeholder_tokens.extend(
[image_padding_token_id] * self.image_segment_len
)
return padded_placeholder_tokens
def preprocess_image(
self,
image: PIL.Image.Image,
max_partition,
covering_threshold,
do_convert_rgb,
return_tensors,
):
def _preprocess(img: PIL.Image.Image, side):
# first resize and preprocess
w, h = img.size
if w == h:
new_width = new_height = side
elif w > h:
new_width = side
new_height = int(h / w * new_width)
else:
new_height = side
new_width = int(w / h * new_height)
new_size = dict(height=new_height, width=new_width)
pixel_values = self.image_processor.preprocess(
img, size=new_size, return_tensors=return_tensors
)["pixel_values"]
# then pad to square
square_values = torch.zeros(
[1, 3, side, side], dtype=pixel_values.dtype, device=pixel_values.device
)
new_height, new_width = pixel_values.shape[2:]
if new_height == new_width:
square_values[:, :, :, :] = pixel_values
elif new_height > new_width:
from_index = (side - new_width) // 2
square_values[:, :, :, from_index : from_index + new_width] = (
pixel_values
)
else:
from_index = (side - new_height) // 2
square_values[:, :, from_index : from_index + new_height, :] = (
pixel_values
)
return square_values
def _partition(img, grid) -> list[tuple[int, int, int, int]]:
w, h = img.size
row_height = h // grid[0]
col_width = w // grid[1]
partition = []
for row in range(grid[0]):
for col in range(grid[1]):
left = col * col_width
upper = row * row_height
right = w if col == grid[1] - 1 else (col + 1) * col_width
lower = h if row == grid[0] - 1 else (row + 1) * row_height
partition.append((left, upper, right, lower))
return partition
def _covering_area(left, upper, right, lower, side):
w = right - left
h = lower - upper
w, h = max(w, h), min(w, h)
if w > side:
h = h / w * side
w = side
return w * h
def _get_best_grid(img, side):
img_area = img.size[0] * img.size[1]
candidate_grids = []
for i in range(1, max_partition + 1):
for j in range(1, max_partition + 1):
if i * j <= max_partition:
candidate_grids.append((i, j))
all_grids = []
good_grids = []
for grid in candidate_grids:
partition = _partition(img, grid)
covering_ratio = (
sum([_covering_area(*p, side) for p in partition]) / img_area
)
assert covering_ratio <= 1.0
all_grids.append((grid, covering_ratio))
if covering_ratio > covering_threshold:
good_grids.append((grid, covering_ratio))
if len(good_grids) > 0:
# pick the good partition with minimum #sub_images and break the tie using covering_ratio
return sorted(good_grids, key=lambda x: (x[0][0] * x[0][1], -x[1]))[0][
0
]
else:
# pick the partition with maximum covering_ratio and break the tie using #sub_images
return sorted(all_grids, key=lambda x: (-x[1], x[0][0] * x[0][1]))[0][0]
if do_convert_rgb:
image = convert_image_mode(image, "RGB")
sides = self.get_image_size()
if sides[0] != sides[1]:
raise ValueError("get_image_size() returns non-square size")
side = sides[0]
grid = _get_best_grid(image, side)
partition = _partition(image, grid)
crops = [image.crop(p) for p in partition]
if len(crops) > 1:
crops.insert(0, image)
pixel_values = torch.cat([_preprocess(crop, side) for crop in crops], dim=0)
image_placeholders = self.construct_image_placeholders(grid)
return torch.tensor(pixel_values), image_placeholders, torch.tensor(grid)
def batch_decode(self, *args, **kwargs):
"""This method forwards all its arguments to Qwen2TokenizerFast's [`~PythonBackend.batch_decode`]. Please
refer to the docstring of this method for more information.
"""
return self.tokenizer.batch_decode(*args, **kwargs)
def decode(self, *args, **kwargs):
"""This method forwards all its arguments to Qwen2TokenizerFast's [`~PythonBackend.decode`]. Please refer to
the docstring of this method for more information.
"""
return self.tokenizer.decode(*args, **kwargs)
def post_process_image_text_to_text(self, generated_outputs):
"""Post-process the output of the model to decode the text.
Args:
generated_outputs (`torch.Tensor` or `np.ndarray`):
The output of the model `generate` function. The output is expected to be a tensor of shape `(batch_size, sequence_length)`
or `(sequence_length,)`.
Returns:
`list[str]`: The decoded text.
"""
return self.tokenizer.batch_decode(
generated_outputs,
skip_special_tokens=True,
clean_up_tokenization_spaces=False,
)
@property
def model_input_names(self):
tokenizer_input_names = self.tokenizer.model_input_names
image_processor_input_names = self.image_processor.model_input_names
names_from_processor = list(
dict.fromkeys(tokenizer_input_names + image_processor_input_names)
)
return names_from_processor + ["second_per_grid_ts"]
@@ -1,476 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import math
from functools import cached_property
import numpy as np
import PIL
import torch
from transformers import BatchFeature
from transformers.image_utils import ImageInput
from transformers.processing_utils import ProcessingKwargs, ProcessorMixin, Unpack
from transformers.tokenization_utils_base import PreTokenizedInput, TextInput
__all__ = ["Ovis2_5Processor"]
IMAGE_TOKEN = "<image>"
VIDEO_TOKEN = "<video>"
MIN_PIXELS = 448 * 448
MAX_PIXELS = 1792 * 1792
class Ovis2_5ProcessorKwargs(ProcessingKwargs, total=False): # type: ignore[call-arg]
_defaults = {
"text_kwargs": {
"padding": False,
},
"images_kwargs": {
"do_convert_rgb": True,
},
"videos_kwargs": {
"do_convert_rgb": True,
},
}
class Ovis2_5Processor(ProcessorMixin):
r"""Constructs an Ovis processor which wraps an Ovis image processor
and a Qwen2 tokenizer into a single processor.
[`OvisProcessor`] offers all the functionalities of
[`Qwen2VLImageProcessor`] and [`Qwen2TokenizerFast`].
See the [`~OvisProcessor.__call__`] and [`~OvisProcessor.decode`]
for more information.
Args:
image_processor ([`Qwen2VLImageProcessor`], *optional*):
The image processor is a required input.
tokenizer ([`Qwen2TokenizerFast`], *optional*):
The tokenizer is a required input.
chat_template (`str`, *optional*): A Jinja template which will
be used to convert lists of messages in a chat into
a tokenizable string.
"""
attributes = ["image_processor", "tokenizer"]
valid_kwargs = ["chat_template", "image_pad_token"]
image_processor_class = "AutoImageProcessor"
tokenizer_class = "AutoTokenizer"
def __init__(
self,
image_processor=None,
tokenizer=None,
chat_template=None,
image_pad_token=None,
patch_size=16,
hidden_stride=2,
temporal_patch_size=1,
**kwargs,
):
self.image_token = IMAGE_TOKEN
self.video_token = VIDEO_TOKEN
self.image_pad_token = "<|image_pad|>"
self.patch_size = patch_size
self.hidden_stride = hidden_stride
self.temporal_patch_size = temporal_patch_size
super().__init__(image_processor, tokenizer, chat_template=chat_template)
@cached_property
def extra_special_tokens(self):
required_tokens = {
"image_token": "<image>",
"video_token": "<video>",
"visual_atom": "<ovis_visual_atom>",
"image_start": "<ovis_image_start>",
"image_end": "<ovis_image_end>",
"video_start": "<ovis_video_start>",
"video_end": "<ovis_video_end>",
"image_pad": "<|image_pad|>",
}
# The checkpoint defines both `additional_special_tokens` and
# `extra_special_tokens`, with the latter empty. Transformers ignores
# the former because the latter is explicitly empty, so the tokens are
# missing from the vocab. Re-add them to restore the expected ids.
self.tokenizer.add_tokens(list(required_tokens.values()), special_tokens=True)
# in TokenizerPool tokenizer.get_added_vocab() gets values from an instance
# that was updated with tokenizer.add_tokens(), so it is favourable over
# self.tokenizer.convert_tokens_to_ids(), which refers to a copy before
# mutation by tokenizer.add_tokens()
added_vocab = self.tokenizer.get_added_vocab()
return {
key: added_vocab[token_name] for key, token_name in required_tokens.items()
}
def __call__(
self,
images: ImageInput = None,
videos: np.ndarray | list[ImageInput] = None,
text: TextInput
| PreTokenizedInput
| list[TextInput]
| list[PreTokenizedInput] = None,
**kwargs: Unpack[Ovis2_5ProcessorKwargs],
) -> BatchFeature:
"""Main method to prepare for the model one or several sequences(s)
and image(s). This method forwards the `text`and `kwargs` arguments
to Qwen2TokenizerFast's [`~Qwen2TokenizerFast.__call__`] if `text`
is not `None` to encode the text. To prepare the vision inputs,
this method forwards the `vision_infos` and `kwargs` arguments to
Qwen2VLImageProcessor's [`~Qwen2VLImageProcessor.__call__`]
if `vision_infos` is not `None`.
Args:
images (`PIL.Image.Image`, `np.ndarray`, `torch.Tensor`, or a list):
The image or batch of images to be prepared. Each image can
be a PIL image, NumPy array or PyTorch tensor. Both
channels-first and channels-last formats are supported.
videos (`np.ndarray`, `torch.Tensor`, or a list of either):
The video or batch of videos to be prepared. Each video can
be a 4D NumPy array or PyTorch tensor, or a nested list of
3D frames. Both channels-first and channels-last formats are
supported.
text (`str`, `list[str]`, `list[list[str]]`):
The sequence or batch of sequences to be encoded. Each
sequence can be a string or a list of strings (pretokenized
string). If the sequences are provided as list of strings
(pretokenized), you must set `is_split_into_words=True` (to
lift the ambiguity with a batch of sequences).
return_tensors (`str` or [`~utils.TensorType`], *optional*):
If set, will return tensors of a particular framework.
Acceptable values are:
- `'tf'`: Return TensorFlow `tf.constant` objects.
- `'pt'`: Return PyTorch `torch.Tensor` objects.
- `'np'`: Return NumPy `np.ndarray` objects.
- `'jax'`: Return JAX `jnp.ndarray` objects.
Returns:
[`BatchFeature`]: A [`BatchFeature`] with the following fields:
- **input_ids** -- list of token ids to be fed to a model.
Returned when `text` is not `None`.
- **attention_mask** -- list of indices specifying which tokens
should be attended to by the model (when
`return_attention_mask=True` or if *"attention_mask"*
is in `self.model_input_names` and if `text` is not `None`).
- **pixel_values** -- Pixel values to be fed to a model.
Returned when `images` is not `None`.
- **pixel_values_videos** -- Pixel values of videos to be fed to
a model. Returned when `videos` is not `None`.
- **image_grid_thw** -- list of image 3D grid in LLM. Returned
when `images` is not `None`.
- **video_grid_thw** -- list of video 3D grid in LLM. Returned
when `videos` is not `None`.
- **second_per_grid_ts** -- list of video seconds per time grid.
Returned when `videos` is not `None`.
"""
output_kwargs = self._merge_kwargs(
Ovis2_5ProcessorKwargs,
tokenizer_init_kwargs=self.tokenizer.init_kwargs,
**kwargs,
)
# Process all images first
visual_features = {}
output = BatchFeature()
if images is not None:
processed_images = []
image_placeholders_list = []
grids = []
# Process each image
for image in images if isinstance(images, list) else [images]:
pixel_values, image_placeholders, grid = self.preprocess_multidata(
images=image,
**output_kwargs["images_kwargs"],
)
processed_images.append(pixel_values)
image_placeholders_list.append(image_placeholders)
grids.append(grid)
# assign all processed images
if processed_images:
visual_features["image_placeholders"] = image_placeholders_list
output["pixel_values"] = processed_images
output["grids"] = grids
if videos is not None:
processed_videos = []
videos_placeholders_list = []
grids = []
# Process each video
for video in videos if isinstance(videos, list) else [videos]:
pixel_values, video_placeholders, grid = self.preprocess_multidata(
video=video,
**output_kwargs["videos_kwargs"],
)
processed_videos.append(pixel_values)
videos_placeholders_list.append(video_placeholders)
grids.append(grid)
# assign all processed videos
if processed_videos:
visual_features["video_placeholders"] = videos_placeholders_list
output["video_pixel_values"] = processed_videos
output["video_grids"] = grids
# Process text input
if text is not None:
if not isinstance(text, list):
text = [text]
tokenized_batched_text = self._tokenize_with_visual_symbol(text)
image_token_id = self.get_token_value("image_token")
video_token_id = self.get_token_value("video_token")
replaced_ids_list = []
image_idx = 0
video_idx = 0
for ids_tensor in tokenized_batched_text:
has_image_tokens = (
image_token_id in ids_tensor
and "image_placeholders" in visual_features
and image_idx < len(visual_features["image_placeholders"])
)
has_video_tokens = (
video_token_id in ids_tensor
and "video_placeholders" in visual_features
and video_idx < len(visual_features["video_placeholders"])
)
if has_image_tokens or has_video_tokens:
# Convert to list for easier manipulation
ids_list = ids_tensor.tolist()
new_ids = []
# Replace placeholders
for token_id in ids_list:
if token_id == image_token_id:
new_ids.extend(
visual_features["image_placeholders"][image_idx]
)
image_idx += 1
elif token_id == video_token_id:
new_ids.extend(
visual_features["video_placeholders"][video_idx]
)
video_idx += 1
else:
new_ids.append(token_id)
# Convert back to tensor
ids_tensor = torch.tensor(new_ids, dtype=torch.long)
replaced_ids_list.append(ids_tensor)
if replaced_ids_list:
replaced_and_tokenized_ids = torch.stack(replaced_ids_list)
else:
replaced_and_tokenized_ids = torch.tensor([], dtype=torch.long)
output["input_ids"] = replaced_and_tokenized_ids
return output
# If only images were provided
return BatchFeature(data=visual_features)
def _tokenize_with_visual_symbol(self, text_list: list[str]) -> torch.LongTensor:
batch_token_ids = []
for text in text_list:
token_ids = []
video_token_id = self.get_token_value("video_token")
image_token_id = self.get_token_value("image_token")
video_split_texts = text.split(self.video_token)
for j, video_segment in enumerate(video_split_texts):
image_split_texts = video_segment.split(self.image_token)
text_chunks = [
self.tokenizer(chunk, add_special_tokens=False).input_ids
for chunk in image_split_texts
]
segment_tokens = []
for i, chunk in enumerate(text_chunks):
segment_tokens.extend(chunk)
if i < len(text_chunks) - 1:
segment_tokens.append(image_token_id)
token_ids.extend(segment_tokens)
if j < len(video_split_texts) - 1:
token_ids.append(video_token_id)
batch_token_ids.append(token_ids)
return torch.tensor(batch_token_ids, dtype=torch.long)
# Copied from qwen2_vl
def smart_resize(
self,
height: int,
width: int,
factor: int = 28,
min_pixels: int = MIN_PIXELS,
max_pixels: int = MAX_PIXELS,
):
"""Rescales the image so that the following conditions are met:
1. Both dimensions (height and width) are divisible by 'factor'.
2. The total number of pixels is within the range
['min_pixels', 'max_pixels'].
3. The aspect ratio of the image is maintained as closely as possible.
"""
if height < factor or width < factor:
print(
f"height:{height} or width:{width} must be larger than factor:{factor}"
)
if height < width:
width = round(factor / height * width)
height = factor
else:
height = round(factor / width * height)
width = factor
elif max(height, width) / min(height, width) > 200:
print(
f"absolute aspect ratio must be smaller than 200, "
f"got {max(height, width) / min(height, width)}"
)
if height > width:
height = 200 * width
else:
width = 200 * height
h_bar = round(height / factor) * factor
w_bar = round(width / factor) * factor
if h_bar * w_bar > max_pixels:
beta = math.sqrt((height * width) / max_pixels)
h_bar = math.floor(height / beta / factor) * factor
w_bar = math.floor(width / beta / factor) * factor
elif h_bar * w_bar < min_pixels:
beta = math.sqrt(min_pixels / (height * width))
h_bar = math.ceil(height * beta / factor) * factor
w_bar = math.ceil(width * beta / factor) * factor
return h_bar, w_bar
def get_token_value(self, tok):
return self.extra_special_tokens[tok]
def construct_visual_indicators(self, grid, is_video: bool = False):
if is_video:
start_token = self.get_token_value("video_start")
end_token = self.get_token_value("video_end")
else:
start_token = self.get_token_value("image_start")
end_token = self.get_token_value("image_end")
image_placeholders = [start_token, self.get_token_value("visual_atom")]
if grid[0] * grid[1] > 1:
for r in range(grid[0]):
for c in range(grid[1]):
image_placeholders.append(self.get_token_value("visual_atom"))
image_placeholders.append(end_token)
return image_placeholders
def construct_visual_placeholders(self, grid, is_video: bool = False):
visual_placeholders = self.construct_visual_indicators((1, 1), is_video)
image_atom_token_id = self.get_token_value("visual_atom")
# Extract the padding token ID from tokenizer
image_padding_token_id = self.get_token_value("image_pad")
num_image_atoms = grid[0] * grid[1] * grid[2]
num_image_atoms //= self.hidden_stride**2
num_image_atoms //= self.temporal_patch_size
# Create a new list with padding tokens inserted
padded_placeholder_tokens = []
for token in visual_placeholders:
if token == image_atom_token_id:
padded_placeholder_tokens.extend(
[image_padding_token_id] * num_image_atoms
)
else:
padded_placeholder_tokens.append(image_padding_token_id)
return padded_placeholder_tokens
def preprocess_multidata(
self,
images: PIL.Image.Image | list[PIL.Image.Image] | None = None,
video: list[PIL.Image.Image] | np.ndarray | None = None,
do_convert_rgb: bool | None = True,
min_pixels: int = MIN_PIXELS,
max_pixels: int = MAX_PIXELS,
return_tensors: str | None = "pt",
):
is_video = False
if images is not None:
if not isinstance(images, list):
images = [images]
elif video is not None:
is_video = True
# type of video in dummy_mm_data is np.ndarray
if isinstance(video, np.ndarray):
images = []
for i in range(video.shape[0]):
image = PIL.Image.fromarray(video[i].astype(np.uint8))
images.append(image)
elif isinstance(video, list):
images = video
else:
raise ValueError("Either images or video should be provided.")
assert images is not None
min_pixels = min(
max_pixels if max_pixels is not None else MAX_PIXELS,
min_pixels if min_pixels is not None else MIN_PIXELS,
)
images = [
image.convert("RGB") if do_convert_rgb and image.mode != "RGB" else image
for image in images
]
width, height = images[0].size
resized_height, resized_width = height, width
processed_images = []
for image in images:
resized_height, resized_width = self.smart_resize(
height,
width,
factor=self.patch_size * self.hidden_stride,
min_pixels=min_pixels,
max_pixels=max_pixels,
)
new_size = dict(height=resized_height, width=resized_width)
image_pt = self.image_processor.preprocess(image, size=new_size)[
"pixel_values"
][0]
processed_images.append(image_pt)
patches = np.array(processed_images)
if patches.shape[0] % self.temporal_patch_size != 0:
num_to_pad = self.temporal_patch_size - (
patches.shape[0] % self.temporal_patch_size
)
repeats = np.repeat(patches[-1][np.newaxis], num_to_pad, axis=0)
patches = np.concatenate([patches, repeats], axis=0)
channel = patches.shape[1]
grid_t = patches.shape[0] // self.temporal_patch_size
grid_h = resized_height // self.patch_size
grid_w = resized_width // self.patch_size
patches = patches.reshape(
grid_t,
self.temporal_patch_size,
channel,
grid_h // self.hidden_stride,
self.hidden_stride,
self.patch_size,
grid_w // self.hidden_stride,
self.hidden_stride,
self.patch_size,
)
patches = patches.transpose(0, 3, 6, 4, 7, 2, 1, 5, 8)
flatten_patches = patches.reshape(
grid_t * grid_h * grid_w,
channel * self.temporal_patch_size * self.patch_size * self.patch_size,
)
visual_placeholders = self.construct_visual_placeholders(
[grid_t, grid_h, grid_w], is_video
)
return (
torch.tensor(flatten_patches),
visual_placeholders,
torch.tensor([[grid_t, grid_h, grid_w]]),
)
@@ -1,79 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import torch
from mistral_common.protocol.instruct.chunk import ImageChunk
from mistral_common.tokens.tokenizers.multimodal import ImageEncoder
from PIL import Image
from transformers import BatchFeature, ImageProcessingMixin, ProcessorMixin, TensorType
from transformers.image_utils import ImageInput
from vllm.tokenizers.mistral import MistralTokenizer
class MistralCommonImageProcessor(ImageProcessingMixin):
"""Provide a HF-compatible interface for
`mistral_common.tokens.tokenizers.multimodal.ImageEncoder`.
"""
def __init__(self, mm_encoder: ImageEncoder) -> None:
self.mm_encoder = mm_encoder
def __call__(
self,
images: ImageInput,
return_tensors: str | TensorType | None = None,
**kwargs,
) -> BatchFeature:
images_lst = [images] if not isinstance(images, list) else images
images_processed = list[torch.Tensor]()
for image in images_lst:
image_inputs = self.mm_encoder(ImageChunk(image=image))
image_processed = torch.tensor(image_inputs.image)
images_processed.append(image_processed)
return BatchFeature({"images": images_processed}, tensor_type=return_tensors)
def get_number_of_image_patches(
self,
height: int,
width: int,
) -> tuple[int, int, int]:
image = Image.new("RGB", (width, height))
ncols, nrows = self.mm_encoder._image_to_num_tokens(image)
return ncols * nrows, nrows, ncols
# Copied from Transformers (Apache-2.0):
# https://github.com/huggingface/transformers/blob/d20946079fd422335fbae3eeb98b7cd88334612f/src/transformers/image_processing_base.py#L473
def fetch_images(self, image_url_or_urls):
from transformers.image_utils import is_valid_image, load_image
if isinstance(image_url_or_urls, (list, tuple)):
return [self.fetch_images(x) for x in image_url_or_urls]
if isinstance(image_url_or_urls, str):
return load_image(image_url_or_urls)
if is_valid_image(image_url_or_urls):
return image_url_or_urls
raise TypeError(
"only a single or a list of entries is supported but got "
f"type={type(image_url_or_urls)}"
)
class MistralCommonPixtralProcessor(ProcessorMixin):
attributes = ["image_processor", "tokenizer"]
def __init__(
self,
tokenizer: MistralTokenizer,
image_processor: MistralCommonImageProcessor,
) -> None:
self.tokenizer = tokenizer.transformers_tokenizer
self.image_processor = image_processor
image_special_ids = self.image_processor.mm_encoder.special_ids
self.image_break_id = image_special_ids.img_break
self.image_token_id = image_special_ids.img
self.image_end_id = image_special_ids.img_end
@@ -1,514 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from itertools import product
from math import ceil
import numpy as np
import torch
from PIL import Image
from torchvision import transforms
from torchvision.transforms.functional import InterpolationMode
from transformers import (
BatchFeature,
ImageProcessingMixin,
ProcessorMixin,
TensorType,
)
from vllm.tokenizers import TokenizerLike
MAX_IMAGE_SIZE: int = 3024
ImageWithPatches = tuple[Image.Image, list[Image.Image], list[bool]]
class Step3VisionProcessor:
def __init__(self, size, interpolation_mode="bicubic", patch_size=None):
mean = [0.48145466, 0.4578275, 0.40821073]
std = [0.26862954, 0.26130258, 0.27577711]
patch_size = patch_size if patch_size is not None else size
self.transform = transforms.Compose(
[
transforms.ToTensor(),
transforms.Normalize(mean, std),
transforms.Resize(
(size, size),
interpolation=InterpolationMode.BICUBIC
if interpolation_mode == "bicubic"
else InterpolationMode.BILINEAR,
antialias=True,
),
]
)
self.patch_transform = (
transforms.Compose(
[
transforms.ToTensor(),
transforms.Normalize(mean, std),
transforms.Resize(
(patch_size, patch_size),
interpolation=InterpolationMode.BICUBIC
if interpolation_mode == "bicubic"
else InterpolationMode.BILINEAR,
antialias=True,
),
]
)
if patch_size is not None
else None
)
def __call__(self, image, is_patch=False):
if is_patch:
assert self.patch_transform is not None
return {"pixel_values": self.patch_transform(image).unsqueeze(0)}
return {"pixel_values": self.transform(image).unsqueeze(0)}
class ImagePatcher:
def __init__(self, enable_patch: bool = True) -> None:
self.enable_patch = enable_patch
def determine_window_size(self, long: int, short: int) -> int:
if long < 728:
return short if long / short > 1.5 else 0
return min(short, 504) if long / short > 4 else 504
def slide_window(
self,
width: int,
height: int,
sizes: list[tuple[int, int]],
steps: list[tuple[int, int]],
img_rate_thr: float = 0.6,
) -> tuple[list[tuple[int, int, int, int]], tuple[int, int]]:
assert 1 >= img_rate_thr >= 0, "The `in_rate_thr` should lie in 0~1"
windows = []
# Sliding windows.
for size, step in zip(sizes, steps):
size_w, size_h = size
step_w, step_h = step
x_num = 1 if width <= size_w else ceil((width - size_w) / step_w + 1)
x_start = [step_w * i for i in range(x_num)]
if len(x_start) > 1 and x_start[-1] + size_w > width:
x_start[-1] = width - size_w
y_num = 1 if height <= size_h else ceil((height - size_h) / step_h + 1)
y_start = [step_h * i for i in range(y_num)]
if len(y_start) > 1 and y_start[-1] + size_h > height:
y_start[-1] = height - size_h
start = np.array(list(product(y_start, x_start)), dtype=int)
start[:, [0, 1]] = start[:, [1, 0]]
windows.append(np.concatenate([start, start + size], axis=1))
windows = np.concatenate(windows, axis=0)
return [
(int(box[0]), int(box[1]), int(box[2] - box[0]), int(box[3] - box[1]))
for box in windows
], (x_num, y_num)
def square_pad(self, img: Image.Image) -> Image.Image:
w, h = img.size
if w == h:
return img
size = max(w, h)
padded = Image.new(img.mode, (size, size), 0)
padded.paste(img, (0, 0))
return padded
def get_image_size_for_padding(
self, img_width: int, img_height: int
) -> tuple[int, int]:
ratio = img_width / img_height
if min(img_height, img_width) < 32 and (ratio > 4 or ratio < 1 / 4):
new_size = max(img_height, img_width)
return new_size, new_size
return img_width, img_height
def get_image_size_for_preprocess(
self, img_width: int, img_height: int
) -> tuple[int, int]:
if max(img_height, img_width) > MAX_IMAGE_SIZE:
scale_factor = MAX_IMAGE_SIZE / max(img_height, img_width)
img_width = int(img_width * scale_factor)
img_height = int(img_height * scale_factor)
return img_width, img_height
def get_image_size_for_crop(
self, img_width: int, img_height: int, window_size: int
):
w_ratio = img_width / window_size
h_ratio = img_height / window_size
if w_ratio < 1:
width_new = img_width
else:
decimal_w = w_ratio - img_width // window_size
w_ratio = int(w_ratio) + 1 if decimal_w > 0.2 else int(w_ratio)
width_new = window_size * w_ratio
if h_ratio < 1:
height_new = img_height
else:
decimal_h = h_ratio - img_height // window_size
h_ratio = int(h_ratio) + 1 if decimal_h > 0.2 else int(h_ratio)
height_new = window_size * h_ratio
return int(width_new), int(height_new)
def patch_crop(self, img: Image.Image, i: int, j: int, th: int, tw: int):
target = img.crop((j, i, j + tw, i + th))
return target
def get_num_patches(self, img_width: int, img_height: int) -> tuple[int, int]:
img_width, img_height = self.get_image_size_for_padding(img_width, img_height)
img_width, img_height = self.get_image_size_for_preprocess(
img_width, img_height
)
window_size = self.determine_window_size(
max(img_height, img_width), min(img_height, img_width)
)
if window_size == 0 or not self.enable_patch:
return 0, 0
else:
img_width, img_height = self.get_image_size_for_crop(
img_width, img_height, window_size
)
center_list, (x_num, y_num) = self.slide_window(
img_width,
img_height,
[(window_size, window_size)],
[(window_size, window_size)],
)
full_rows = (len(center_list) - 1) // x_num + 1
if len(center_list) > 0 and len(center_list) % x_num == 0:
full_rows -= 1
return len(center_list), full_rows
def __call__(
self, img: Image.Image
) -> tuple[Image.Image, list[Image.Image], list[bool]]:
img_width, img_height = img.size
new_img_width, new_img_height = self.get_image_size_for_padding(
img_width, img_height
)
if new_img_width != img_width or new_img_height != img_height:
img = self.square_pad(img)
img_width, img_height = img.size
new_img_width, new_img_height = self.get_image_size_for_preprocess(
img_width, img_height
)
img = img.resize((new_img_width, new_img_height), Image.Resampling.BILINEAR)
window_size = self.determine_window_size(
max(new_img_height, new_img_width), min(new_img_height, new_img_width)
)
if window_size == 0 or not self.enable_patch:
return img, [], []
else:
new_img_width, new_img_height = self.get_image_size_for_crop(
new_img_width, new_img_height, window_size
)
if (new_img_width, new_img_height) != (img_width, img_height):
img_for_crop = img.resize(
(new_img_width, new_img_height), Image.Resampling.BILINEAR
)
else:
img_for_crop = img
patches = []
newlines = []
center_list, (x_num, y_num) = self.slide_window(
new_img_width,
new_img_height,
[(window_size, window_size)],
[(window_size, window_size)],
)
for patch_id, center_lf_point in enumerate(center_list):
x, y, patch_w, patch_h = center_lf_point
big_patch = self.patch_crop(img_for_crop, y, x, patch_h, patch_w)
patches.append(big_patch)
if (patch_id + 1) % x_num == 0:
newlines.append(patch_id)
if newlines and newlines[-1] == len(patches) - 1:
newlines.pop()
return (
img,
patches,
[i in newlines for i in range(len(patches))],
)
class Step3VLImageProcessor(ImageProcessingMixin):
def __init__(
self,
image_size: int = 728,
patch_size: int = 504,
num_image_feature_size: int = 169,
num_patch_feature_size: int = 81,
enable_patch: bool = True,
) -> None:
self.image_size = image_size
self.patch_size = patch_size
self.num_image_feature_size = num_image_feature_size
self.num_patch_feature_size = num_patch_feature_size
self.image_preprocessor = Step3VisionProcessor(
image_size, "bilinear", patch_size
)
self.patcher = ImagePatcher(enable_patch=enable_patch)
def get_num_image_tokens(self, img_width: int, img_height: int) -> int:
num_patches, num_newlines = self.patcher.get_num_patches(img_width, img_height)
return (
num_patches * (self.num_patch_feature_size + 2)
+ self.num_image_feature_size
+ 2
+ num_newlines
)
def _split_images(self, images: list[Image.Image]) -> list[ImageWithPatches]:
result = []
for img in images:
result.append(self.patcher(img))
return result
def _convert_images_to_pixel_values(
self,
images: list[Image.Image],
is_patch: bool = False,
) -> list[torch.Tensor]:
return [
self.image_preprocessor(img, is_patch=is_patch)["pixel_values"]
for img in images
]
def __call__(
self,
images: Image.Image | list[Image.Image],
return_tensors: str | TensorType | None = None,
) -> BatchFeature:
if not isinstance(images, list):
images = [images]
split_images_data = self._split_images(images)
pixel_values_lst = []
patch_pixel_values_lst = []
patch_newline_mask_lst = []
num_patches = []
for raw_img, img_patches, patch_newline_mask in split_images_data:
pixel_values_lst.extend(self._convert_images_to_pixel_values([raw_img]))
num_patches.append(len(img_patches))
patch_pixel_values_lst.extend(
self._convert_images_to_pixel_values(img_patches, is_patch=True)
)
patch_newline_mask_lst.extend(patch_newline_mask)
pixel_values = torch.cat(pixel_values_lst)
patch_size = self.patch_size
image_inputs = {
"pixel_values": pixel_values,
"num_patches": num_patches,
"patch_pixel_values": (
torch.cat(patch_pixel_values_lst)
if patch_pixel_values_lst
else pixel_values.new_empty((0, 3, patch_size, patch_size))
),
"patch_newline_mask": torch.tensor(
patch_newline_mask_lst, dtype=torch.bool
),
}
return BatchFeature(image_inputs, tensor_type=return_tensors)
class Step3VLProcessor(ProcessorMixin):
attributes = ["image_processor", "tokenizer"]
def __init__(
self,
image_processor: Step3VLImageProcessor,
tokenizer: TokenizerLike,
) -> None:
self.image_processor = image_processor
self.tokenizer = tokenizer
self.image_start_token = image_start_token = "<im_start>"
self.image_end_token = image_end_token = "<im_end>"
self.patch_start_token = patch_start_token = "<patch_start>"
self.patch_end_token = patch_end_token = "<patch_end>"
self.patch_newline_token = patch_newline_token = "<patch_newline>"
self.image_start_token_id = tokenizer.convert_tokens_to_ids(image_start_token)
self.image_end_token_id = tokenizer.convert_tokens_to_ids(image_end_token)
self.patch_start_token_id = tokenizer.convert_tokens_to_ids(patch_start_token)
self.patch_end_token_id = tokenizer.convert_tokens_to_ids(patch_end_token)
self.patch_newline_token_id = tokenizer.convert_tokens_to_ids(
patch_newline_token
)
self.image_token = image_token = "<im_patch>"
self.image_feature_tokens = image_token * image_processor.num_image_feature_size
self.patch_feature_tokens = image_token * image_processor.num_patch_feature_size
self.image_token_id = image_token_id = tokenizer.convert_tokens_to_ids(
image_token
)
self.image_feature_token_ids = [
image_token_id
] * image_processor.num_image_feature_size
self.patch_feature_token_ids = [
image_token_id
] * image_processor.num_patch_feature_size
def _get_patch_repl_text(
self,
num_patches: int,
patch_newline_mask: list[bool],
) -> str:
assert len(patch_newline_mask) == num_patches
parts = []
for i in range(num_patches):
parts.extend(
[
self.patch_start_token,
self.patch_feature_tokens,
self.patch_end_token,
]
)
if patch_newline_mask[i]:
parts.append(self.patch_newline_token)
return "".join(parts)
def _get_patch_repl_ids(
self,
num_patches: int,
patch_newline_mask: list[bool],
) -> list[int]:
assert len(patch_newline_mask) == num_patches
parts = []
for i in range(num_patches):
parts.extend(
[
self.patch_start_token_id,
*self.patch_feature_token_ids,
self.patch_end_token_id,
]
)
if patch_newline_mask[i]:
parts.append(self.patch_newline_token_id)
return parts
def _get_image_repl_text(
self,
num_images: int,
) -> str:
parts = [
self.image_start_token,
self.image_feature_tokens,
self.image_end_token,
] * num_images
return "".join(parts)
def _get_image_repl_ids(
self,
num_images: int,
) -> list[int]:
part = [
self.image_start_token_id,
*self.image_feature_token_ids,
self.image_end_token_id,
]
return part * num_images
def get_image_repl_feature_text(
self,
num_images: int,
num_patches: int,
patch_new_line_idx: list[bool],
) -> str:
patch_repl = self._get_patch_repl_text(num_patches, patch_new_line_idx)
image_repl = self._get_image_repl_text(num_images)
return patch_repl + image_repl
def get_image_repl_feature_ids(
self,
num_images: int,
num_patches: int,
patch_new_line_idx: list[bool],
) -> list[int]:
patch_repl = self._get_patch_repl_ids(num_patches, patch_new_line_idx)
image_repl = self._get_image_repl_ids(num_images)
return patch_repl + image_repl
def replace_placeholder(self, text: str, placeholder: str, repls: list[str]) -> str:
parts = text.split(placeholder)
if len(parts) - 1 != len(repls):
raise ValueError(
"The number of placeholders does not match the number of replacements."
)
result = [parts[0]]
for i, repl in enumerate(repls):
result.append(repl)
result.append(parts[i + 1])
return "".join(result)
def __call__(
self,
text: str | list[str] | None = None,
images: Image.Image | list[Image.Image] | None = None,
return_tensors: str | TensorType | None = None,
) -> BatchFeature:
if images is not None:
image_inputs = self.image_processor(
images=images,
return_tensors=return_tensors,
)
num_patches = image_inputs["num_patches"]
patch_newline_mask = image_inputs["patch_newline_mask"]
else:
image_inputs = {}
num_patches = []
patch_newline_mask = []
if text is not None:
if not isinstance(text, list):
text = [text]
if image_inputs:
image_token = self.image_token
image_repl_str_lst = []
start = 0
for n_patches in num_patches:
image_repl_str = self.get_image_repl_feature_text(
1, n_patches, patch_newline_mask[start : start + n_patches]
)
image_repl_str_lst.append(image_repl_str)
start += n_patches
text = [
self.replace_placeholder(t, image_token, image_repl_str_lst)
for t in text
]
text_inputs = self.tokenizer(text)
else:
text_inputs = {}
return BatchFeature(
data={**text_inputs, **image_inputs},
tensor_type=return_tensors,
)
@@ -1,112 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from math import ceil
import numpy as np
import torch
from mistral_common.tokens.tokenizers.audio import AudioEncoder
from transformers import (
BatchFeature,
ProcessorMixin,
SequenceFeatureExtractor,
TensorType,
)
from transformers.audio_utils import AudioInput
from vllm.tokenizers.mistral import MistralTokenizer
class MistralCommonFeatureExtractor(SequenceFeatureExtractor):
"""Provide a HF-compatible interface for
`mistral_common.tokens.tokenizers.multimodal.AudioEncoder`.
"""
def __init__(self, audio_encoder: AudioEncoder) -> None:
self.audio_encoder = audio_encoder
@property
def sampling_rate(self):
return self.audio_encoder.audio_config.sampling_rate
@property
def frame_rate(self):
return self.audio_encoder.audio_config.frame_rate
def __call__(
self,
audios: AudioInput,
return_tensors: str | TensorType | None = None,
**kwargs,
) -> BatchFeature:
audios_lst = [audios] if not isinstance(audios, list) else audios
audios_processed = list[torch.Tensor]()
for audio in audios_lst:
audio = np.asarray(audio, dtype=np.float32).ravel()
if not self.audio_encoder.audio_config.is_streaming:
audio = self.audio_encoder.pad(audio, self.sampling_rate)
audios_processed.append(torch.from_numpy(audio))
return BatchFeature(
{"audio_arrays": audios_processed}, tensor_type=return_tensors
)
def get_num_audio_tokens(self, audio_length: int) -> int:
return ceil(audio_length / (self.sampling_rate // self.frame_rate))
def fetch_audio(self, audio_url_or_urls, sampling_rate=None):
"""HF-compatible duck-typed ``fetch_audio``.
Mirrors :meth:`transformers.SequenceFeatureExtractor.fetch_audio` so
:class:`transformers.ProcessorMixin.prepare_inputs_layout` works on this
duck-typed feature extractor.
Accepts the same shapes as ``SequenceFeatureExtractor.fetch_audio``:
* ``np.ndarray`` / ``torch.Tensor`` — returned as-is.
* ``list[float]`` — returned as-is (a single audio sample).
* ``str`` URL or path — delegated to
:func:`transformers.audio_utils.load_audio`.
* ``list`` of any of the above — recursed element-wise.
``ProcessorMixin.prepare_inputs_layout`` always passes already-decoded
audio (numpy array or torch tensor), so the str / list-of-str branches
exist only to keep the contract identical to the upstream method.
"""
from transformers.audio_utils import is_valid_audio
sampling_rate = sampling_rate if sampling_rate else self.sampling_rate
if is_valid_audio(audio_url_or_urls):
return audio_url_or_urls
if isinstance(audio_url_or_urls, (list, tuple)):
return [
self.fetch_audio(x, sampling_rate=sampling_rate)
for x in audio_url_or_urls
]
if isinstance(audio_url_or_urls, str):
from transformers.audio_utils import load_audio
return load_audio(audio_url_or_urls, sampling_rate=sampling_rate)
raise TypeError(
"only a numpy array, torch tensor, str URL/path, or list of those "
f"is supported but got type={type(audio_url_or_urls)}"
)
class MistralCommonVoxtralProcessor(ProcessorMixin):
attributes = ["feature_extractor", "tokenizer"]
def __init__(
self,
tokenizer: MistralTokenizer,
feature_extractor: MistralCommonFeatureExtractor,
) -> None:
self.tokenizer = tokenizer.transformers_tokenizer
self.feature_extractor = feature_extractor
audio_special_ids = self.feature_extractor.audio_encoder.special_ids
self.audio_token_id = audio_special_ids.audio
self.begin_audio_token_id = audio_special_ids.begin_audio