[Mini-VLLM] Cut dead subsystems across serving, connectors, parsers
- Remove P/D transfer connectors: nixl, nixl_chat, mooncake x2, hf3fs, moriio, hisparse, example and hidden-states connectors - Reduce weight_transfer to the base API used by model loading - Remove kv_offload tiering, simple_kv_offload, ec cpu/example, snapshot command, engine snapshot server - Remove transformers model fallback, dsv4 tokenizers, dead platforms cpu/tpu/xpu, flashmla third_party, dsa, megamoe/flydsl, dead attn ops, dead layers (short_conv, fused_embed_norm, indexer_topk, relu2 gemma_ar, lightning_attn, minimax_rms_norm), gemm_rs_ar - Keep MoE LoRA op (imported eagerly by punica_gpu); drop fp8 variant - Rebuild parser hubs around Qwen3 adapters only (36 dead adapters) - Remove dead tests with the code they covered Co-authored-by: Hinata <hinata@idihore.id> Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
@@ -14,71 +14,11 @@ from __future__ import annotations
|
||||
import importlib
|
||||
|
||||
_CLASS_TO_MODULE: dict[str, str] = {
|
||||
"BagelConfig": "vllm.transformers_utils.configs.bagel",
|
||||
"BailingMoeV3TextConfig": "vllm.transformers_utils.configs.bailing_moe_v3_vl",
|
||||
"BailingMoeV3VisionConfig": "vllm.transformers_utils.configs.bailing_moe_v3_vl",
|
||||
"BailingMoeV3VLConfig": "vllm.transformers_utils.configs.bailing_moe_v3_vl",
|
||||
"ChatGLMConfig": "vllm.transformers_utils.configs.chatglm",
|
||||
"ColModernVBertConfig": "vllm.transformers_utils.configs.colmodernvbert",
|
||||
"ColPaliConfig": "vllm.transformers_utils.configs.colpali",
|
||||
"ColQwen3Config": "vllm.transformers_utils.configs.colqwen3",
|
||||
"OpsColQwen3Config": "vllm.transformers_utils.configs.colqwen3",
|
||||
"Qwen3VLNemotronEmbedConfig": "vllm.transformers_utils.configs.colqwen3",
|
||||
"Cosmos3EdgeConfig": "vllm.transformers_utils.configs.cosmos3_edge",
|
||||
"Cosmos3EdgeProjectorConfig": "vllm.transformers_utils.configs.cosmos3_edge",
|
||||
"Cosmos3EdgeTextConfig": "vllm.transformers_utils.configs.cosmos3_edge",
|
||||
"Cosmos3EdgeVisionConfig": "vllm.transformers_utils.configs.cosmos3_edge",
|
||||
"DeepseekVLV2Config": "vllm.transformers_utils.configs.deepseek_vl2",
|
||||
"DeepseekV41Config": "vllm.transformers_utils.configs.deepseek_v41",
|
||||
"Dots3NoteConfig": "vllm.transformers_utils.configs.dots3_note",
|
||||
"K3DSparkConfig": "vllm.transformers_utils.configs.k3_dspark",
|
||||
"DotsOCRConfig": "vllm.transformers_utils.configs.dotsocr",
|
||||
"EAGLEConfig": "vllm.transformers_utils.configs.eagle",
|
||||
"FunAudioChatConfig": "vllm.transformers_utils.configs.funaudiochat",
|
||||
"FunAudioChatAudioEncoderConfig": "vllm.transformers_utils.configs.funaudiochat",
|
||||
"HYV4Config": "vllm.transformers_utils.configs.hy_v4",
|
||||
"IsaacConfig": "vllm.transformers_utils.configs.isaac",
|
||||
# RWConfig is for the original tiiuae/falcon-40b(-instruct) and
|
||||
# tiiuae/falcon-7b(-instruct) models. Newer Falcon models will use the
|
||||
# `FalconConfig` class from the official HuggingFace transformers library.
|
||||
"RWConfig": "vllm.transformers_utils.configs.falcon",
|
||||
"MiDashengLMConfig": "vllm.transformers_utils.configs.midashenglm",
|
||||
"MiniMaxM3Config": "vllm.transformers_utils.configs.minimax_m3",
|
||||
"MiniMaxM3MTPConfig": "vllm.transformers_utils.configs.minimax_m3",
|
||||
"MiniMaxM3TextConfig": "vllm.transformers_utils.configs.minimax_m3",
|
||||
"Moondream3Config": "vllm.transformers_utils.configs.moondream3",
|
||||
"Moondream3TextConfig": "vllm.transformers_utils.configs.moondream3",
|
||||
"Moondream3VisionConfig": "vllm.transformers_utils.configs.moondream3",
|
||||
"MossTranscribeDiarizeConfig": (
|
||||
"vllm.transformers_utils.configs.moss_transcribe_diarize"
|
||||
),
|
||||
"MoonViTConfig": "vllm.transformers_utils.configs.moonvit",
|
||||
"KimiLinearConfig": "vllm.transformers_utils.configs.kimi_linear",
|
||||
"KimiVLConfig": "vllm.transformers_utils.configs.kimi_vl",
|
||||
"MuseGlimmerConfig": "vllm.transformers_utils.configs.muse_glimmer",
|
||||
"MuseGlimmerTextConfig": "vllm.transformers_utils.configs.muse_glimmer",
|
||||
"MuseGlimmerVisionConfig": "vllm.transformers_utils.configs.muse_glimmer",
|
||||
"MuseGlimmerAssistantConfig": "vllm.transformers_utils.configs.muse_glimmer",
|
||||
"KimiK3Config": "vllm.transformers_utils.configs.kimi_k3",
|
||||
"KimiK3VisionConfig": "vllm.transformers_utils.configs.kimi_k3",
|
||||
"OpenVLAConfig": "vllm.transformers_utils.configs.openvla",
|
||||
"OvisConfig": "vllm.transformers_utils.configs.ovis",
|
||||
"PixelShuffleSiglip2VisionConfig": "vllm.transformers_utils.configs.isaac",
|
||||
"RadioConfig": "vllm.transformers_utils.configs.radio",
|
||||
"SpeculatorsConfig": "vllm.transformers_utils.configs.speculators",
|
||||
"UltravoxConfig": "vllm.transformers_utils.configs.ultravox",
|
||||
"UnlimitedOCRConfig": "vllm.transformers_utils.configs.unlimited_ocr",
|
||||
"Step3VLConfig": "vllm.transformers_utils.configs.step3_vl",
|
||||
"Step3VisionEncoderConfig": "vllm.transformers_utils.configs.step3_vl",
|
||||
"Step3TextConfig": "vllm.transformers_utils.configs.step3_vl",
|
||||
"Step3p5Config": "vllm.transformers_utils.configs.step3p5",
|
||||
"QianfanOCRConfig": "vllm.transformers_utils.configs.qianfan_ocr",
|
||||
"QianfanOCRVisionConfig": "vllm.transformers_utils.configs.qianfan_ocr",
|
||||
"Qwen3ASRConfig": "vllm.transformers_utils.configs.qwen3_asr",
|
||||
"InklingModelConfig": "vllm.models.inkling.configs",
|
||||
"InklingAudioConfig": "vllm.models.inkling.configs",
|
||||
"InklingVisionConfig": "vllm.models.inkling.configs",
|
||||
"InklingMMConfig": "vllm.models.inkling.configs",
|
||||
"MossTranscribeDiarizeConfig": (),
|
||||
# Upstream Transformers classes registered in _CONFIG_REGISTRY
|
||||
"DeepseekV3Config": "transformers",
|
||||
"HyperCLOVAXConfig": "transformers",
|
||||
|
||||
@@ -1,345 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
# Copyright 2025 Antgroup and The HuggingFace Inc. team. All rights reserved.
|
||||
# Adapted from
|
||||
# https://huggingface.co/inclusionAI/Ling-3.0-flash/blob/51fd444268df5267074ef0b289de0e1c4cb9b381/configuration_bailing_moe_v3.py
|
||||
"""Bailing V3 vision-language model configuration."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
from transformers.configuration_utils import PreTrainedConfig
|
||||
|
||||
|
||||
def _normalize_mrope_parameters(
|
||||
config: PreTrainedConfig,
|
||||
mrope_section: list[int],
|
||||
) -> None:
|
||||
"""Add the top-level Bailing M-RoPE split to a text config."""
|
||||
rope_parameters = dict(getattr(config, "rope_parameters", None) or {})
|
||||
text_mrope_section = rope_parameters.get("mrope_section")
|
||||
if text_mrope_section is not None and list(text_mrope_section) != mrope_section:
|
||||
raise ValueError(
|
||||
"Bailing V3 VL has conflicting M-RoPE sections: "
|
||||
f"text={text_mrope_section}, top-level={mrope_section}"
|
||||
)
|
||||
|
||||
if "rope_type" not in rope_parameters and "type" not in rope_parameters:
|
||||
rope_parameters["rope_type"] = "default"
|
||||
rope_parameters.setdefault("rope_theta", getattr(config, "rope_theta", 10_000))
|
||||
rope_parameters["mrope_section"] = list(mrope_section)
|
||||
config.rope_parameters = rope_parameters
|
||||
|
||||
|
||||
def _build_layer_types(
|
||||
num_hidden_layers: int,
|
||||
layer_group_size: int,
|
||||
) -> list[str]:
|
||||
if layer_group_size <= 0:
|
||||
raise ValueError(
|
||||
f"Bailing V3 VL layer_group_size must be positive, got {layer_group_size}"
|
||||
)
|
||||
|
||||
grouped_layers = num_hidden_layers // layer_group_size * layer_group_size
|
||||
return [
|
||||
"linear_attention"
|
||||
if ((layer_idx + 1) % layer_group_size != 0 and layer_idx < grouped_layers)
|
||||
else "full_attention"
|
||||
for layer_idx in range(num_hidden_layers)
|
||||
]
|
||||
|
||||
|
||||
class BailingMoeV3TextConfig(PreTrainedConfig):
|
||||
model_type = "bailing_hybrid"
|
||||
base_config_key = "text_config"
|
||||
keys_to_ignore_at_inference = ["past_key_values"]
|
||||
ignore_keys_at_rope_validation = {"mrope_section"}
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
vocab_size: int = 157184,
|
||||
hidden_size: int = 2048,
|
||||
intermediate_size: int = 5120,
|
||||
num_hidden_layers: int = 20,
|
||||
num_attention_heads: int = 16,
|
||||
num_key_value_heads: int = 4,
|
||||
hidden_act: str = "silu",
|
||||
use_qkv_bias: bool = False,
|
||||
use_bias: bool = False,
|
||||
rms_norm_eps: float = 1e-6,
|
||||
tie_word_embeddings: bool = False,
|
||||
embedding_dropout: float = 0.0,
|
||||
attention_dropout: float = 0.0,
|
||||
output_dropout: float = 0.0,
|
||||
initializer_range: float = 0.02,
|
||||
max_position_embeddings: int = 32768,
|
||||
rope_theta: float = 600000.0,
|
||||
rope_parameters: dict[str, Any] | None = None,
|
||||
rope_scaling: dict[str, Any] | None = None,
|
||||
partial_rotary_factor: float | None = None,
|
||||
mrope_section: list[int] | tuple[int, ...] | None = None,
|
||||
use_cache: bool = True,
|
||||
max_window_layers: int = 20,
|
||||
pad_token_id: int = 156892,
|
||||
eos_token_id: int = 156892,
|
||||
num_experts: int = 256,
|
||||
num_shared_experts: int = 1,
|
||||
num_experts_per_tok: int = 8,
|
||||
n_group: int = 8,
|
||||
topk_group: int = 4,
|
||||
moe_intermediate_size: int = 512,
|
||||
moe_shared_expert_intermediate_size: int = 512,
|
||||
first_k_dense_replace: int = 1,
|
||||
head_dim: int | None = 128,
|
||||
output_router_logits: bool = False,
|
||||
use_qk_norm: bool = True,
|
||||
num_nextn_predict_layers: int = 0,
|
||||
mtp_loss_scaling_factor: float = 0,
|
||||
moe_router_enable_expert_bias: bool = True,
|
||||
routed_scaling_factor: float = 1.0,
|
||||
layer_group_size: int = 5,
|
||||
layer_types: list[str] | None = None,
|
||||
layers_block_type: list[str] | None = None,
|
||||
kv_lora_rank: int = 512,
|
||||
q_lora_rank: int | None = None,
|
||||
qk_rope_head_dim: int = 64,
|
||||
v_head_dim: int = 128,
|
||||
qk_nope_head_dim: int = 128,
|
||||
rope_interleave: bool = True,
|
||||
score_function: str = "sigmoid",
|
||||
scoring_func: str = "sigmoid",
|
||||
seq_aux: bool = True,
|
||||
topk_method: str = "noaux_tc",
|
||||
router_dtype: str = "fp32",
|
||||
gated_attention_proj_granularity_type: str | None = None,
|
||||
no_kda_lora: bool = False,
|
||||
kda_safe_gate: bool = False,
|
||||
kda_lower_bound: float | None = None,
|
||||
short_conv_kernel_size: int = 4,
|
||||
architectures: list[str] | None = None,
|
||||
**kwargs: Any,
|
||||
) -> None:
|
||||
kwargs.pop("model_type", None)
|
||||
if architectures is None:
|
||||
architectures = ["BailingMoeV3ForCausalLM"]
|
||||
|
||||
if rope_parameters is not None:
|
||||
self.rope_parameters = dict(rope_parameters)
|
||||
elif rope_scaling is not None:
|
||||
self.rope_parameters = dict(rope_scaling)
|
||||
else:
|
||||
self.rope_parameters = {}
|
||||
if (
|
||||
"rope_type" not in self.rope_parameters
|
||||
and "type" not in self.rope_parameters
|
||||
):
|
||||
self.rope_parameters["rope_type"] = "default"
|
||||
self.rope_parameters.setdefault("rope_theta", rope_theta)
|
||||
if mrope_section is not None:
|
||||
_normalize_mrope_parameters(self, list(mrope_section))
|
||||
|
||||
self.vocab_size = vocab_size
|
||||
self.hidden_size = hidden_size
|
||||
self.intermediate_size = intermediate_size
|
||||
self.num_hidden_layers = num_hidden_layers
|
||||
self.num_attention_heads = num_attention_heads
|
||||
self.num_key_value_heads = num_key_value_heads
|
||||
self.hidden_act = hidden_act
|
||||
self.use_qkv_bias = use_qkv_bias
|
||||
self.use_bias = use_bias
|
||||
self.rms_norm_eps = rms_norm_eps
|
||||
self.embedding_dropout = embedding_dropout
|
||||
self.attention_dropout = attention_dropout
|
||||
self.output_dropout = output_dropout
|
||||
self.initializer_range = initializer_range
|
||||
self.max_position_embeddings = max_position_embeddings
|
||||
self.rope_theta = rope_theta
|
||||
self.partial_rotary_factor = partial_rotary_factor
|
||||
self.use_cache = use_cache
|
||||
self.max_window_layers = max_window_layers
|
||||
self.num_experts = num_experts
|
||||
self.num_shared_experts = num_shared_experts
|
||||
self.num_experts_per_tok = num_experts_per_tok
|
||||
self.n_group = n_group
|
||||
self.topk_group = topk_group
|
||||
self.moe_intermediate_size = moe_intermediate_size
|
||||
self.moe_shared_expert_intermediate_size = moe_shared_expert_intermediate_size
|
||||
self.first_k_dense_replace = first_k_dense_replace
|
||||
self.head_dim = head_dim or hidden_size // num_attention_heads
|
||||
self.output_router_logits = output_router_logits
|
||||
self.use_qk_norm = use_qk_norm
|
||||
self.num_nextn_predict_layers = num_nextn_predict_layers
|
||||
self.mtp_loss_scaling_factor = mtp_loss_scaling_factor
|
||||
self.moe_router_enable_expert_bias = moe_router_enable_expert_bias
|
||||
self.routed_scaling_factor = routed_scaling_factor
|
||||
self.layer_group_size = layer_group_size
|
||||
self.kv_lora_rank = kv_lora_rank
|
||||
self.q_lora_rank = q_lora_rank
|
||||
self.qk_rope_head_dim = qk_rope_head_dim
|
||||
self.v_head_dim = v_head_dim
|
||||
self.qk_nope_head_dim = qk_nope_head_dim
|
||||
self.qk_head_dim = qk_nope_head_dim + qk_rope_head_dim
|
||||
self.rope_interleave = rope_interleave
|
||||
self.score_function = score_function
|
||||
self.scoring_func = scoring_func
|
||||
self.seq_aux = seq_aux
|
||||
self.topk_method = topk_method
|
||||
self.router_dtype = router_dtype
|
||||
self.gated_attention_proj_granularity_type = (
|
||||
gated_attention_proj_granularity_type
|
||||
)
|
||||
self.no_kda_lora = no_kda_lora
|
||||
self.kda_safe_gate = kda_safe_gate
|
||||
self.kda_lower_bound = kda_lower_bound
|
||||
self.short_conv_kernel_size = short_conv_kernel_size
|
||||
|
||||
expected_layer_types = _build_layer_types(
|
||||
num_hidden_layers,
|
||||
layer_group_size,
|
||||
)
|
||||
if layers_block_type is not None:
|
||||
legacy_type_mapping = {
|
||||
"attention": "full_attention",
|
||||
"mamba": "linear_attention",
|
||||
"conv": "linear_attention",
|
||||
}
|
||||
normalized_block_types = [
|
||||
legacy_type_mapping.get(layer_type, layer_type)
|
||||
for layer_type in layers_block_type
|
||||
]
|
||||
if layer_types is not None and layer_types != normalized_block_types:
|
||||
raise ValueError(
|
||||
"Bailing V3 VL has conflicting layer_types and layers_block_type"
|
||||
)
|
||||
layer_types = normalized_block_types
|
||||
if layer_types is not None and layer_types != expected_layer_types:
|
||||
raise ValueError(
|
||||
"Bailing V3 VL layer_types must match the layer_group_size schedule"
|
||||
)
|
||||
self.layer_types = expected_layer_types
|
||||
|
||||
super().__init__(
|
||||
architectures=architectures,
|
||||
pad_token_id=pad_token_id,
|
||||
eos_token_id=eos_token_id,
|
||||
tie_word_embeddings=tie_word_embeddings,
|
||||
**kwargs,
|
||||
)
|
||||
self.validate_layer_type()
|
||||
|
||||
|
||||
class BailingMoeV3VisionConfig(PreTrainedConfig):
|
||||
model_type = "qwen3_moe_vit"
|
||||
base_config_key = "vision_config"
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
depth: int = 27,
|
||||
hidden_size: int = 1152,
|
||||
hidden_act: str = "gelu_pytorch_tanh",
|
||||
intermediate_size: int = 4304,
|
||||
num_heads: int = 16,
|
||||
in_channels: int = 3,
|
||||
patch_size: int = 16,
|
||||
spatial_merge_size: int = 2,
|
||||
temporal_patch_size: int = 2,
|
||||
out_hidden_size: int = 3584,
|
||||
num_position_embeddings: int = 2304,
|
||||
deepstack_visual_indexes: list[int] | None = None,
|
||||
initializer_range: float = 0.02,
|
||||
**kwargs: Any,
|
||||
) -> None:
|
||||
kwargs.pop("model_type", None)
|
||||
super().__init__(**kwargs)
|
||||
self.depth = depth
|
||||
self.hidden_size = hidden_size
|
||||
self.hidden_act = hidden_act
|
||||
self.intermediate_size = intermediate_size
|
||||
self.num_heads = num_heads
|
||||
self.in_channels = in_channels
|
||||
self.patch_size = patch_size
|
||||
self.spatial_merge_size = spatial_merge_size
|
||||
self.temporal_patch_size = temporal_patch_size
|
||||
self.out_hidden_size = out_hidden_size
|
||||
self.num_position_embeddings = num_position_embeddings
|
||||
self.deepstack_visual_indexes = (
|
||||
[8, 16, 24]
|
||||
if deepstack_visual_indexes is None
|
||||
else deepstack_visual_indexes
|
||||
)
|
||||
self.initializer_range = initializer_range
|
||||
|
||||
|
||||
class BailingMoeV3VLConfig(PreTrainedConfig):
|
||||
model_type = "bailing_moe_v3_vl"
|
||||
keys_to_ignore_at_inference = ["past_key_values"]
|
||||
sub_configs = {
|
||||
"text_config": BailingMoeV3TextConfig,
|
||||
"vision_config": BailingMoeV3VisionConfig,
|
||||
}
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
text_config: BailingMoeV3TextConfig | dict[str, Any] | None = None,
|
||||
vision_config: BailingMoeV3VisionConfig | dict[str, Any] | None = None,
|
||||
image_token_id: int = 151655,
|
||||
video_token_id: int = 151656,
|
||||
vision_start_token_id: int = 151652,
|
||||
vision_end_token_id: int = 151653,
|
||||
tie_word_embeddings: bool = False,
|
||||
mrope_section: list[int] | tuple[int, ...] | None = None,
|
||||
architectures: list[str] | None = None,
|
||||
**kwargs: Any,
|
||||
) -> None:
|
||||
kwargs.pop("model_type", None)
|
||||
if mrope_section is None:
|
||||
mrope_section = [12, 10, 10]
|
||||
self.mrope_section = list(mrope_section)
|
||||
|
||||
if text_config is None:
|
||||
text_config = BailingMoeV3TextConfig()
|
||||
elif isinstance(text_config, dict):
|
||||
text_config = dict(text_config)
|
||||
nested_mrope_section = text_config.pop("mrope_section", None)
|
||||
if (
|
||||
nested_mrope_section is not None
|
||||
and list(nested_mrope_section) != self.mrope_section
|
||||
):
|
||||
raise ValueError(
|
||||
"Bailing V3 VL has conflicting M-RoPE sections: "
|
||||
f"text={nested_mrope_section}, "
|
||||
f"top-level={self.mrope_section}"
|
||||
)
|
||||
text_config = BailingMoeV3TextConfig(**text_config)
|
||||
self.text_config = text_config
|
||||
_normalize_mrope_parameters(self.text_config, self.mrope_section)
|
||||
|
||||
if vision_config is None:
|
||||
vision_config = BailingMoeV3VisionConfig()
|
||||
elif isinstance(vision_config, dict):
|
||||
vision_config = BailingMoeV3VisionConfig(**vision_config)
|
||||
self.vision_config = vision_config
|
||||
|
||||
self.image_token_id = image_token_id
|
||||
self.video_token_id = video_token_id
|
||||
self.vision_start_token_id = vision_start_token_id
|
||||
self.vision_end_token_id = vision_end_token_id
|
||||
if architectures is None:
|
||||
architectures = ["BailingMoeV3VLForConditionalGeneration"]
|
||||
|
||||
super().__init__(
|
||||
architectures=architectures,
|
||||
tie_word_embeddings=tie_word_embeddings,
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
|
||||
__all__ = [
|
||||
"BailingMoeV3TextConfig",
|
||||
"BailingMoeV3VisionConfig",
|
||||
"BailingMoeV3VLConfig",
|
||||
]
|
||||
@@ -1,75 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
# Adapted from
|
||||
# https://github.com/zai-org/ChatGLM2-6B
|
||||
from transformers import PreTrainedConfig
|
||||
|
||||
|
||||
class ChatGLMConfig(PreTrainedConfig):
|
||||
model_type = "chatglm"
|
||||
attribute_map = {
|
||||
"num_hidden_layers": "num_layers",
|
||||
"n_head_kv": "multi_query_group_num",
|
||||
}
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
num_layers=28,
|
||||
padded_vocab_size=65024,
|
||||
hidden_size=4096,
|
||||
ffn_hidden_size=13696,
|
||||
kv_channels=128,
|
||||
num_attention_heads=32,
|
||||
seq_length=2048,
|
||||
hidden_dropout=0.0,
|
||||
attention_dropout=0.0,
|
||||
layernorm_epsilon=1e-5,
|
||||
rmsnorm=True,
|
||||
apply_residual_connection_post_layernorm=False,
|
||||
post_layer_norm=True,
|
||||
add_bias_linear=False,
|
||||
add_qkv_bias=False,
|
||||
interleaved_qkv=False,
|
||||
bias_dropout_fusion=True,
|
||||
multi_query_attention=False,
|
||||
multi_query_group_num=1,
|
||||
apply_query_key_layer_scaling=True,
|
||||
attention_softmax_in_fp32=True,
|
||||
fp32_residual_connection=False,
|
||||
quantization_bit=0,
|
||||
pre_seq_len=None,
|
||||
prefix_projection=False,
|
||||
**kwargs,
|
||||
):
|
||||
self.num_layers = num_layers
|
||||
self.vocab_size = padded_vocab_size
|
||||
self.padded_vocab_size = padded_vocab_size
|
||||
self.hidden_size = hidden_size
|
||||
self.ffn_hidden_size = ffn_hidden_size
|
||||
self.kv_channels = kv_channels
|
||||
self.num_attention_heads = num_attention_heads
|
||||
self.seq_length = seq_length
|
||||
# It is to be compatible with long lora.
|
||||
self.max_position_embeddings = seq_length
|
||||
self.hidden_dropout = hidden_dropout
|
||||
self.attention_dropout = attention_dropout
|
||||
self.layernorm_epsilon = layernorm_epsilon
|
||||
self.rmsnorm = rmsnorm
|
||||
self.apply_residual_connection_post_layernorm = (
|
||||
apply_residual_connection_post_layernorm
|
||||
)
|
||||
self.post_layer_norm = post_layer_norm
|
||||
self.add_bias_linear = add_bias_linear
|
||||
self.add_qkv_bias = add_qkv_bias
|
||||
self.bias_dropout_fusion = bias_dropout_fusion
|
||||
self.multi_query_attention = multi_query_attention
|
||||
self.multi_query_group_num = multi_query_group_num
|
||||
self.apply_query_key_layer_scaling = apply_query_key_layer_scaling
|
||||
self.attention_softmax_in_fp32 = attention_softmax_in_fp32
|
||||
self.fp32_residual_connection = fp32_residual_connection
|
||||
self.quantization_bit = quantization_bit
|
||||
self.pre_seq_len = pre_seq_len
|
||||
self.prefix_projection = prefix_projection
|
||||
self.interleaved_qkv = interleaved_qkv
|
||||
super().__init__(**kwargs)
|
||||
@@ -1,63 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Configuration for ColModernVBERT visual document retrieval model.
|
||||
|
||||
ColModernVBERT combines SigLIP vision encoder + ModernBERT text encoder
|
||||
with a pixel shuffle connector and ColBERT-style 128-dim per-token embeddings.
|
||||
|
||||
Reference: https://huggingface.co/ModernVBERT/colmodernvbert-merged
|
||||
"""
|
||||
|
||||
from transformers import ModernBertConfig, PreTrainedConfig, SiglipVisionConfig
|
||||
|
||||
|
||||
class ColModernVBertConfig(PreTrainedConfig):
|
||||
model_type = "colmodernvbert"
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
embedding_dim: int = 128,
|
||||
image_token_id: int = 50407,
|
||||
pixel_shuffle_factor: int = 4,
|
||||
text_config: dict | None = None,
|
||||
vision_config: dict | None = None,
|
||||
**kwargs,
|
||||
):
|
||||
self.embedding_dim = embedding_dim
|
||||
self.image_token_id = image_token_id
|
||||
self.pixel_shuffle_factor = pixel_shuffle_factor
|
||||
|
||||
text_config = text_config or {}
|
||||
self.hidden_size = text_config.get("hidden_size", 768)
|
||||
|
||||
self.text_config = ModernBertConfig(
|
||||
vocab_size=text_config.get("vocab_size", 50408),
|
||||
hidden_size=text_config.get("hidden_size", 768),
|
||||
intermediate_size=text_config.get("intermediate_size", 1152),
|
||||
num_hidden_layers=text_config.get("num_hidden_layers", 22),
|
||||
num_attention_heads=text_config.get("num_attention_heads", 12),
|
||||
mlp_bias=text_config.get("mlp_bias", False),
|
||||
max_position_embeddings=text_config.get("max_position_embeddings", 8192),
|
||||
)
|
||||
|
||||
vision_config = vision_config or {}
|
||||
self.vision_config = SiglipVisionConfig(
|
||||
hidden_size=vision_config.get("hidden_size", 768),
|
||||
image_size=vision_config.get("image_size", 512),
|
||||
patch_size=vision_config.get("patch_size", 16),
|
||||
num_hidden_layers=vision_config.get("num_hidden_layers", 12),
|
||||
intermediate_size=vision_config.get("intermediate_size", 3072),
|
||||
num_attention_heads=vision_config.get("num_attention_heads", 12),
|
||||
)
|
||||
|
||||
# Ensure architectures is set so vLLM routes to our model class
|
||||
kwargs.setdefault("architectures", ["ColModernVBertForRetrieval"])
|
||||
super().__init__(**kwargs)
|
||||
|
||||
@property
|
||||
def image_seq_len(self) -> int:
|
||||
ps = self.vision_config.image_size // self.vision_config.patch_size
|
||||
return (ps * ps) // (self.pixel_shuffle_factor**2)
|
||||
|
||||
def get_text_config(self, **kwargs):
|
||||
return self.text_config
|
||||
@@ -1,56 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""ColPali configuration that extends PaliGemmaConfig with embedding projection
|
||||
fields. This allows ColPali models to be loaded without trust_remote_code
|
||||
by mapping their custom model_type (colpali) to a standard config class
|
||||
that vLLM understands.
|
||||
|
||||
Supported model_types:
|
||||
- colpali (vidore/colpali-v1.3-hf)
|
||||
"""
|
||||
|
||||
from transformers import PaliGemmaConfig
|
||||
|
||||
|
||||
class ColPaliConfig(PaliGemmaConfig):
|
||||
"""Configuration class for ColPali models.
|
||||
|
||||
Extends PaliGemmaConfig with additional fields used by ColPali variants
|
||||
for the embedding projection layer.
|
||||
"""
|
||||
|
||||
model_type = "colpali"
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
embedding_dim: int | None = None,
|
||||
embed_dim: int | None = None,
|
||||
dim: int | None = None,
|
||||
colbert_dim: int | None = None,
|
||||
pooling: str | None = None,
|
||||
vlm_config: dict | None = None,
|
||||
**kwargs,
|
||||
):
|
||||
# Store embedding projection config fields
|
||||
self.embedding_dim = embedding_dim
|
||||
self.embed_dim = embed_dim
|
||||
self.dim = dim
|
||||
self.colbert_dim = colbert_dim
|
||||
self.pooling = pooling
|
||||
|
||||
# The HF checkpoint nests PaliGemma config inside "vlm_config".
|
||||
# Flatten it so PaliGemmaConfig receives vision_config, text_config,
|
||||
# image_token_index, etc. directly.
|
||||
# Use setdefault to avoid overwriting keys already set (e.g.
|
||||
# model_type="colpali" would be clobbered by "paligemma" from
|
||||
# vlm_config).
|
||||
if vlm_config is not None:
|
||||
vlm_dict = (
|
||||
vlm_config if isinstance(vlm_config, dict) else vlm_config.to_dict()
|
||||
)
|
||||
_conflicting = {"model_type", "_name_or_path"}
|
||||
for key, value in vlm_dict.items():
|
||||
if key not in _conflicting:
|
||||
kwargs.setdefault(key, value)
|
||||
|
||||
super().__init__(**kwargs)
|
||||
@@ -1,147 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from transformers import NemotronHConfig
|
||||
from transformers.configuration_utils import PreTrainedConfig
|
||||
from transformers.models.siglip2.configuration_siglip2 import Siglip2VisionConfig
|
||||
|
||||
|
||||
def _normalize_mrope_parameters(config: NemotronHConfig) -> None:
|
||||
"""Normalize the checkpoint's mRoPE configuration for vLLM."""
|
||||
rope_parameters = dict(getattr(config, "rope_parameters", None) or {})
|
||||
mrope_section = rope_parameters.get("mrope_section")
|
||||
if mrope_section is None:
|
||||
return
|
||||
|
||||
rope_parameters["mrope_section"] = list(mrope_section)
|
||||
rope_parameters.setdefault("mrope_interleaved", True)
|
||||
config.rope_parameters = rope_parameters
|
||||
|
||||
|
||||
class Cosmos3EdgeTextConfig(NemotronHConfig):
|
||||
"""Dense Cosmos3 Edge text config backed by Nemotron-H layers."""
|
||||
|
||||
model_type = "cosmos3_edge_text"
|
||||
has_no_defaults_at_init = True
|
||||
ignore_keys_at_rope_validation = {"mrope_section", "mrope_interleaved"}
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
num_hidden_layers: int = 28,
|
||||
hidden_act: str = "relu2",
|
||||
rms_norm_eps: float = 1e-5,
|
||||
layers_block_type: list[str] | None = None,
|
||||
**kwargs,
|
||||
) -> None:
|
||||
# Each decoder block is an attention layer followed by an MLP layer
|
||||
if layers_block_type is None:
|
||||
layers_block_type = ["full_attention", "mlp"] * num_hidden_layers
|
||||
super().__init__(
|
||||
layers_block_type=layers_block_type,
|
||||
mlp_hidden_act=hidden_act,
|
||||
layer_norm_epsilon=rms_norm_eps,
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
def to_dict(self) -> dict:
|
||||
config_dict = super().to_dict()
|
||||
config_dict["num_hidden_layers"] = self.num_hidden_layers // 2
|
||||
config_dict["hidden_act"] = config_dict.pop("mlp_hidden_act")
|
||||
config_dict["rms_norm_eps"] = config_dict.pop("layer_norm_epsilon")
|
||||
return config_dict
|
||||
|
||||
|
||||
class Cosmos3EdgeProjectorConfig(PreTrainedConfig):
|
||||
model_type = "cosmos3_edge_projector"
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
input_hidden_size: int = 1024,
|
||||
use_postshuffle_norm: bool = False,
|
||||
spatial_merge_size: int = 2,
|
||||
merger_intermediate_size: int = 11520,
|
||||
out_hidden_size: int = 1024,
|
||||
**kwargs,
|
||||
) -> None:
|
||||
super().__init__(**kwargs)
|
||||
self.input_hidden_size = input_hidden_size
|
||||
self.use_postshuffle_norm = use_postshuffle_norm
|
||||
self.spatial_merge_size = spatial_merge_size
|
||||
self.merger_intermediate_size = merger_intermediate_size
|
||||
self.out_hidden_size = out_hidden_size
|
||||
|
||||
|
||||
class Cosmos3EdgeVisionConfig(Siglip2VisionConfig):
|
||||
model_type = "cosmos3_edge_vision"
|
||||
|
||||
|
||||
class Cosmos3EdgeConfig(PreTrainedConfig):
|
||||
model_type = "cosmos3_edge"
|
||||
keys_to_ignore_at_inference = ["past_key_values"]
|
||||
has_no_defaults_at_init = True
|
||||
sub_configs = {
|
||||
"vision_config": Cosmos3EdgeVisionConfig,
|
||||
"projector_config": Cosmos3EdgeProjectorConfig,
|
||||
"text_config": Cosmos3EdgeTextConfig,
|
||||
}
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
text_config: Cosmos3EdgeTextConfig | dict | None = None,
|
||||
vision_config: Cosmos3EdgeVisionConfig | dict | None = None,
|
||||
projector_config: Cosmos3EdgeProjectorConfig | dict | None = None,
|
||||
image_token_id: int = 19,
|
||||
video_token_id: int = 18,
|
||||
vision_start_token_id: int = 20,
|
||||
vision_end_token_id: int = 21,
|
||||
tie_word_embeddings: bool = False,
|
||||
**kwargs,
|
||||
) -> None:
|
||||
if text_config is None:
|
||||
text_config = {}
|
||||
if vision_config is None:
|
||||
vision_config = {}
|
||||
if projector_config is None:
|
||||
projector_config = {}
|
||||
|
||||
self.text_config = (
|
||||
Cosmos3EdgeTextConfig(**text_config)
|
||||
if isinstance(text_config, dict)
|
||||
else text_config
|
||||
)
|
||||
_normalize_mrope_parameters(self.text_config)
|
||||
|
||||
self.vision_config = (
|
||||
Cosmos3EdgeVisionConfig(**vision_config)
|
||||
if isinstance(vision_config, dict)
|
||||
else vision_config
|
||||
)
|
||||
self.projector_config = (
|
||||
Cosmos3EdgeProjectorConfig(**projector_config)
|
||||
if isinstance(projector_config, dict)
|
||||
else projector_config
|
||||
)
|
||||
|
||||
# Qwen3-VL processing and M-RoPE read these attributes from the
|
||||
# vision config, while this checkpoint stores them in projector config.
|
||||
self.vision_config.spatial_merge_size = self.projector_config.spatial_merge_size
|
||||
self.vision_config.temporal_patch_size = 1
|
||||
self.vision_config.out_hidden_size = self.projector_config.out_hidden_size
|
||||
|
||||
self.image_token_id = image_token_id
|
||||
self.video_token_id = video_token_id
|
||||
self.vision_start_token_id = vision_start_token_id
|
||||
self.vision_end_token_id = vision_end_token_id
|
||||
|
||||
super().__init__(
|
||||
tie_word_embeddings=tie_word_embeddings,
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
|
||||
__all__ = [
|
||||
"Cosmos3EdgeConfig",
|
||||
"Cosmos3EdgeProjectorConfig",
|
||||
"Cosmos3EdgeTextConfig",
|
||||
"Cosmos3EdgeVisionConfig",
|
||||
]
|
||||
@@ -1,62 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from typing import Any
|
||||
|
||||
from transformers import PreTrainedConfig
|
||||
|
||||
|
||||
class DeepseekV41Config(PreTrainedConfig):
|
||||
"""DeepSeek V4.1 config.
|
||||
|
||||
The HF config nests the text model under ``text_config`` and the vision
|
||||
tower under ``vision_config``. The vLLM model code (ported from
|
||||
``deepseek_v4``) reads flat attributes, so both sub-configs are flattened
|
||||
onto the top level here: text fields are exposed as-is, vision fields
|
||||
with the ``vision_*`` naming used by ``deepseek_v41.common.vision``.
|
||||
"""
|
||||
|
||||
model_type = "deepseek_v41"
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
text_config: dict[str, Any] | None = None,
|
||||
vision_config: dict[str, Any] | None = None,
|
||||
**kwargs,
|
||||
):
|
||||
text_config = dict(text_config or {})
|
||||
vision_config = dict(vision_config or {})
|
||||
|
||||
# ``rope_scaling`` is a property in Transformers v5 (backed by
|
||||
# ``rope_parameters``); capture it here and restore after
|
||||
# super().__init__, which re-standardizes rope params.
|
||||
rope_scaling = text_config.pop("rope_scaling", None)
|
||||
|
||||
for key, value in text_config.items():
|
||||
if key == "model_type":
|
||||
continue
|
||||
# Don't clobber PreTrainedConfig properties (e.g. is_encoder_decoder).
|
||||
if isinstance(getattr(type(self), key, None), property):
|
||||
continue
|
||||
setattr(self, key, value)
|
||||
|
||||
super().__init__(**kwargs)
|
||||
if rope_scaling is not None:
|
||||
self.rope_parameters = rope_scaling
|
||||
|
||||
# The v4.1 quantization_config carries the expert dtype; the model
|
||||
# code reads it from the top-level hf_config.
|
||||
quant_cfg = getattr(self, "quantization_config", None) or {}
|
||||
if not hasattr(self, "expert_dtype") and "expert_dtype" in quant_cfg:
|
||||
self.expert_dtype = quant_cfg["expert_dtype"]
|
||||
|
||||
vision_n_layers = vision_config.get("num_hidden_layers", 0)
|
||||
self.vision_n_layers = vision_n_layers
|
||||
self.vision_dim = vision_config.get("hidden_size", 1024)
|
||||
self.vision_n_heads = vision_config.get("num_attention_heads", 16)
|
||||
self.vision_inter_dim = vision_config.get("intermediate_size", 2816)
|
||||
self.vision_patch_size = vision_config.get("patch_size", 14)
|
||||
self.vision_rope_theta = vision_config.get("rope_theta", 10000.0)
|
||||
self.vision_downsample_ratio = vision_config.get("downsample_ratio", 3)
|
||||
self.vision_max_n_token = vision_config.get("max_image_tokens", 1024)
|
||||
self.vision_min_pixels = vision_config.get("min_pixels", 295936)
|
||||
self.vision_max_wh_ratio = vision_config.get("max_wh_ratio")
|
||||
@@ -1,126 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
# adapted from https://github.com/deepseek-ai/DeepSeek-VL2/blob/faf18023f24b962b32d9f0a2d89e402a8d383a78/deepseek_vl2/models/modeling_deepseek_vl_v2.py#L115-L268
|
||||
|
||||
from huggingface_hub.dataclasses import strict
|
||||
from transformers import DeepseekV2Config, PreTrainedConfig
|
||||
|
||||
|
||||
class VisionEncoderConfig(PreTrainedConfig):
|
||||
model_type: str = "vision"
|
||||
|
||||
model_name: str = "vit_so400m_patch14_siglip_384.webli"
|
||||
image_size: int = 384
|
||||
patch_size: int = 16
|
||||
width: int = 1024
|
||||
layers: int = 24
|
||||
heads: int = 16
|
||||
mlp_ratio: int = 4
|
||||
global_pool: str = "map"
|
||||
ignore_head: bool = True
|
||||
class_token: bool = False
|
||||
num_classes: int = 0
|
||||
use_checkpoint: bool = False
|
||||
weight_init: str = "skip"
|
||||
deterministic: bool = False
|
||||
num_recomputing_layers: int = 0
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
model_name: str = "vit_so400m_patch14_siglip_384.webli",
|
||||
image_size: int = 384,
|
||||
patch_size: int = 16,
|
||||
width: int = 1024,
|
||||
layers: int = 24,
|
||||
heads: int = 16,
|
||||
mlp_ratio: int = 4,
|
||||
global_pool: str = "map",
|
||||
ignore_head: bool = True,
|
||||
class_token: bool = False,
|
||||
num_classes: int = 0,
|
||||
use_checkpoint: bool = False,
|
||||
**kwargs,
|
||||
):
|
||||
self.model_name = model_name
|
||||
self.image_size = image_size
|
||||
self.patch_size = patch_size
|
||||
self.width = width
|
||||
self.layers = layers
|
||||
self.heads = heads
|
||||
self.mlp_ratio = mlp_ratio
|
||||
self.global_pool = global_pool
|
||||
self.ignore_head = ignore_head
|
||||
self.class_token = class_token
|
||||
self.num_classes = num_classes
|
||||
self.use_checkpoint = use_checkpoint
|
||||
|
||||
super().__init__(**kwargs)
|
||||
|
||||
|
||||
class MlpProjectorConfig(PreTrainedConfig):
|
||||
model_type = "mlp_projector"
|
||||
projector_type: str = "downsample_mlp_gelu"
|
||||
input_dim: int = 1152
|
||||
n_embed: int = 2048
|
||||
depth: int = 2
|
||||
mlp_ratio: int = 1
|
||||
downsample_ratio: int = 2
|
||||
token_pooling: bool = False
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
projector_type: str = "downsample_mlp_gelu",
|
||||
input_dim: int = 1152,
|
||||
n_embed: int = 2048,
|
||||
depth: int = 2,
|
||||
mlp_ratio: int = 1,
|
||||
downsample_ratio: int = 2,
|
||||
**kwargs,
|
||||
):
|
||||
self.projector_type = projector_type
|
||||
self.input_dim = input_dim
|
||||
self.n_embed = n_embed
|
||||
self.depth = depth
|
||||
self.mlp_ratio = mlp_ratio
|
||||
self.downsample_ratio = downsample_ratio
|
||||
|
||||
super().__init__(**kwargs)
|
||||
|
||||
|
||||
@strict
|
||||
class DeepseekVLV2TextConfig(DeepseekV2Config):
|
||||
kv_lora_rank: int | None = DeepseekV2Config.kv_lora_rank
|
||||
|
||||
|
||||
class DeepseekVLV2Config(PreTrainedConfig):
|
||||
model_type = "deepseek_vl_v2"
|
||||
|
||||
tile_tag: str = "2D"
|
||||
global_view_pos: str = "head"
|
||||
candidate_resolutions: tuple[tuple[int, int]] = ((384, 384),)
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
tile_tag: str = "tile_tag",
|
||||
global_view_pos: str = "head",
|
||||
candidate_resolutions: tuple[tuple[int, int]] = ((384, 384),),
|
||||
**kwargs,
|
||||
):
|
||||
architectures = kwargs.setdefault("architectures", ["DeepseekVLV2ForCausalLM"])
|
||||
|
||||
self.vision_config = VisionEncoderConfig(**kwargs.pop("vision_config", {}))
|
||||
self.projector_config = MlpProjectorConfig(**kwargs.pop("projector_config", {}))
|
||||
self.text_config = DeepseekVLV2TextConfig(**kwargs.pop("language_config", {}))
|
||||
|
||||
self.tile_tag = tile_tag
|
||||
self.global_view_pos = global_view_pos
|
||||
self.candidate_resolutions = candidate_resolutions
|
||||
self.vocab_size = self.text_config.vocab_size
|
||||
|
||||
# update model_type for OCR models
|
||||
if "DeepseekOCRForCausalLM" in architectures:
|
||||
kwargs["model_type"] = "deepseek_ocr"
|
||||
elif "DeepseekOCR2ForCausalLM" in architectures:
|
||||
kwargs["model_type"] = "deepseek_ocr2"
|
||||
super().__init__(**kwargs)
|
||||
@@ -1,25 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from transformers import DeepseekV3Config
|
||||
|
||||
|
||||
class Dots3NoteConfig(DeepseekV3Config):
|
||||
"""Configuration for the Dots3Note hybrid DSA/SWA model."""
|
||||
|
||||
model_type = "dots3_note"
|
||||
|
||||
def __init__(self, **kwargs):
|
||||
# These fields are absent from existing Note config.json files. Do
|
||||
# not inherit DeepSeek-V3's 8-group/4-group router defaults: Note was
|
||||
# trained with an ungrouped (1/1) noaux_tc router, as defined by the
|
||||
# reference Dots3Config in SGLang haimain. A different grouping
|
||||
# changes the selected experts at every MoE layer.
|
||||
kwargs.setdefault("n_group", 1)
|
||||
kwargs.setdefault("topk_group", 1)
|
||||
# Dots3 projects the indexer RoPE coordinates in adjacent (GPT-J)
|
||||
# pairs. DeepSeek-V3.2 defaults to the split-half NeoX layout when
|
||||
# this flag is absent, which rotates different learned coordinates.
|
||||
kwargs.setdefault("indexer_rope_interleave", True)
|
||||
kwargs.setdefault("num_nextn_predict_layers", 1)
|
||||
super().__init__(**kwargs)
|
||||
@@ -1,89 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
# Adapted from
|
||||
# https://huggingface.co/tiiuae/falcon-7b/blob/main/configuration_RW.py
|
||||
# Copyright 2023 The vLLM team.
|
||||
# Copyright 2022 the Big Science Workshop and HuggingFace Inc. team.
|
||||
# All rights reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
"""Falcon configuration."""
|
||||
|
||||
from transformers.configuration_utils import PreTrainedConfig
|
||||
|
||||
|
||||
class RWConfig(PreTrainedConfig):
|
||||
model_type = "falcon"
|
||||
keys_to_ignore_at_inference = ["past_key_values"]
|
||||
attribute_map = {
|
||||
"num_hidden_layers": "n_layer",
|
||||
"num_attention_heads": "n_head",
|
||||
"num_kv_heads": "n_head_kv",
|
||||
}
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
vocab_size=250880,
|
||||
hidden_size=64,
|
||||
n_layer=2,
|
||||
n_head=8,
|
||||
layer_norm_epsilon=1e-5,
|
||||
initializer_range=0.02,
|
||||
use_cache=True,
|
||||
bos_token_id=1,
|
||||
eos_token_id=2,
|
||||
hidden_dropout=0.0,
|
||||
attention_dropout=0.0,
|
||||
multi_query=True,
|
||||
n_head_kv=None,
|
||||
alibi=False,
|
||||
bias=False,
|
||||
parallel_attn=False,
|
||||
new_decoder_architecture=False,
|
||||
**kwargs,
|
||||
) -> None:
|
||||
self.vocab_size = vocab_size
|
||||
# Backward compatibility with n_embed kwarg
|
||||
n_embed = kwargs.pop("n_embed", None)
|
||||
self.hidden_size = hidden_size if n_embed is None else n_embed
|
||||
self.n_layer = n_layer
|
||||
self.n_head = n_head
|
||||
self.layer_norm_epsilon = layer_norm_epsilon
|
||||
self.initializer_range = initializer_range
|
||||
self.use_cache = use_cache
|
||||
self.hidden_dropout = hidden_dropout
|
||||
self.attention_dropout = attention_dropout
|
||||
|
||||
self.bos_token_id = bos_token_id
|
||||
self.eos_token_id = eos_token_id
|
||||
self.multi_query = multi_query
|
||||
self.n_head_kv = 1 if n_head_kv is None else n_head_kv
|
||||
self.alibi = alibi
|
||||
self.bias = bias
|
||||
self.parallel_attn = parallel_attn
|
||||
self.new_decoder_architecture = new_decoder_architecture
|
||||
|
||||
if self.hidden_size == 8192:
|
||||
# Hack for falcon-40b
|
||||
self.new_decoder_architecture = True
|
||||
|
||||
super().__init__(bos_token_id=bos_token_id, eos_token_id=eos_token_id, **kwargs)
|
||||
|
||||
@property
|
||||
def head_dim(self):
|
||||
return self.hidden_size // self.n_head
|
||||
|
||||
@property
|
||||
def rotary(self):
|
||||
return not self.alibi
|
||||
@@ -1,120 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from transformers import CONFIG_MAPPING, PreTrainedConfig
|
||||
|
||||
# NOTE: Temporary shim for FunAudioChat checkpoints.
|
||||
# These checkpoints use `model_type="funaudiochat"`, which is not currently
|
||||
# recognized by released Transformers, and the public checkpoint does not
|
||||
# provide an `auto_map` to enable `trust_remote_code=True`.
|
||||
# Remove this file once Transformers adds native support (or the checkpoint
|
||||
# provides an `auto_map`) and vLLM can rely on `AutoConfig.from_pretrained()`.
|
||||
|
||||
|
||||
class FunAudioChatAudioEncoderConfig(PreTrainedConfig):
|
||||
model_type = "funaudiochat_audio_encoder"
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
_attn_implementation: str | None = None,
|
||||
num_mel_bins: int = 128,
|
||||
encoder_layers: int = 32,
|
||||
encoder_attention_heads: int = 20,
|
||||
encoder_ffn_dim: int = 5120,
|
||||
d_model: int = 1280,
|
||||
dropout: float = 0.0,
|
||||
attention_dropout: float = 0.0,
|
||||
activation_function: str = "gelu",
|
||||
activation_dropout: float = 0.0,
|
||||
scale_embedding: bool = False,
|
||||
initializer_range: float = 0.02,
|
||||
max_source_positions: int = 1500,
|
||||
n_window: int = 100,
|
||||
output_dim: int = 3584,
|
||||
bos_token_id: int | None = None,
|
||||
codebook_size: int | None = None,
|
||||
continuous_features_mode: str = "replace",
|
||||
crq_transformer_config: dict | None = None,
|
||||
eos_token_id: int | None = None,
|
||||
group_size: int = 5,
|
||||
enable_audio_invert_tower: bool = True,
|
||||
pad_token_id: int | None = None,
|
||||
**kwargs,
|
||||
) -> None:
|
||||
attn_impl = kwargs.pop("_attn_implementation", None) or _attn_implementation
|
||||
super().__init__(**kwargs)
|
||||
# Match HF default for attention implementation selection.
|
||||
self._attn_implementation = attn_impl or "sdpa"
|
||||
|
||||
self.num_mel_bins = num_mel_bins
|
||||
self.d_model = d_model
|
||||
self.encoder_layers = encoder_layers
|
||||
self.encoder_attention_heads = encoder_attention_heads
|
||||
self.encoder_ffn_dim = encoder_ffn_dim
|
||||
self.dropout = dropout
|
||||
self.attention_dropout = attention_dropout
|
||||
self.activation_function = activation_function
|
||||
self.activation_dropout = activation_dropout
|
||||
self.num_hidden_layers = encoder_layers
|
||||
self.initializer_range = initializer_range
|
||||
self.scale_embedding = scale_embedding
|
||||
self.max_source_positions = max_source_positions
|
||||
self.n_window = n_window
|
||||
self.output_dim = output_dim
|
||||
|
||||
self.bos_token_id = bos_token_id
|
||||
self.codebook_size = codebook_size
|
||||
self.continuous_features_mode = continuous_features_mode
|
||||
self.crq_transformer_config = crq_transformer_config
|
||||
self.eos_token_id = eos_token_id
|
||||
self.group_size = group_size
|
||||
self.enable_audio_invert_tower = enable_audio_invert_tower
|
||||
self.pad_token_id = pad_token_id
|
||||
|
||||
|
||||
class FunAudioChatConfig(PreTrainedConfig):
|
||||
model_type = "funaudiochat"
|
||||
attribute_map = {
|
||||
"audio_token_id": "audio_token_index",
|
||||
}
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
audio_config: PreTrainedConfig | dict | None = None,
|
||||
text_config: PreTrainedConfig | dict | None = None,
|
||||
audio_token_index: int = 151646,
|
||||
ignore_index: int = -100,
|
||||
hidden_size: int | None = None,
|
||||
**kwargs,
|
||||
) -> None:
|
||||
self.audio_token_index = audio_token_index
|
||||
self.ignore_index = ignore_index
|
||||
|
||||
if audio_config is None:
|
||||
self.audio_config = FunAudioChatAudioEncoderConfig()
|
||||
elif isinstance(audio_config, dict):
|
||||
default_model_type = FunAudioChatAudioEncoderConfig.model_type
|
||||
audio_config.setdefault("model_type", default_model_type)
|
||||
self.audio_config = FunAudioChatAudioEncoderConfig(**audio_config)
|
||||
else:
|
||||
self.audio_config = audio_config
|
||||
|
||||
if text_config is None:
|
||||
self.text_config = CONFIG_MAPPING["qwen2"]()
|
||||
elif isinstance(text_config, dict):
|
||||
# Default to qwen2 for backwards compatibility; FunAudioChat uses
|
||||
# qwen3 in practice for recent checkpoints.
|
||||
text_config.setdefault("model_type", "qwen2")
|
||||
self.text_config = CONFIG_MAPPING[text_config["model_type"]](**text_config)
|
||||
else:
|
||||
self.text_config = text_config
|
||||
|
||||
self.hidden_size = (
|
||||
int(self.text_config.hidden_size)
|
||||
if hidden_size is None
|
||||
else int(hidden_size)
|
||||
)
|
||||
|
||||
super().__init__(**kwargs)
|
||||
@@ -1,173 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from huggingface_hub.dataclasses import strict
|
||||
from transformers.configuration_utils import PreTrainedConfig
|
||||
from transformers.modeling_rope_utils import RopeParameters
|
||||
|
||||
|
||||
@strict
|
||||
class HYV4Config(PreTrainedConfig):
|
||||
r"""Configuration class for the HY V4 (`HYV4ForCausalLM`) model.
|
||||
|
||||
Kept field-for-field in sync with
|
||||
`transformers.models.hy_v4.configuration_hy_v4.HYV4Config` so that a
|
||||
checkpoint resolves to the same values through either class. Registering it
|
||||
in vLLM's `_CONFIG_REGISTRY` lets the architecture load without
|
||||
`trust_remote_code` and without an `auto_map` module next to the weights.
|
||||
|
||||
HYV4 is a mixture-of-experts causal language model using Multi-head Latent
|
||||
Attention (MLA), DeepSeek-style sparse attention (DSA), gated MLA,
|
||||
learnable attention sinks, and independent Hyper-Connections (iHC).
|
||||
|
||||
The `layer_types`, `num_nextn_predict_layers` and `mtp_loss_factor` fields
|
||||
below are vLLM-only additions: upstream leaves them to `**kwargs`, but
|
||||
vLLM reads them directly (sparse-attention selection and MTP speculative
|
||||
decoding), so they are declared to give checkpoints that omit them a
|
||||
working default instead of a `None`.
|
||||
|
||||
Args:
|
||||
mlp_layer_types: Per-layer MLP kind, `"dense"` or `"sparse"`. Defaults
|
||||
to one dense layer followed by sparse MoE layers.
|
||||
layer_types: Per-layer attention kind, e.g. `"full_attention"` or
|
||||
`"deepseek_sparse_attention"`.
|
||||
indexer_types: Per-layer DSA indexer kind, `"full"` or `"shared"`. A
|
||||
shared layer reuses the most recent full indexer in the same
|
||||
forward request.
|
||||
index_topk: Maximum number of key positions selected by each DSA query.
|
||||
index_head_dim: Hidden dimension of each DSA indexer head.
|
||||
index_n_heads: Number of DSA indexer heads.
|
||||
enable_lm_head_fp32: Whether the language-model head emits float32
|
||||
logits. Surfaced as ``head_dtype`` so `LogitsProcessor` accumulates
|
||||
the projection into fp32 while the weight stays in the model dtype.
|
||||
enable_ihc: Whether independent Hyper-Connections are enabled.
|
||||
hc_mult: Number of hidden-state channels maintained by iHC.
|
||||
hc_magnitude: Scale applied to the iHC post-gating branch.
|
||||
hc_eps: Numerical epsilon added to iHC sigmoid gates.
|
||||
gated_mla: Whether to gate the MLA output.
|
||||
gating_type: MLA gate granularity, `"elementwise"` or `"headwise"`.
|
||||
learnable_sink: Whether to add a learned per-head attention sink.
|
||||
learnable_sink_init: Initial value of each learned attention-sink
|
||||
logit.
|
||||
swiglu_limit: Magnitude of the routed-expert SwiGLU clamp. Values at or
|
||||
below zero disable the clamp.
|
||||
num_nextn_predict_layers: Number of MTP layers in the checkpoint.
|
||||
mtp_loss_factor: Training-time MTP loss weight (unused at inference).
|
||||
|
||||
"""
|
||||
|
||||
model_type = "hy_v4"
|
||||
keys_to_ignore_at_inference = ["past_key_values"]
|
||||
attribute_map = {
|
||||
"num_local_experts": "n_routed_experts",
|
||||
}
|
||||
base_model_tp_plan = {
|
||||
"layers.*.self_attn.q_b_proj": "colwise",
|
||||
"layers.*.self_attn.kv_a_proj_with_mqa": "mla_kv_a_proj",
|
||||
"layers.*.self_attn.kv_b_proj": "colwise",
|
||||
"layers.*.self_attn.o_proj": "rowwise",
|
||||
"layers.*.self_attn.linear_gate": "colwise",
|
||||
"layers.*.mlp.experts.gate_up_proj": "packed_colwise",
|
||||
"layers.*.mlp.experts.down_proj": "rowwise",
|
||||
"layers.*.mlp.experts": "moe_tp_experts",
|
||||
"layers.*.mlp.shared_experts.gate_proj": "colwise",
|
||||
"layers.*.mlp.shared_experts.up_proj": "colwise",
|
||||
"layers.*.mlp.shared_experts.down_proj": "rowwise",
|
||||
"layers.*.mlp.gate_proj": "colwise",
|
||||
"layers.*.mlp.up_proj": "colwise",
|
||||
"layers.*.mlp.down_proj": "rowwise",
|
||||
}
|
||||
base_model_pp_plan = {
|
||||
"embed_tokens": (["input_ids"], ["inputs_embeds"]),
|
||||
"layers": (["hidden_states", "attention_mask"], ["hidden_states"]),
|
||||
"norm": (["hidden_states"], ["hidden_states"]),
|
||||
}
|
||||
|
||||
vocab_size: int = 120832
|
||||
hidden_size: int = 2816
|
||||
intermediate_size: int = 6912
|
||||
moe_intermediate_size: int = 768
|
||||
num_hidden_layers: int = 34
|
||||
num_attention_heads: int = 32
|
||||
num_key_value_heads: int = 32
|
||||
head_dim: int = 256
|
||||
hidden_act: str = "silu"
|
||||
max_position_embeddings: int = 262144
|
||||
initializer_range: float = 0.006
|
||||
rms_norm_eps: float = 1e-5
|
||||
use_cache: bool = True
|
||||
pad_token_id: int | None = 120002
|
||||
bos_token_id: int | None = 120000
|
||||
eos_token_id: int | list[int] | None = 120025
|
||||
tie_word_embeddings: bool = False
|
||||
attention_bias: bool = False
|
||||
attention_dropout: float = 0.0
|
||||
n_routed_experts: int = 256
|
||||
n_shared_experts: int = 1
|
||||
num_experts_per_tok: int = 8
|
||||
routed_scaling_factor: float = 2.827
|
||||
norm_topk_prob: bool = True
|
||||
q_lora_rank: int = 1536
|
||||
kv_lora_rank: int = 512
|
||||
qk_nope_head_dim: int = 192
|
||||
qk_rope_head_dim: int = 64
|
||||
v_head_dim: int = 256
|
||||
mlp_layer_types: list[str] | None = None
|
||||
layer_types: list[str] | None = None
|
||||
index_topk: int = 2048
|
||||
index_head_dim: int = 128
|
||||
index_n_heads: int = 16
|
||||
indexer_types: list[str] | None = None
|
||||
enable_lm_head_fp32: bool = True
|
||||
enable_ihc: bool = True
|
||||
hc_mult: int = 4
|
||||
hc_magnitude: float = 2.0
|
||||
hc_eps: float = 1e-6
|
||||
gated_mla: bool = True
|
||||
gating_type: str = "elementwise"
|
||||
learnable_sink: bool = True
|
||||
learnable_sink_init: float = 0.0
|
||||
swiglu_limit: float = 10.0
|
||||
rope_parameters: RopeParameters | dict | None = None
|
||||
num_nextn_predict_layers: int = 1
|
||||
mtp_loss_factor: float = 0.1
|
||||
|
||||
def __post_init__(self, **kwargs):
|
||||
self.qk_head_dim = self.qk_nope_head_dim + self.qk_rope_head_dim
|
||||
# RoPE applies only to the rope slice, so point `head_dim` at it: the
|
||||
# inherited rotary embedding reads `config.head_dim` and then computes
|
||||
# the right frequencies with no override.
|
||||
self.head_dim = self.qk_rope_head_dim
|
||||
|
||||
if self.mlp_layer_types is None:
|
||||
self.mlp_layer_types = ["dense"] * min(1, self.num_hidden_layers) + [
|
||||
"sparse"
|
||||
] * max(self.num_hidden_layers - 1, 0)
|
||||
if self.indexer_types is None:
|
||||
self.indexer_types = [
|
||||
"full" if layer_idx == 0 or (layer_idx - 1) % 4 == 0 else "shared"
|
||||
for layer_idx in range(self.num_hidden_layers)
|
||||
]
|
||||
# `PreTrainedConfig` validates `layer_types` against a fixed allow-list
|
||||
# in which "deepseek_sparse_attention" is the only accepted spelling of
|
||||
# a DSA layer. The model treats "sparse_attention" and "sparse" as
|
||||
# equivalent, so fold them in before the base class validates.
|
||||
if self.layer_types is not None:
|
||||
self.layer_types = [
|
||||
"deepseek_sparse_attention"
|
||||
if lt in ("sparse_attention", "sparse")
|
||||
else lt
|
||||
for lt in self.layer_types
|
||||
]
|
||||
|
||||
# `ModelConfig.head_dtype` reads this off the HF config, so an fp32 head
|
||||
# needs no model-side dtype juggling: LogitsProcessor picks the
|
||||
# `torch.mm(out_dtype=float32)` path and leaves the weight in the model
|
||||
# dtype. Only fill it in when the checkpoint (or --hf-overrides) has not
|
||||
# already pinned a value.
|
||||
if self.enable_lm_head_fp32 and getattr(self, "head_dtype", None) is None:
|
||||
self.head_dtype = "float32"
|
||||
|
||||
super().__post_init__(**kwargs)
|
||||
|
||||
|
||||
__all__ = ["HYV4Config"]
|
||||
@@ -1,99 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from __future__ import annotations
|
||||
|
||||
from transformers import Qwen3Config
|
||||
from transformers.models.siglip2.configuration_siglip2 import Siglip2VisionConfig
|
||||
|
||||
|
||||
class PixelShuffleSiglip2VisionConfig(Siglip2VisionConfig):
|
||||
"""Vision configuration for Isaac with Pixel Shuffle support.
|
||||
|
||||
Extends Siglip2VisionConfig with additional fields for pixel shuffle.
|
||||
"""
|
||||
|
||||
model_type = "pixel_shuffle_siglip2"
|
||||
base_config_key = "vision_config"
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
pixel_shuffle_scale_factor: int = 1,
|
||||
num_patches: int = 256,
|
||||
**kwargs,
|
||||
):
|
||||
super().__init__(**kwargs)
|
||||
|
||||
# Add our custom fields
|
||||
self.pixel_shuffle_scale_factor = pixel_shuffle_scale_factor
|
||||
self.num_patches = num_patches
|
||||
|
||||
|
||||
class IsaacConfig(Qwen3Config):
|
||||
"""Configuration class for Isaac multimodal model."""
|
||||
|
||||
model_type = "isaac"
|
||||
sub_configs = {
|
||||
"vision_config": PixelShuffleSiglip2VisionConfig,
|
||||
"text_config": Qwen3Config,
|
||||
}
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
text_config=None,
|
||||
vision_config=None,
|
||||
vision_patch_size: int = 16,
|
||||
vision_max_num_patches: int = 256,
|
||||
vision_min_num_patches: int | None = None,
|
||||
pixel_shuffle_scale: int = 1,
|
||||
max_sequence_length: int = 16384,
|
||||
vision_token: str = "<image>",
|
||||
vision_attn_implementation: str | None = None,
|
||||
**kwargs,
|
||||
):
|
||||
if isinstance(text_config, dict):
|
||||
# from HF config
|
||||
self.text_config = self.sub_configs["text_config"](**text_config)
|
||||
elif text_config is None:
|
||||
# For BC use all kwargs to init text config.
|
||||
self.text_config = self.sub_configs["text_config"](**kwargs)
|
||||
else:
|
||||
# from Qwen3Config
|
||||
self.text_config = text_config
|
||||
|
||||
# EventStreamProcessor parameters (for backward compatibility)
|
||||
self.video_patch_size = vision_patch_size
|
||||
self.vision_max_num_patches = vision_max_num_patches
|
||||
self.vision_min_num_patches = vision_min_num_patches
|
||||
self.pixel_shuffle_scale = pixel_shuffle_scale
|
||||
|
||||
# Processing parameters
|
||||
self.max_sequence_length = max_sequence_length
|
||||
self.vision_token = vision_token
|
||||
|
||||
# Handle vision config - PixelShuffleSiglip2VisionConfig instance
|
||||
if isinstance(vision_config, dict):
|
||||
self.vision_config = PixelShuffleSiglip2VisionConfig(**vision_config)
|
||||
elif vision_config is None:
|
||||
self.vision_config = PixelShuffleSiglip2VisionConfig()
|
||||
else:
|
||||
self.vision_config = vision_config
|
||||
|
||||
# Ensure compatibility with pretrained checkpoints
|
||||
self.vision_config.pixel_shuffle_scale_factor = getattr(
|
||||
self.vision_config,
|
||||
"pixel_shuffle_scale_factor",
|
||||
pixel_shuffle_scale,
|
||||
)
|
||||
self.vision_config.num_patches = getattr(
|
||||
self.vision_config,
|
||||
"num_patches",
|
||||
vision_max_num_patches,
|
||||
)
|
||||
self.vision_attn_implementation = vision_attn_implementation
|
||||
super().__init__(**kwargs)
|
||||
|
||||
|
||||
__all__ = [
|
||||
"IsaacConfig",
|
||||
"PixelShuffleSiglip2VisionConfig",
|
||||
]
|
||||
@@ -1,77 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from transformers import DeepseekV3Config
|
||||
|
||||
|
||||
class K3DSparkConfig(DeepseekV3Config):
|
||||
"""Configuration for a dense MLA DSpark draft model."""
|
||||
|
||||
model_type = "k3_dspark"
|
||||
has_no_defaults_at_init = True
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
mla_use_nope: bool = False,
|
||||
mla_use_output_gate: bool = False,
|
||||
mla_use_qk_norm: bool = False,
|
||||
rope_theta: float = 50000.0,
|
||||
**kwargs,
|
||||
) -> None:
|
||||
# DeepseekV3Config defaults to a MoE topology. Zero these fields so
|
||||
# generic vLLM config logic also recognizes this draft as dense.
|
||||
kwargs.setdefault("n_routed_experts", 0)
|
||||
kwargs.setdefault("n_shared_experts", 0)
|
||||
kwargs.setdefault("num_experts_per_tok", 0)
|
||||
|
||||
rope_parameters = kwargs.get("rope_parameters")
|
||||
if rope_parameters is None:
|
||||
kwargs["rope_parameters"] = {
|
||||
"rope_type": "default",
|
||||
"rope_theta": rope_theta,
|
||||
}
|
||||
else:
|
||||
rope_parameters = dict(rope_parameters)
|
||||
rope_parameters.setdefault("rope_type", "default")
|
||||
rope_parameters.setdefault("rope_theta", rope_theta)
|
||||
kwargs["rope_parameters"] = rope_parameters
|
||||
|
||||
super().__init__(**kwargs)
|
||||
self.mla_use_nope = mla_use_nope
|
||||
self.mla_use_output_gate = mla_use_output_gate
|
||||
self.mla_use_qk_norm = mla_use_qk_norm
|
||||
|
||||
unsupported = [
|
||||
name
|
||||
for name in (
|
||||
"mla_use_nope",
|
||||
"mla_use_output_gate",
|
||||
"mla_use_qk_norm",
|
||||
"dspark_bonus_anchor",
|
||||
)
|
||||
if getattr(self, name, False)
|
||||
]
|
||||
if self.q_lora_rank is None:
|
||||
unsupported.append("q_lora_rank=None")
|
||||
if unsupported:
|
||||
raise ValueError(
|
||||
"MLA DSpark does not support " + ", ".join(unsupported) + "."
|
||||
)
|
||||
|
||||
self.draft_vocab_size = (
|
||||
getattr(self, "draft_vocab_size", None) or self.vocab_size
|
||||
)
|
||||
if self.draft_vocab_size != self.vocab_size:
|
||||
raise ValueError(
|
||||
"MLA DSpark requires draft_vocab_size to equal vocab_size when "
|
||||
"sharing the target embedding and LM head."
|
||||
)
|
||||
|
||||
target_layer_ids = getattr(self, "target_layer_ids", None)
|
||||
if not target_layer_ids or getattr(self, "num_target_layers", None) != len(
|
||||
target_layer_ids
|
||||
):
|
||||
raise ValueError(
|
||||
"MLA DSpark requires non-empty target_layer_ids and a matching "
|
||||
"num_target_layers."
|
||||
)
|
||||
@@ -1,139 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Kimi-K3 multimodal configuration."""
|
||||
|
||||
from transformers.configuration_utils import PreTrainedConfig
|
||||
|
||||
from vllm.logger import init_logger
|
||||
from vllm.transformers_utils.configs.kimi_linear import KimiLinearConfig
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
class KimiK3VisionConfig(PreTrainedConfig):
|
||||
model_type = "kimi_k3_vision"
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
patch_size: int = 14,
|
||||
init_pos_emb_height: int = 64,
|
||||
init_pos_emb_width: int = 64,
|
||||
init_pos_emb_time: int = 4,
|
||||
pos_emb_type: str = "divided_fixed",
|
||||
vt_num_attention_heads: int = 12,
|
||||
vt_num_hidden_layers: int = 27,
|
||||
vt_hidden_size: int = 1024,
|
||||
vt_intermediate_size: int = 4096,
|
||||
merge_kernel_size: tuple[int, int] = (2, 2),
|
||||
video_attn_type: str = "spatial_temporal",
|
||||
merge_type: str = "sd2_tpool",
|
||||
_attn_implementation: str = "flash_attention_2",
|
||||
mm_projector_type: str = "patchmergerv2",
|
||||
mm_hidden_size: int | None = None,
|
||||
projector_hidden_act: str = "gelu",
|
||||
projector_ln_eps: float = 1e-5,
|
||||
qkv_hidden_size: int = 1536,
|
||||
norm_type: str = "rmsnorm",
|
||||
attn_bias: bool = False,
|
||||
patch_embed_proj_bias: bool = False,
|
||||
mlp_type: str = "mlp2",
|
||||
linear_bias: bool = False,
|
||||
activation_func: str = "gelu_pytorch_tanh",
|
||||
pos_emb_interpolation_mode: str = "bilinear",
|
||||
text_hidden_size: int = 2304,
|
||||
**kwargs,
|
||||
):
|
||||
super().__init__(**kwargs)
|
||||
|
||||
self.patch_size = patch_size
|
||||
self.init_pos_emb_height = init_pos_emb_height
|
||||
self.init_pos_emb_width = init_pos_emb_width
|
||||
self.init_pos_emb_time = init_pos_emb_time
|
||||
self.pos_emb_type = pos_emb_type
|
||||
self.vt_num_attention_heads = vt_num_attention_heads
|
||||
self.vt_num_hidden_layers = vt_num_hidden_layers
|
||||
self.vt_hidden_size = vt_hidden_size
|
||||
self.vt_intermediate_size = vt_intermediate_size
|
||||
self.merge_kernel_size = tuple(merge_kernel_size)
|
||||
self.video_attn_type = video_attn_type
|
||||
self.merge_type = merge_type
|
||||
self._attn_implementation = _attn_implementation
|
||||
|
||||
self.mm_projector_type = mm_projector_type
|
||||
self.mm_hidden_size = (
|
||||
mm_hidden_size if mm_hidden_size is not None else vt_hidden_size
|
||||
)
|
||||
self.projector_hidden_act = projector_hidden_act
|
||||
self.projector_ln_eps = projector_ln_eps
|
||||
self.text_hidden_size = text_hidden_size
|
||||
|
||||
self.qkv_hidden_size = qkv_hidden_size
|
||||
self.norm_type = norm_type
|
||||
self.attn_bias = attn_bias
|
||||
self.patch_embed_proj_bias = patch_embed_proj_bias
|
||||
self.mlp_type = mlp_type
|
||||
self.linear_bias = linear_bias
|
||||
self.activation_func = activation_func
|
||||
self.pos_emb_interpolation_mode = pos_emb_interpolation_mode
|
||||
|
||||
# Aliases consumed by the Kimi-K2.5 vision implementation.
|
||||
self.num_attention_heads = vt_num_attention_heads
|
||||
self.num_hidden_layers = vt_num_hidden_layers
|
||||
self.hidden_size = vt_hidden_size
|
||||
self.intermediate_size = vt_intermediate_size
|
||||
|
||||
|
||||
class KimiK3Config(PreTrainedConfig):
|
||||
model_type = "kimi_k3"
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
text_config: dict | KimiLinearConfig | None = None,
|
||||
vision_config: dict | KimiK3VisionConfig | None = None,
|
||||
ignore_index: int = -100,
|
||||
media_placeholder_token_id: int = 163605,
|
||||
pad_token_id: int = 0,
|
||||
image_placeholder: str = "<|kimi_image_placeholder|>",
|
||||
**kwargs,
|
||||
):
|
||||
if text_config is None:
|
||||
self.text_config = KimiLinearConfig()
|
||||
elif isinstance(text_config, dict):
|
||||
self.text_config = KimiLinearConfig(**text_config)
|
||||
else:
|
||||
self.text_config = text_config
|
||||
|
||||
if vision_config is None:
|
||||
self.vision_config = KimiK3VisionConfig()
|
||||
elif isinstance(vision_config, dict):
|
||||
self.vision_config = KimiK3VisionConfig(**vision_config)
|
||||
else:
|
||||
self.vision_config = vision_config
|
||||
|
||||
# K3's vision projector output must match the text model's hidden size.
|
||||
# Override any value provided in vision_config for safety.
|
||||
if self.vision_config.text_hidden_size != self.text_config.hidden_size:
|
||||
logger.info(
|
||||
"Overriding vision_config.text_hidden_size from %s to %s "
|
||||
"to match text_config.hidden_size",
|
||||
self.vision_config.text_hidden_size,
|
||||
self.text_config.hidden_size,
|
||||
)
|
||||
self.vision_config.text_hidden_size = self.text_config.hidden_size
|
||||
|
||||
self.ignore_index = ignore_index
|
||||
self.media_placeholder_token_id = media_placeholder_token_id
|
||||
self.image_placeholder = image_placeholder
|
||||
|
||||
if getattr(self.text_config, "quantization_config", None) is not None:
|
||||
self.quantization_config = self.text_config.quantization_config
|
||||
|
||||
super().__init__(pad_token_id=pad_token_id, **kwargs)
|
||||
|
||||
@property
|
||||
def hidden_size(self) -> int:
|
||||
return self.text_config.hidden_size
|
||||
|
||||
@property
|
||||
def vocab_size(self) -> int:
|
||||
return self.text_config.vocab_size
|
||||
@@ -1,161 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from transformers.configuration_utils import PreTrainedConfig
|
||||
|
||||
|
||||
class KimiLinearConfig(PreTrainedConfig):
|
||||
model_type = "kimi_linear"
|
||||
keys_to_ignore_at_inference = ["past_key_values"]
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
model_type="kimi_linear",
|
||||
vocab_size=163840,
|
||||
hidden_size=4096,
|
||||
head_dim=None,
|
||||
intermediate_size=11008,
|
||||
num_hidden_layers=32,
|
||||
num_attention_heads=32,
|
||||
num_key_value_heads=None,
|
||||
hidden_act="silu",
|
||||
initializer_range=0.02,
|
||||
rms_norm_eps=1e-6,
|
||||
use_cache=True,
|
||||
pad_token_id=0,
|
||||
bos_token_id=1,
|
||||
eos_token_id=2,
|
||||
rope_parameters=None,
|
||||
tie_word_embeddings=False,
|
||||
moe_intermediate_size: int | None = None,
|
||||
moe_renormalize: bool = True,
|
||||
moe_router_activation_func: str = "sigmoid",
|
||||
num_experts: int | None = None,
|
||||
num_experts_per_token: int | None = None,
|
||||
num_shared_experts: int = 0,
|
||||
routed_scaling_factor: float = 1.0,
|
||||
first_k_dense_replace: int = 0,
|
||||
moe_layer_freq: int = 1,
|
||||
use_grouped_topk: bool = True,
|
||||
num_expert_group: int = 1,
|
||||
topk_group: int = 1,
|
||||
q_lora_rank: int | None = None,
|
||||
kv_lora_rank: int | None = None,
|
||||
qk_nope_head_dim: int | None = None,
|
||||
qk_rope_head_dim: int | None = None,
|
||||
v_head_dim: int | None = None,
|
||||
mla_use_nope: bool | None = False,
|
||||
mla_use_output_gate: bool | None = False,
|
||||
num_nextn_predict_layers: int = 0,
|
||||
linear_attn_config: dict | None = None,
|
||||
attn_res_block_size: int | None = None,
|
||||
latent_moe_use_norm: bool = False,
|
||||
activation_situ_beta: float | None = None,
|
||||
activation_situ_linear_beta: float | None = None,
|
||||
max_position_embeddings: int = 4096,
|
||||
routed_expert_hidden_size: int | None = None,
|
||||
topk_method: str = "noaux_tc",
|
||||
**kwargs,
|
||||
):
|
||||
self.model_type = model_type
|
||||
self.vocab_size = vocab_size
|
||||
self.hidden_size = hidden_size
|
||||
self.head_dim = (
|
||||
head_dim if head_dim is not None else hidden_size // num_attention_heads
|
||||
)
|
||||
self.intermediate_size = intermediate_size
|
||||
self.num_hidden_layers = num_hidden_layers
|
||||
self.num_attention_heads = num_attention_heads
|
||||
|
||||
# for backward compatibility
|
||||
if num_key_value_heads is None:
|
||||
num_key_value_heads = num_attention_heads
|
||||
|
||||
self.num_key_value_heads = num_key_value_heads
|
||||
self.hidden_act = hidden_act
|
||||
self.initializer_range = initializer_range
|
||||
self.rms_norm_eps = rms_norm_eps
|
||||
self.use_cache = use_cache
|
||||
# Try to set `rope_scaling` if available, otherwise use `rope_parameters`
|
||||
rope_scaling = kwargs.pop("rope_scaling", None)
|
||||
rope_parameters = rope_scaling or rope_parameters or {"rope_type": "default"}
|
||||
rope_theta = kwargs.pop("rope_theta", 10000.0)
|
||||
if "rope_theta" not in rope_parameters:
|
||||
rope_parameters["rope_theta"] = rope_theta
|
||||
self.rope_parameters = rope_parameters
|
||||
|
||||
self.q_lora_rank = q_lora_rank
|
||||
self.kv_lora_rank = kv_lora_rank
|
||||
self.qk_nope_head_dim = qk_nope_head_dim
|
||||
self.qk_rope_head_dim = qk_rope_head_dim
|
||||
self.v_head_dim = v_head_dim
|
||||
self.mla_use_nope = mla_use_nope
|
||||
self.mla_use_output_gate = mla_use_output_gate
|
||||
# moe config
|
||||
self.num_experts = num_experts
|
||||
self.num_experts_per_token = num_experts_per_token
|
||||
self.moe_renormalize = moe_renormalize
|
||||
self.num_shared_experts = num_shared_experts
|
||||
self.routed_scaling_factor = routed_scaling_factor
|
||||
self.moe_router_activation_func = moe_router_activation_func
|
||||
assert self.moe_router_activation_func in ("softmax", "sigmoid")
|
||||
self.moe_intermediate_size = moe_intermediate_size
|
||||
self.first_k_dense_replace = first_k_dense_replace
|
||||
self.moe_layer_freq = moe_layer_freq
|
||||
self.use_grouped_topk = use_grouped_topk
|
||||
self.num_expert_group = num_expert_group
|
||||
self.topk_group = topk_group
|
||||
self.num_nextn_predict_layers = num_nextn_predict_layers
|
||||
|
||||
self.attn_res_block_size = attn_res_block_size
|
||||
self.latent_moe_use_norm = latent_moe_use_norm
|
||||
self.activation_situ_beta = activation_situ_beta
|
||||
self.activation_situ_linear_beta = activation_situ_linear_beta
|
||||
self.max_position_embeddings = max_position_embeddings
|
||||
self.routed_expert_hidden_size = routed_expert_hidden_size
|
||||
self.topk_method = topk_method
|
||||
|
||||
if linear_attn_config is not None:
|
||||
assert linear_attn_config["kda_layers"] is not None
|
||||
assert linear_attn_config["full_attn_layers"] is not None
|
||||
self.linear_attn_config = linear_attn_config
|
||||
|
||||
super().__init__(
|
||||
pad_token_id=pad_token_id,
|
||||
bos_token_id=bos_token_id,
|
||||
eos_token_id=eos_token_id,
|
||||
tie_word_embeddings=tie_word_embeddings,
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
@property
|
||||
def is_mla(self):
|
||||
return (
|
||||
self.q_lora_rank is not None
|
||||
or self.kv_lora_rank is not None
|
||||
or self.qk_nope_head_dim is not None
|
||||
or self.qk_rope_head_dim is not None
|
||||
or self.v_head_dim is not None
|
||||
or self.mla_use_nope is True
|
||||
)
|
||||
|
||||
@property
|
||||
def is_moe(self):
|
||||
return self.num_experts is not None
|
||||
|
||||
@property
|
||||
def is_linear_attn(self) -> bool:
|
||||
return not (
|
||||
self.linear_attn_config is None
|
||||
or (
|
||||
isinstance(self.linear_attn_config, dict)
|
||||
and self.linear_attn_config["kda_layers"] is not None
|
||||
and len(self.linear_attn_config["kda_layers"]) == 0
|
||||
)
|
||||
)
|
||||
|
||||
def is_kda_layer(self, layer_idx: int):
|
||||
return (
|
||||
self.linear_attn_config is not None
|
||||
and (layer_idx + 1) in self.linear_attn_config["kda_layers"]
|
||||
)
|
||||
@@ -1,38 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
# Adapted from https://huggingface.co/moonshotai/Kimi-VL-A3B-Instruct/blob/main/configuration_kimi_vl.py
|
||||
|
||||
from transformers import DeepseekV2Config
|
||||
from transformers.configuration_utils import PreTrainedConfig
|
||||
|
||||
from vllm.transformers_utils.configs.moonvit import MoonViTConfig
|
||||
|
||||
|
||||
class KimiVLConfig(PreTrainedConfig):
|
||||
model_type = "kimi_vl"
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
vision_config: dict | MoonViTConfig | None = None,
|
||||
text_config: dict | DeepseekV2Config | None = None,
|
||||
ignore_index: int = -100,
|
||||
media_placeholder_token_id: int = 163605,
|
||||
pad_token_id: int = 0,
|
||||
**kwargs,
|
||||
):
|
||||
if vision_config is None:
|
||||
vision_config = MoonViTConfig()
|
||||
elif isinstance(vision_config, dict):
|
||||
vision_config = MoonViTConfig(**vision_config)
|
||||
self.vision_config = vision_config
|
||||
|
||||
if text_config is None:
|
||||
text_config = DeepseekV2Config()
|
||||
elif isinstance(text_config, dict):
|
||||
text_config = DeepseekV2Config(**text_config)
|
||||
self.text_config: DeepseekV2Config = text_config
|
||||
|
||||
self.ignore_index = ignore_index
|
||||
self.media_placeholder_token_id = media_placeholder_token_id
|
||||
|
||||
super().__init__(pad_token_id=pad_token_id, **kwargs)
|
||||
@@ -1,103 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
# Copyright 2025 Horizon team, Xiaomi MiLM Plus.
|
||||
# Copyright 2024 The Qwen team.
|
||||
# Copyright 2023 The vLLM team.
|
||||
# Copyright 2022 EleutherAI and the HuggingFace Inc. team. All rights reserved.
|
||||
#
|
||||
# This code is based on EleutherAI's GPT-NeoX library and the GPT-NeoX
|
||||
# and OPT implementations in this library. It has been modified from its
|
||||
# original forms to accommodate minor architectural differences compared
|
||||
# to GPT-NeoX and OPT used by the Meta AI team that trained the model.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
from transformers import PreTrainedConfig
|
||||
from transformers.models.qwen2_5_omni.configuration_qwen2_5_omni import (
|
||||
Qwen2_5OmniTextConfig,
|
||||
)
|
||||
|
||||
|
||||
class DashengConfig(PreTrainedConfig):
|
||||
model_type = "midashenglm_dasheng_encoder"
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
embed_dim: int = 768,
|
||||
outputdim: int = 527,
|
||||
patch_size: int | tuple[int, int] = 16,
|
||||
patch_stride: int | tuple[int, int] = 16,
|
||||
input_channels: int = 1,
|
||||
target_length: int = 1012,
|
||||
depth: int = 12,
|
||||
num_heads: int = 12,
|
||||
mlp_ratio: float = 4.0,
|
||||
qkv_bias: bool = True,
|
||||
init_values: float | None = None,
|
||||
drop_rate: float = 0.0,
|
||||
attn_drop_rate: float = 0.0,
|
||||
f_min: float = 0.0,
|
||||
f_max: float = 8000.0,
|
||||
center: bool = True,
|
||||
win_length: int = 512,
|
||||
hop_length: int = 160,
|
||||
sample_rate: int = 16000,
|
||||
n_fft: int = 512,
|
||||
n_mels: int = 64,
|
||||
**kwargs,
|
||||
):
|
||||
self.embed_dim = embed_dim
|
||||
self.outputdim = outputdim
|
||||
self.patch_size = patch_size
|
||||
self.patch_stride = patch_stride
|
||||
self.input_channels = input_channels
|
||||
self.target_length = target_length
|
||||
self.depth = depth
|
||||
self.num_heads = num_heads
|
||||
self.mlp_ratio = mlp_ratio
|
||||
self.qkv_bias = qkv_bias
|
||||
self.init_values = init_values
|
||||
self.drop_rate = drop_rate
|
||||
self.attn_drop_rate = attn_drop_rate
|
||||
self.f_min = f_min
|
||||
self.f_max = f_max
|
||||
self.center = center
|
||||
self.win_length = win_length
|
||||
self.hop_length = hop_length
|
||||
self.sample_rate = sample_rate
|
||||
self.n_fft = n_fft
|
||||
self.n_mels = n_mels
|
||||
super().__init__(**kwargs)
|
||||
|
||||
|
||||
class MiDashengLMConfig(PreTrainedConfig):
|
||||
model_type = "midashenglm"
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
audio_encoder_config: dict | None = None,
|
||||
subsample_factor: int = 5,
|
||||
text_config: dict | None = None,
|
||||
audio_token_id: int | None = None,
|
||||
**kwargs,
|
||||
):
|
||||
self.audio_encoder_config = DashengConfig(**(audio_encoder_config or {}))
|
||||
self.subsample_factor = subsample_factor
|
||||
self.text_config = (
|
||||
Qwen2_5OmniTextConfig(**text_config)
|
||||
if text_config
|
||||
else Qwen2_5OmniTextConfig()
|
||||
)
|
||||
self.text_config.rope_parameters = None # uses_mrope is false
|
||||
self.audio_token_id = audio_token_id
|
||||
super().__init__(**kwargs)
|
||||
@@ -1,149 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from typing import Any
|
||||
|
||||
from transformers import PreTrainedConfig
|
||||
|
||||
|
||||
class MiniMaxM3TextConfig(PreTrainedConfig):
|
||||
"""Config for the MiniMax M3 text backbone (MiniMaxM3SparseForCausalLM).
|
||||
|
||||
Defaults mirror the ``text_config`` of the MiniMax-M3-preview checkpoint.
|
||||
"""
|
||||
|
||||
model_type = "minimax_m3_text"
|
||||
architectures = ["MiniMaxM3SparseForCausalLM"]
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
vocab_size: int = 200064,
|
||||
hidden_size: int = 6144,
|
||||
intermediate_size: int = 3072,
|
||||
dense_intermediate_size: int = 12288,
|
||||
shared_intermediate_size: int = 3072,
|
||||
num_hidden_layers: int = 60,
|
||||
num_attention_heads: int = 64,
|
||||
num_key_value_heads: int = 4,
|
||||
head_dim: int = 128,
|
||||
max_position_embeddings: int = 524288,
|
||||
rms_norm_eps: float = 1e-6,
|
||||
use_gemma_norm: bool = True,
|
||||
attention_output_gate: bool = False,
|
||||
rope_theta: float = 5000000,
|
||||
rotary_dim: int = 64,
|
||||
partial_rotary_factor: float = 0.5,
|
||||
hidden_act: str = "swigluoai",
|
||||
swiglu_alpha: float = 1.702,
|
||||
# SwiGLU-OAI uses the (up + 1) bias, i.e. beta=1.0 (matches the
|
||||
# reference: gate * sigmoid(gate * alpha) * (up + 1)). The checkpoint
|
||||
# config omits swiglu_beta, so this default must stay 1.0.
|
||||
swiglu_beta: float = 1.0,
|
||||
swiglu_limit: float = 7.0,
|
||||
use_qk_norm: bool = True,
|
||||
qk_norm_type: str = "per_head",
|
||||
num_local_experts: int = 128,
|
||||
num_experts_per_tok: int = 4,
|
||||
n_shared_experts: int = 1,
|
||||
scoring_func: str = "sigmoid",
|
||||
use_routing_bias: bool = True,
|
||||
routed_scaling_factor: float = 2.0,
|
||||
num_mtp_modules: int = 1,
|
||||
moe_layer_freq: list[int] | None = None,
|
||||
sparse_attention_config: dict[str, Any] | None = None,
|
||||
tie_word_embeddings: bool = False,
|
||||
**kwargs,
|
||||
):
|
||||
self.vocab_size = vocab_size
|
||||
self.hidden_size = hidden_size
|
||||
self.intermediate_size = intermediate_size
|
||||
self.dense_intermediate_size = dense_intermediate_size
|
||||
self.shared_intermediate_size = shared_intermediate_size
|
||||
self.num_hidden_layers = num_hidden_layers
|
||||
self.num_attention_heads = num_attention_heads
|
||||
self.num_key_value_heads = num_key_value_heads
|
||||
self.head_dim = head_dim
|
||||
self.max_position_embeddings = max_position_embeddings
|
||||
self.rms_norm_eps = rms_norm_eps
|
||||
self.use_gemma_norm = use_gemma_norm
|
||||
self.attention_output_gate = attention_output_gate
|
||||
self.rope_theta = rope_theta
|
||||
self.rotary_dim = rotary_dim
|
||||
self.partial_rotary_factor = partial_rotary_factor
|
||||
self.hidden_act = hidden_act
|
||||
self.swiglu_alpha = swiglu_alpha
|
||||
self.swiglu_beta = swiglu_beta
|
||||
self.swiglu_limit = swiglu_limit
|
||||
self.use_qk_norm = use_qk_norm
|
||||
self.qk_norm_type = qk_norm_type
|
||||
self.num_local_experts = num_local_experts
|
||||
self.num_experts_per_tok = num_experts_per_tok
|
||||
self.n_shared_experts = n_shared_experts
|
||||
self.scoring_func = scoring_func
|
||||
self.use_routing_bias = use_routing_bias
|
||||
self.routed_scaling_factor = routed_scaling_factor
|
||||
self.num_mtp_modules = num_mtp_modules
|
||||
# First 3 layers are dense; the remaining 57 are sparse MoE.
|
||||
self.moe_layer_freq = (
|
||||
moe_layer_freq if moe_layer_freq is not None else [0] * 3 + [1] * 57
|
||||
)
|
||||
self.sparse_attention_config = (
|
||||
sparse_attention_config
|
||||
if sparse_attention_config is not None
|
||||
else {
|
||||
"use_sparse_attention": True,
|
||||
"sparse_index_dim": 128,
|
||||
"sparse_num_index_heads": 4,
|
||||
"sparse_topk_blocks": 16,
|
||||
"sparse_block_size": 128,
|
||||
"sparse_disable_index_value": [0] * 3 + [1] * 57,
|
||||
"sparse_score_type": "max",
|
||||
"sparse_init_block": 0,
|
||||
"sparse_local_block": 1,
|
||||
"sparse_attention_freq": [0] * 3 + [1] * 57,
|
||||
}
|
||||
)
|
||||
super().__init__(tie_word_embeddings=tie_word_embeddings, **kwargs)
|
||||
|
||||
|
||||
class MiniMaxM3MTPConfig(MiniMaxM3TextConfig):
|
||||
"""Config for a standalone MiniMax M3 MTP (multi-token prediction) head.
|
||||
|
||||
The MTP transformer layer is structurally a single MiniMax M3 decoder
|
||||
layer, so this reuses the text backbone schema. Standalone MTP checkpoints
|
||||
use ``model_type='minimax_m3_mtp'`` and a single hidden layer.
|
||||
"""
|
||||
|
||||
model_type = "minimax_m3_mtp"
|
||||
architectures = ["MiniMaxM3MTP"]
|
||||
|
||||
def __init__(self, num_hidden_layers: int = 1, **kwargs):
|
||||
super().__init__(num_hidden_layers=num_hidden_layers, **kwargs)
|
||||
|
||||
|
||||
class MiniMaxM3Config(PreTrainedConfig):
|
||||
"""Top-level MiniMax M3 (VL) config.
|
||||
|
||||
Holds the text backbone as ``text_config`` so that
|
||||
``config.get_text_config()`` extracts the MiniMaxM3SparseForCausalLM
|
||||
backbone. Vision components are kept as a raw dict passthrough and are
|
||||
not modeled here.
|
||||
"""
|
||||
|
||||
model_type = "minimax_m3_vl"
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
text_config: dict | MiniMaxM3TextConfig | None = None,
|
||||
vision_config: dict | None = None,
|
||||
**kwargs,
|
||||
):
|
||||
if text_config is None:
|
||||
text_config = MiniMaxM3TextConfig()
|
||||
elif isinstance(text_config, dict):
|
||||
text_config = MiniMaxM3TextConfig(**text_config)
|
||||
self.text_config = text_config
|
||||
self.vision_config = vision_config
|
||||
|
||||
self.hidden_size = text_config.hidden_size
|
||||
|
||||
super().__init__(**kwargs)
|
||||
@@ -1,153 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Configuration for Moondream3 model."""
|
||||
|
||||
from transformers import PreTrainedConfig
|
||||
|
||||
|
||||
class Moondream3VisionConfig(PreTrainedConfig):
|
||||
"""Vision encoder configuration for Moondream3."""
|
||||
|
||||
model_type = "moondream3_vision"
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
enc_dim: int = 1152,
|
||||
enc_patch_size: int = 14,
|
||||
enc_n_layers: int = 27,
|
||||
enc_ff_dim: int = 4304,
|
||||
enc_n_heads: int = 16,
|
||||
proj_inner_dim: int = 8192,
|
||||
crop_size: int = 378,
|
||||
max_crops: int = 12,
|
||||
overlap_margin: int = 4,
|
||||
**kwargs,
|
||||
):
|
||||
super().__init__(**kwargs)
|
||||
self.enc_dim = enc_dim
|
||||
self.enc_patch_size = enc_patch_size
|
||||
self.enc_n_layers = enc_n_layers
|
||||
self.enc_ff_dim = enc_ff_dim
|
||||
self.enc_n_heads = enc_n_heads
|
||||
self.proj_inner_dim = proj_inner_dim
|
||||
self.crop_size = crop_size
|
||||
self.max_crops = max_crops
|
||||
self.overlap_margin = overlap_margin
|
||||
|
||||
# Standard HuggingFace attributes for vision config
|
||||
self.hidden_size = enc_dim
|
||||
self.num_attention_heads = enc_n_heads
|
||||
self.num_hidden_layers = enc_n_layers
|
||||
self.intermediate_size = enc_ff_dim
|
||||
self.patch_size = enc_patch_size
|
||||
self.image_size = crop_size
|
||||
|
||||
|
||||
class Moondream3TextConfig(PreTrainedConfig):
|
||||
"""Text decoder configuration for Moondream3."""
|
||||
|
||||
model_type = "moondream3_text"
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
dim: int = 2048,
|
||||
ff_dim: int = 8192,
|
||||
n_layers: int = 24,
|
||||
vocab_size: int = 51200,
|
||||
max_context: int = 4096,
|
||||
n_heads: int = 32,
|
||||
n_kv_heads: int = 32,
|
||||
prefix_attn: int = 730,
|
||||
rope_theta: float = 1500000.0,
|
||||
moe: dict | None = None,
|
||||
**kwargs,
|
||||
):
|
||||
super().__init__(**kwargs)
|
||||
|
||||
# Store original moondream3 config names
|
||||
self.dim = dim
|
||||
self.ff_dim = ff_dim
|
||||
self.n_layers = n_layers
|
||||
self.n_heads = n_heads
|
||||
self.n_kv_heads = n_kv_heads
|
||||
self.prefix_attn = prefix_attn
|
||||
self.max_context = max_context
|
||||
self.rope_theta = rope_theta
|
||||
|
||||
# MoE config
|
||||
moe = moe or {}
|
||||
self.moe_start_layer = moe.get("start_layer", 4)
|
||||
self.moe_num_experts = moe.get("n_experts", 64)
|
||||
self.moe_experts_per_token = moe.get("n_experts_per_tok", 8)
|
||||
self.moe_expert_inner_dim = moe.get("expert_inner_dim", 1024)
|
||||
|
||||
# Standard HuggingFace attributes (required by vLLM)
|
||||
self.hidden_size = dim
|
||||
self.num_attention_heads = n_heads
|
||||
self.num_key_value_heads = n_kv_heads
|
||||
self.num_hidden_layers = n_layers
|
||||
self.intermediate_size = ff_dim
|
||||
self.vocab_size = vocab_size
|
||||
self.max_position_embeddings = max_context
|
||||
|
||||
# Moondream3 uses token 0 (<|endoftext|>) as both BOS and EOS.
|
||||
# Token 3 (<|md_reserved_2|>) is an answer delimiter that the model
|
||||
# implementation suppresses during generation.
|
||||
self.bos_token_id = 0
|
||||
self.eos_token_id = 0
|
||||
|
||||
# MoE standard attributes
|
||||
self.num_local_experts = self.moe_num_experts
|
||||
self.num_experts_per_tok = self.moe_experts_per_token
|
||||
|
||||
|
||||
class Moondream3Config(PreTrainedConfig):
|
||||
"""Combined configuration for Moondream3 multimodal model."""
|
||||
|
||||
model_type = "moondream3"
|
||||
is_composition = True
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
config: dict | None = None,
|
||||
**kwargs,
|
||||
):
|
||||
config = config or {}
|
||||
|
||||
# Parse text config
|
||||
text_config = config.get("text", {})
|
||||
self.text_config: Moondream3TextConfig = Moondream3TextConfig(**text_config)
|
||||
|
||||
# Parse vision config
|
||||
vision_config = config.get("vision", {})
|
||||
self.vision_config = Moondream3VisionConfig(**vision_config)
|
||||
|
||||
# Store the original config dict for model access
|
||||
self.config = config
|
||||
tokenizer_config = config.get("tokenizer", {})
|
||||
self.answer_token_id = tokenizer_config.get("answer_id", 3)
|
||||
|
||||
super().__init__(**kwargs)
|
||||
|
||||
# Expose key attributes at top level for vLLM compatibility
|
||||
self.hidden_size = self.text_config.hidden_size
|
||||
self.num_attention_heads = self.text_config.num_attention_heads
|
||||
self.num_key_value_heads = self.text_config.num_key_value_heads
|
||||
self.num_hidden_layers = self.text_config.num_hidden_layers
|
||||
self.vocab_size = self.text_config.vocab_size
|
||||
self.intermediate_size = self.text_config.intermediate_size
|
||||
|
||||
# Moondream3 uses token 0 (<|endoftext|>) as both BOS and EOS.
|
||||
# Token 3 (<|md_reserved_2|>) is an answer delimiter that the model
|
||||
# implementation suppresses during generation.
|
||||
self.bos_token_id = 0
|
||||
self.eos_token_id = 0
|
||||
|
||||
def get_text_config(self, decoder: bool = False) -> "Moondream3TextConfig":
|
||||
"""Return the text config for vLLM's text_config detection.
|
||||
|
||||
Args:
|
||||
decoder: Ignored. Only used for encoder-decoder models.
|
||||
|
||||
"""
|
||||
return self.text_config
|
||||
@@ -1,33 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
# Adapted from https://huggingface.co/moonshotai/Kimi-VL-A3B-Instruct/blob/main/configuration_kimi_vl.py
|
||||
from transformers.configuration_utils import PreTrainedConfig
|
||||
|
||||
|
||||
class MoonViTConfig(PreTrainedConfig):
|
||||
model_type = "moonvit"
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
patch_size: int = 14,
|
||||
init_pos_emb_height: int = 64,
|
||||
init_pos_emb_width: int = 64,
|
||||
num_attention_heads: int = 16,
|
||||
num_hidden_layers: int = 27,
|
||||
hidden_size: int = 1152,
|
||||
intermediate_size: int = 4304,
|
||||
merge_kernel_size: tuple[int, int] = (2, 2),
|
||||
**kwargs,
|
||||
):
|
||||
super().__init__(**kwargs)
|
||||
self.patch_size = patch_size
|
||||
# Positional embedding config
|
||||
self.init_pos_emb_height = init_pos_emb_height
|
||||
self.init_pos_emb_width = init_pos_emb_width
|
||||
# Transformer config
|
||||
self.num_hidden_layers = num_hidden_layers
|
||||
self.num_attention_heads = num_attention_heads
|
||||
self.hidden_size = hidden_size
|
||||
self.intermediate_size = intermediate_size
|
||||
# Patch merger config
|
||||
self.merge_kernel_size = merge_kernel_size
|
||||
@@ -1,108 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from typing import Any
|
||||
|
||||
from transformers import PreTrainedConfig, Qwen3Config
|
||||
from transformers.models.whisper.configuration_whisper import WhisperConfig
|
||||
|
||||
|
||||
class MossTranscribeDiarizeConfig(PreTrainedConfig):
|
||||
"""Configuration for MOSS-Transcribe-Diarize."""
|
||||
|
||||
model_type = "moss_transcribe_diarize"
|
||||
sub_configs = {"text_config": Qwen3Config, "audio_config": WhisperConfig}
|
||||
keys_to_ignore_at_inference = ["past_key_values"]
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
text_config: dict[str, Any] | Qwen3Config | None = None,
|
||||
audio_config: dict[str, Any] | WhisperConfig | None = None,
|
||||
audio_token_id: int = 151671,
|
||||
audio_merge_size: int = 4,
|
||||
adaptor_input_dim: int | None = None,
|
||||
tie_word_embeddings: bool = True,
|
||||
**kwargs: Any,
|
||||
) -> None:
|
||||
text_config_obj: Qwen3Config
|
||||
if text_config is None:
|
||||
text_config_obj = Qwen3Config(
|
||||
vocab_size=151936,
|
||||
hidden_size=1024,
|
||||
intermediate_size=3072,
|
||||
num_hidden_layers=28,
|
||||
num_attention_heads=16,
|
||||
num_key_value_heads=8,
|
||||
head_dim=128,
|
||||
max_position_embeddings=40960,
|
||||
tie_word_embeddings=tie_word_embeddings,
|
||||
rope_theta=1_000_000.0,
|
||||
layer_types=["full_attention"] * 28,
|
||||
)
|
||||
elif isinstance(text_config, dict):
|
||||
text_config_obj = Qwen3Config(**text_config)
|
||||
else:
|
||||
text_config_obj = text_config
|
||||
|
||||
audio_config_obj: WhisperConfig
|
||||
if audio_config is None:
|
||||
audio_config_obj = WhisperConfig(
|
||||
num_mel_bins=80,
|
||||
d_model=1024,
|
||||
encoder_layers=24,
|
||||
encoder_attention_heads=16,
|
||||
encoder_ffn_dim=4096,
|
||||
max_source_positions=1500,
|
||||
dropout=0.0,
|
||||
attention_dropout=0.0,
|
||||
activation_dropout=0.0,
|
||||
activation_function="gelu",
|
||||
encoder_layerdrop=0.0,
|
||||
scale_embedding=False,
|
||||
)
|
||||
elif isinstance(audio_config, dict):
|
||||
audio_config_obj = WhisperConfig(**audio_config)
|
||||
else:
|
||||
audio_config_obj = audio_config
|
||||
|
||||
text_config_obj.tie_word_embeddings = tie_word_embeddings
|
||||
if not getattr(text_config_obj, "layer_types", None):
|
||||
text_config_obj.layer_types = [
|
||||
"full_attention"
|
||||
] * text_config_obj.num_hidden_layers
|
||||
|
||||
super().__init__(tie_word_embeddings=tie_word_embeddings, **kwargs)
|
||||
|
||||
self.text_config = text_config_obj
|
||||
self.audio_config = audio_config_obj
|
||||
self.audio_token_id = int(audio_token_id)
|
||||
self.audio_merge_size = int(audio_merge_size)
|
||||
self.adaptor_input_dim = (
|
||||
int(adaptor_input_dim)
|
||||
if adaptor_input_dim is not None
|
||||
else int(audio_config_obj.d_model) * int(audio_merge_size)
|
||||
)
|
||||
|
||||
self.vocab_size = int(text_config_obj.vocab_size)
|
||||
self.hidden_size = int(text_config_obj.hidden_size)
|
||||
self.intermediate_size = int(text_config_obj.intermediate_size)
|
||||
self.num_hidden_layers = int(text_config_obj.num_hidden_layers)
|
||||
self.num_attention_heads = int(text_config_obj.num_attention_heads)
|
||||
self.num_key_value_heads = int(text_config_obj.num_key_value_heads)
|
||||
self.head_dim = int(text_config_obj.head_dim)
|
||||
self.hidden_act = text_config_obj.hidden_act
|
||||
self.max_position_embeddings = int(text_config_obj.max_position_embeddings)
|
||||
self.rms_norm_eps = float(text_config_obj.rms_norm_eps)
|
||||
rope_parameters = getattr(text_config_obj, "rope_parameters", None)
|
||||
rope_theta = float(getattr(text_config_obj, "rope_theta", 1_000_000.0))
|
||||
if rope_parameters is None:
|
||||
rope_parameters = {
|
||||
"rope_type": "default",
|
||||
"rope_theta": rope_theta,
|
||||
}
|
||||
text_config_obj.rope_parameters = rope_parameters
|
||||
self.rope_parameters = rope_parameters
|
||||
self.rope_theta = float(rope_parameters.get("rope_theta", rope_theta))
|
||||
self.attention_bias = bool(text_config_obj.attention_bias)
|
||||
self.attention_dropout = float(text_config_obj.attention_dropout)
|
||||
self.is_causal = True
|
||||
@@ -1,389 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""MuseGlimmer model configuration for vLLM.
|
||||
|
||||
Native vLLM copy of the MuseGlimmer HuggingFace configs
|
||||
(``configuration_muse_glimmer.py``). MuseGlimmer's
|
||||
``model_type`` is not yet registered in released transformers, so vLLM ships
|
||||
this config so partners can serve MuseGlimmer checkpoints *without* trust_remote_code.
|
||||
|
||||
The field set and defaults are kept byte-for-byte in sync with the HF reference
|
||||
so a checkpoint's ``config.json`` deserializes identically here. Both the text
|
||||
and vision configs are consumed by the native multimodal serving path.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from transformers import Qwen3Config
|
||||
from transformers.configuration_utils import PreTrainedConfig
|
||||
|
||||
|
||||
def _default_no_rope_layers(num_hidden_layers: int) -> list[int]:
|
||||
# iRoPE mask: NoPE every 4 layers, counted backward from the last layer.
|
||||
stride = 4
|
||||
return [
|
||||
0 if (num_hidden_layers - 1 - i) % stride == 0 else 1
|
||||
for i in range(num_hidden_layers)
|
||||
]
|
||||
|
||||
|
||||
class MuseGlimmerTextConfig(PreTrainedConfig):
|
||||
model_type = "muse_glimmer_text"
|
||||
keys_to_ignore_at_inference = ["past_key_values"]
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
vocab_size: int = 202_048,
|
||||
hidden_size: int = 6656,
|
||||
intermediate_size: int = 19968,
|
||||
num_hidden_layers: int = 52,
|
||||
num_attention_heads: int = 32,
|
||||
num_key_value_heads: int = 2,
|
||||
head_dim: int = 128,
|
||||
hidden_activation: str = "silu",
|
||||
max_position_embeddings: int = 16_384,
|
||||
initializer_range: float = 0.02,
|
||||
rms_norm_eps: float = 1e-5,
|
||||
use_cache: bool = True,
|
||||
pad_token_id: int | None = None,
|
||||
eos_token_id: int | list[int] | None = 200_001,
|
||||
bos_token_id: int | None = 200_000,
|
||||
tie_word_embeddings: bool = False,
|
||||
rope_parameters: dict | None = None,
|
||||
rope_theta: float | None = None,
|
||||
attention_bias: bool = False,
|
||||
attention_dropout: float = 0.0,
|
||||
query_pre_attn_scalar: int = 256,
|
||||
sliding_window: int | None = 2048,
|
||||
layer_types: list[str] | None = None,
|
||||
final_logit_softcapping: float | None = 20.0,
|
||||
attn_logit_softcapping: float | None = None,
|
||||
use_bidirectional_attention: bool | None = None,
|
||||
# MuseGlimmer-specific
|
||||
qk_scale_factor: float = 43.7840518911,
|
||||
use_qk_norm: bool = True,
|
||||
use_attn_output_gate: bool = True,
|
||||
output_multiplier: float = 0.19611613513818404,
|
||||
normalize_tok_embeddings: bool = True,
|
||||
post_norm_eps: float = 1e-8,
|
||||
no_rope_layers: list[int] | None = None,
|
||||
**kwargs,
|
||||
) -> None:
|
||||
self.vocab_size = vocab_size
|
||||
self.hidden_size = hidden_size
|
||||
self.intermediate_size = intermediate_size
|
||||
self.num_hidden_layers = num_hidden_layers
|
||||
self.num_attention_heads = num_attention_heads
|
||||
self.num_key_value_heads = num_key_value_heads
|
||||
self.head_dim = head_dim
|
||||
self.hidden_activation = hidden_activation
|
||||
self.max_position_embeddings = max_position_embeddings
|
||||
self.initializer_range = initializer_range
|
||||
self.rms_norm_eps = rms_norm_eps
|
||||
self.use_cache = use_cache
|
||||
self.attention_bias = attention_bias
|
||||
self.attention_dropout = attention_dropout
|
||||
self.query_pre_attn_scalar = query_pre_attn_scalar
|
||||
self.sliding_window = sliding_window
|
||||
self.final_logit_softcapping = final_logit_softcapping
|
||||
self.attn_logit_softcapping = attn_logit_softcapping
|
||||
self.use_bidirectional_attention = use_bidirectional_attention
|
||||
|
||||
# RoPE: accept either an explicit rope_parameters dict (HF 5.x) or a
|
||||
# bare rope_theta; normalize to rope_parameters for vLLM's get_rope.
|
||||
if rope_parameters is None:
|
||||
theta = rope_theta if rope_theta is not None else 500_000.0
|
||||
rope_parameters = {"rope_type": "default", "rope_theta": theta}
|
||||
self.rope_parameters = rope_parameters
|
||||
# vLLM reads rope_theta off the config in some codepaths.
|
||||
self.rope_theta = rope_parameters.get("rope_theta", 500_000.0)
|
||||
|
||||
# MuseGlimmer-specific fields
|
||||
self.qk_scale_factor = qk_scale_factor
|
||||
self.use_qk_norm = use_qk_norm
|
||||
self.use_attn_output_gate = use_attn_output_gate
|
||||
self.output_multiplier = output_multiplier
|
||||
self.normalize_tok_embeddings = normalize_tok_embeddings
|
||||
self.post_norm_eps = post_norm_eps
|
||||
|
||||
self.no_rope_layers = (
|
||||
no_rope_layers
|
||||
if no_rope_layers is not None
|
||||
else _default_no_rope_layers(num_hidden_layers)
|
||||
)
|
||||
if layer_types is None:
|
||||
layer_types = [
|
||||
"full_attention" if self.no_rope_layers[i] == 0 else "sliding_attention"
|
||||
for i in range(num_hidden_layers)
|
||||
]
|
||||
self.layer_types = layer_types
|
||||
|
||||
super().__init__(
|
||||
pad_token_id=pad_token_id,
|
||||
bos_token_id=bos_token_id,
|
||||
eos_token_id=eos_token_id,
|
||||
tie_word_embeddings=tie_word_embeddings,
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
|
||||
class MuseGlimmerVisionConfig(PreTrainedConfig):
|
||||
model_type = "muse_glimmer_vision"
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
patch_size: int = 14,
|
||||
pos_emb_height: int = 32,
|
||||
pos_emb_width: int = 32,
|
||||
num_attention_heads: int = 16,
|
||||
num_hidden_layers: int = 50,
|
||||
hidden_size: int = 1536,
|
||||
intermediate_size: int = 8960,
|
||||
hidden_act: str = "gelu",
|
||||
merge_kernel_size: int = 2,
|
||||
rope_parameters: dict | None = None,
|
||||
max_position_embeddings: int = 32 * 32,
|
||||
output_dim: int = 6144,
|
||||
patch_temporal: int = 2,
|
||||
adapter_dim: int = 4096,
|
||||
layer_norm_eps: float = 1e-5,
|
||||
layer_types: list[str] | None = None,
|
||||
**kwargs,
|
||||
) -> None:
|
||||
super().__init__(**kwargs)
|
||||
self.patch_size = patch_size
|
||||
self.pos_emb_height = pos_emb_height
|
||||
self.pos_emb_width = pos_emb_width
|
||||
self.num_attention_heads = num_attention_heads
|
||||
self.num_hidden_layers = num_hidden_layers
|
||||
self.hidden_size = hidden_size
|
||||
self.intermediate_size = intermediate_size
|
||||
self.hidden_act = hidden_act
|
||||
self.merge_kernel_size = merge_kernel_size
|
||||
self.rope_parameters = rope_parameters
|
||||
self.max_position_embeddings = max_position_embeddings
|
||||
self.output_dim = output_dim
|
||||
self.patch_temporal = patch_temporal
|
||||
self.adapter_dim = adapter_dim
|
||||
self.layer_norm_eps = layer_norm_eps
|
||||
if layer_types is None:
|
||||
stride = 4
|
||||
layer_types = [
|
||||
"full_attention"
|
||||
if (i + 1) % stride == 0 or i == num_hidden_layers - 1
|
||||
else "sliding_attention"
|
||||
for i in range(num_hidden_layers)
|
||||
]
|
||||
self.layer_types = layer_types
|
||||
|
||||
|
||||
class MuseGlimmerConfig(PreTrainedConfig):
|
||||
model_type = "muse_glimmer"
|
||||
sub_configs = {
|
||||
"text_config": MuseGlimmerTextConfig,
|
||||
"vision_config": MuseGlimmerVisionConfig,
|
||||
}
|
||||
|
||||
# --- Flat (legacy-converter) -> canonical normalization -------------------
|
||||
# MuseGlimmer checkpoints exist in two config layouts in the wild:
|
||||
#
|
||||
# * CANONICAL (current HF converter, transformers 5.15): nested
|
||||
# ``text_config`` / ``vision_config`` sub-dicts with canonical field
|
||||
# names (``hidden_activation``, ``final_logit_softcapping``,
|
||||
# ``rope_parameters``, ``vision_config.hidden_size`` ...).
|
||||
#
|
||||
# * FLAT (older converter, e.g. Ruan's ``rl_v1/hf``, transformers 5.9):
|
||||
# every field is a top-level key, with different names
|
||||
# (``hidden_act``, ``output_soft_cap_temp``, ``rope_theta``,
|
||||
# ``vision_latent_dim`` ...) and NO ``text_config`` nesting.
|
||||
#
|
||||
# Without normalization a flat config silently deserializes to an
|
||||
# ALL-DEFAULT text config (every checkpoint value ignored) — a dangerous
|
||||
# correctness bug: a checkpoint whose arch differs from the defaults would
|
||||
# load into a wrong-shaped model with no error. So when we detect a flat
|
||||
# config we hoist the flat fields into ``text_config`` / ``vision_config``
|
||||
# and rename them to canonical names.
|
||||
|
||||
# flat text field name -> canonical MuseGlimmerTextConfig field name
|
||||
_FLAT_TEXT_RENAMES = {
|
||||
"hidden_act": "hidden_activation",
|
||||
"output_soft_cap_temp": "final_logit_softcapping",
|
||||
}
|
||||
# canonical MuseGlimmerTextConfig constructor params that may appear flat
|
||||
_FLAT_TEXT_KEYS = frozenset(
|
||||
{
|
||||
"vocab_size",
|
||||
"hidden_size",
|
||||
"intermediate_size",
|
||||
"num_hidden_layers",
|
||||
"num_attention_heads",
|
||||
"num_key_value_heads",
|
||||
"head_dim",
|
||||
"hidden_activation",
|
||||
"hidden_act",
|
||||
"max_position_embeddings",
|
||||
"initializer_range",
|
||||
"rms_norm_eps",
|
||||
"use_cache",
|
||||
"tie_word_embeddings",
|
||||
"rope_parameters",
|
||||
"rope_theta",
|
||||
"attention_bias",
|
||||
"attention_dropout",
|
||||
"query_pre_attn_scalar",
|
||||
"sliding_window",
|
||||
"layer_types",
|
||||
"final_logit_softcapping",
|
||||
"output_soft_cap_temp",
|
||||
"attn_logit_softcapping",
|
||||
"use_bidirectional_attention",
|
||||
"qk_scale_factor",
|
||||
"use_qk_norm",
|
||||
"use_attn_output_gate",
|
||||
"output_multiplier",
|
||||
"normalize_tok_embeddings",
|
||||
"post_norm_eps",
|
||||
"no_rope_layers",
|
||||
}
|
||||
)
|
||||
# flat vision field name -> canonical MuseGlimmerVisionConfig field name
|
||||
_FLAT_VISION_RENAMES = {
|
||||
"vision_latent_dim": "hidden_size",
|
||||
"vision_heads": "num_attention_heads",
|
||||
"vision_layers": "num_hidden_layers",
|
||||
"vision_output_dim": "output_dim",
|
||||
"vision_patch_size": "patch_size",
|
||||
"vision_patch_temporal": "patch_temporal",
|
||||
"vision_adapter_dim": "adapter_dim",
|
||||
"vision_pos_emb_grid_h": "pos_emb_height",
|
||||
"vision_pos_emb_grid_w": "pos_emb_width",
|
||||
"vision_downsample_factor": "merge_kernel_size",
|
||||
}
|
||||
|
||||
@classmethod
|
||||
def _looks_flat(cls, kwargs: dict) -> bool:
|
||||
# Flat if there is no explicit text_config but there ARE text-level
|
||||
# fields at the top level (e.g. hidden_size / num_hidden_layers).
|
||||
if kwargs.get("text_config") is not None:
|
||||
return False
|
||||
return any(k in kwargs for k in ("hidden_size", "num_hidden_layers"))
|
||||
|
||||
@classmethod
|
||||
def _normalize_flat(cls, kwargs: dict) -> dict:
|
||||
kwargs = dict(kwargs)
|
||||
text: dict = {}
|
||||
for key in list(kwargs.keys()):
|
||||
if key in cls._FLAT_TEXT_KEYS:
|
||||
canon = cls._FLAT_TEXT_RENAMES.get(key, key)
|
||||
text.setdefault(canon, kwargs.pop(key))
|
||||
|
||||
vision_mlp_ratio = kwargs.pop("vision_mlp_ratio", None)
|
||||
sparse_factor = kwargs.pop("vision_sparse_attention_factor", None)
|
||||
vision: dict = {}
|
||||
for key in list(kwargs.keys()):
|
||||
if key in cls._FLAT_VISION_RENAMES:
|
||||
vision.setdefault(cls._FLAT_VISION_RENAMES[key], kwargs.pop(key))
|
||||
|
||||
if vision_mlp_ratio is not None:
|
||||
hidden_size = vision.get("hidden_size", 1536)
|
||||
vision["intermediate_size"] = int(vision_mlp_ratio * hidden_size)
|
||||
if sparse_factor is not None:
|
||||
sparse_factor = int(sparse_factor)
|
||||
if sparse_factor <= 0:
|
||||
raise ValueError("vision_sparse_attention_factor must be positive")
|
||||
num_layers = vision.get("num_hidden_layers", 50)
|
||||
vision["layer_types"] = [
|
||||
"full_attention"
|
||||
if (layer_idx + 1) % sparse_factor == 0 or layer_idx == num_layers - 1
|
||||
else "sliding_attention"
|
||||
for layer_idx in range(num_layers)
|
||||
]
|
||||
|
||||
if text:
|
||||
kwargs["text_config"] = text
|
||||
if vision:
|
||||
kwargs["vision_config"] = vision
|
||||
return kwargs
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
text_config: dict | MuseGlimmerTextConfig | None = None,
|
||||
vision_config: dict | MuseGlimmerVisionConfig | None = None,
|
||||
image_token_id: int = 200092,
|
||||
video_token_id: int = 200091,
|
||||
**kwargs,
|
||||
) -> None:
|
||||
# Detect + fold a flat config into nested text/vision before building
|
||||
# the sub-configs. (image/video token ids are handled below; a flat
|
||||
# ``patch_token_id`` alias is mapped too.)
|
||||
if text_config is None and self._looks_flat(kwargs):
|
||||
kwargs = self._normalize_flat(kwargs)
|
||||
text_config = kwargs.pop("text_config", None)
|
||||
vision_config = kwargs.pop("vision_config", vision_config)
|
||||
# flat image-token alias
|
||||
if "patch_token_id" in kwargs and "image_token_id" not in kwargs:
|
||||
image_token_id = kwargs.pop("patch_token_id")
|
||||
|
||||
if text_config is None:
|
||||
self.text_config = MuseGlimmerTextConfig()
|
||||
elif isinstance(text_config, dict):
|
||||
self.text_config = MuseGlimmerTextConfig(**text_config)
|
||||
else:
|
||||
self.text_config = text_config
|
||||
|
||||
if vision_config is None:
|
||||
self.vision_config = MuseGlimmerVisionConfig()
|
||||
elif isinstance(vision_config, dict):
|
||||
self.vision_config = MuseGlimmerVisionConfig(**vision_config)
|
||||
else:
|
||||
self.vision_config = vision_config
|
||||
|
||||
self.image_token_id = image_token_id
|
||||
self.video_token_id = video_token_id
|
||||
self.patch_token_id = image_token_id
|
||||
super().__init__(**kwargs)
|
||||
|
||||
|
||||
class MuseGlimmerAssistantConfig(Qwen3Config):
|
||||
"""Config for the Muse Glimmer DFlash draft head.
|
||||
|
||||
The head is Qwen3-shaped and runs on vLLM's generic ``qwen3_dflash``
|
||||
implementation, so this derives from ``Qwen3Config``. It cannot BE
|
||||
``Qwen3Config``, because two of the checkpoint's values do not survive that
|
||||
class:
|
||||
|
||||
* ``sliding_window`` is gated behind ``use_sliding_window``, which defaults
|
||||
to False -- ``Qwen3Config(sliding_window=2048).sliding_window`` is None.
|
||||
The checkpoint declares ``sliding_window: 2048`` and five
|
||||
``sliding_attention`` layers, so the window silently disappears and the
|
||||
DFlash path then raises "sliding attention requires a window size".
|
||||
* ``vocab_size`` is absent from the checkpoint, so Qwen3's default of
|
||||
151936 applies instead of Muse Glimmer's 202048. That one is *silent*: it
|
||||
builds an all-zero ``draft_id_to_target_id`` remap, and also puts
|
||||
pad/bos/eos/mask token ids out of range.
|
||||
|
||||
Both defaults are set here so an unmodified checkpoint loads correctly.
|
||||
"""
|
||||
|
||||
model_type = "muse_glimmer_assistant"
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
vocab_size: int = 202048,
|
||||
use_sliding_window: bool = True,
|
||||
dflash_config: dict | None = None,
|
||||
**kwargs,
|
||||
) -> None:
|
||||
super().__init__(
|
||||
vocab_size=vocab_size, use_sliding_window=use_sliding_window, **kwargs
|
||||
)
|
||||
# muse_glimmer_assistant uses non-causal attention.
|
||||
self.dflash_config = {"causal": False} | (dflash_config or {})
|
||||
|
||||
|
||||
__all__ = [
|
||||
"MuseGlimmerTextConfig",
|
||||
"MuseGlimmerVisionConfig",
|
||||
"MuseGlimmerConfig",
|
||||
"MuseGlimmerAssistantConfig",
|
||||
]
|
||||
@@ -1,50 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""OpenVLA configuration support.
|
||||
|
||||
OpenVLA checkpoints use a custom ``model_type`` and nest the language model
|
||||
configuration under ``text_config``. This shim lets vLLM load the checkpoint
|
||||
configuration without executing Hugging Face remote code.
|
||||
"""
|
||||
|
||||
from typing import Any
|
||||
|
||||
from transformers import LlamaConfig, PreTrainedConfig
|
||||
|
||||
|
||||
class OpenVLAConfig(PreTrainedConfig):
|
||||
"""Configuration class for OpenVLA models."""
|
||||
|
||||
model_type = "openvla"
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
timm_model_ids: list[str] | None = None,
|
||||
timm_override_act_layers: list[str | None] | None = None,
|
||||
image_sizes: list[int] | None = None,
|
||||
use_fused_vision_backbone: bool = True,
|
||||
image_token_index: int = 32000,
|
||||
n_action_bins: int = 256,
|
||||
text_config: dict[str, Any] | LlamaConfig | None = None,
|
||||
**kwargs: Any,
|
||||
) -> None:
|
||||
kwargs.setdefault("architectures", ["OpenVLAForActionPrediction"])
|
||||
super().__init__(**kwargs)
|
||||
|
||||
self.timm_model_ids = timm_model_ids or [
|
||||
"vit_large_patch14_reg4_dinov2.lvd142m",
|
||||
"vit_so400m_patch14_siglip_224",
|
||||
]
|
||||
self.timm_override_act_layers = timm_override_act_layers or [None, None]
|
||||
self.image_sizes = image_sizes or [224, 224]
|
||||
self.use_fused_vision_backbone = use_fused_vision_backbone
|
||||
self.image_token_index = image_token_index
|
||||
self.n_action_bins = n_action_bins
|
||||
|
||||
if text_config is None:
|
||||
text_config = LlamaConfig(architectures=["LlamaForCausalLM"])
|
||||
elif isinstance(text_config, dict):
|
||||
text_config = text_config.copy()
|
||||
text_config.setdefault("architectures", ["LlamaForCausalLM"])
|
||||
text_config = LlamaConfig(**text_config)
|
||||
self.text_config = text_config
|
||||
@@ -1,124 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
# ruff: noqa: E501
|
||||
# adapted from https://huggingface.co/AIDC-AI/Ovis2-1B/blob/main/configuration_ovis.py
|
||||
from typing import Any
|
||||
|
||||
from transformers import Aimv2VisionConfig, AutoConfig, PreTrainedConfig
|
||||
|
||||
|
||||
# ----------------------------------------------------------------------
|
||||
# Visual Tokenizer Configuration
|
||||
# ----------------------------------------------------------------------
|
||||
class BaseVisualTokenizerConfig(PreTrainedConfig):
|
||||
def __init__(
|
||||
self,
|
||||
vocab_size=16384,
|
||||
tokenize_function="softmax",
|
||||
tau=1.0,
|
||||
depths=None,
|
||||
drop_cls_token=False,
|
||||
backbone_config: PreTrainedConfig | dict | None = None,
|
||||
hidden_stride: int = 1,
|
||||
**kwargs,
|
||||
):
|
||||
super().__init__(**kwargs)
|
||||
self.vocab_size = vocab_size
|
||||
self.tokenize_function = tokenize_function
|
||||
self.tau = tau
|
||||
if isinstance(depths, str):
|
||||
depths = [int(x) for x in depths.split("|")]
|
||||
self.depths = depths
|
||||
self.backbone_kwargs = dict[str, Any]()
|
||||
self.drop_cls_token = drop_cls_token
|
||||
if backbone_config is not None:
|
||||
assert isinstance(backbone_config, (PreTrainedConfig, dict)), (
|
||||
f"expect `backbone_config` to be instance of PreTrainedConfig or dict, but got {type(backbone_config)} type"
|
||||
)
|
||||
if not isinstance(backbone_config, PreTrainedConfig):
|
||||
model_type = backbone_config.pop("model_type")
|
||||
if model_type == "aimv2":
|
||||
backbone_config = Aimv2VisionConfig(**backbone_config)
|
||||
else:
|
||||
backbone_config = AutoConfig.for_model(
|
||||
model_type, **backbone_config
|
||||
)
|
||||
self.backbone_config = backbone_config
|
||||
self.hidden_stride = hidden_stride
|
||||
|
||||
|
||||
class Aimv2VisualTokenizerConfig(BaseVisualTokenizerConfig):
|
||||
model_type = "aimv2_visual_tokenizer"
|
||||
|
||||
def __init__(self, **kwargs):
|
||||
super().__init__(**kwargs)
|
||||
if self.drop_cls_token:
|
||||
self.drop_cls_token = False
|
||||
if self.depths:
|
||||
assert len(self.depths) == 1
|
||||
self.backbone_kwargs["num_hidden_layers"] = self.depths[0]
|
||||
|
||||
|
||||
class SiglipVisualTokenizerConfig(BaseVisualTokenizerConfig):
|
||||
model_type = "siglip_visual_tokenizer"
|
||||
|
||||
def __init__(self, **kwargs):
|
||||
super().__init__(**kwargs)
|
||||
if self.drop_cls_token:
|
||||
self.drop_cls_token = False
|
||||
if self.depths:
|
||||
assert len(self.depths) == 1
|
||||
self.backbone_kwargs["num_hidden_layers"] = self.depths[0]
|
||||
|
||||
|
||||
AutoConfig.register("siglip_visual_tokenizer", SiglipVisualTokenizerConfig)
|
||||
AutoConfig.register("aimv2_visual_tokenizer", Aimv2VisualTokenizerConfig)
|
||||
|
||||
|
||||
# ----------------------------------------------------------------------
|
||||
# Ovis Configuration
|
||||
# ----------------------------------------------------------------------
|
||||
class OvisConfig(PreTrainedConfig):
|
||||
model_type = "ovis"
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
llm_config: PreTrainedConfig | dict | None = None,
|
||||
visual_tokenizer_config: PreTrainedConfig | dict | None = None,
|
||||
multimodal_max_length=8192,
|
||||
hidden_size=None,
|
||||
conversation_formatter_class=None,
|
||||
llm_attn_implementation=None,
|
||||
disable_tie_weight=False,
|
||||
**kwargs,
|
||||
):
|
||||
super().__init__(**kwargs)
|
||||
if llm_config is not None:
|
||||
assert isinstance(llm_config, (PreTrainedConfig, dict)), (
|
||||
f"expect `llm_config` to be instance of PreTrainedConfig or dict, but got {type(llm_config)} type"
|
||||
)
|
||||
if not isinstance(llm_config, PreTrainedConfig):
|
||||
model_type = llm_config["model_type"]
|
||||
llm_config.pop("model_type")
|
||||
llm_config = AutoConfig.for_model(model_type, **llm_config)
|
||||
|
||||
# map llm_config to text_config
|
||||
self.text_config = llm_config
|
||||
if visual_tokenizer_config is not None:
|
||||
assert isinstance(visual_tokenizer_config, (PreTrainedConfig, dict)), (
|
||||
f"expect `visual_tokenizer_config` to be instance of PreTrainedConfig or dict, but got {type(visual_tokenizer_config)} type"
|
||||
)
|
||||
if not isinstance(visual_tokenizer_config, PreTrainedConfig):
|
||||
model_type = visual_tokenizer_config["model_type"]
|
||||
visual_tokenizer_config.pop("model_type")
|
||||
visual_tokenizer_config = AutoConfig.for_model(
|
||||
model_type, **visual_tokenizer_config
|
||||
)
|
||||
|
||||
self.visual_tokenizer_config = visual_tokenizer_config
|
||||
self.multimodal_max_length = multimodal_max_length
|
||||
self.hidden_size = hidden_size
|
||||
self.conversation_formatter_class = conversation_formatter_class
|
||||
self.llm_attn_implementation = llm_attn_implementation
|
||||
self.disable_tie_weight = disable_tie_weight
|
||||
@@ -1,105 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from typing import Any
|
||||
|
||||
from transformers import PreTrainedConfig
|
||||
from transformers.models.auto import CONFIG_MAPPING
|
||||
|
||||
|
||||
class QianfanOCRVisionConfig(PreTrainedConfig):
|
||||
model_type = "qianfan_ocr_vision"
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
hidden_size: int = 1024,
|
||||
intermediate_size: int = 4096,
|
||||
num_hidden_layers: int = 24,
|
||||
num_attention_heads: int = 16,
|
||||
num_channels: int = 3,
|
||||
image_size: int = 448,
|
||||
patch_size: int = 14,
|
||||
hidden_act: str = "gelu",
|
||||
layer_norm_eps: float = 1e-6,
|
||||
attention_dropout: float = 0.0,
|
||||
drop_path_rate: float = 0.1,
|
||||
qkv_bias: bool = True,
|
||||
qk_normalization: bool = False,
|
||||
norm_type: str = "layer_norm",
|
||||
initializer_range: float = 0.02,
|
||||
initializer_factor: float = 0.1,
|
||||
use_mask_token: bool = False,
|
||||
use_mean_pooling: bool = True,
|
||||
**kwargs: Any,
|
||||
):
|
||||
super().__init__(**kwargs)
|
||||
self.hidden_size = hidden_size
|
||||
self.intermediate_size = intermediate_size
|
||||
self.num_hidden_layers = num_hidden_layers
|
||||
self.num_attention_heads = num_attention_heads
|
||||
self.num_channels = num_channels
|
||||
self.image_size = image_size
|
||||
self.patch_size = patch_size
|
||||
self.hidden_act = hidden_act
|
||||
self.layer_norm_eps = layer_norm_eps
|
||||
self.attention_dropout = attention_dropout
|
||||
self.drop_path_rate = drop_path_rate
|
||||
self.qkv_bias = qkv_bias
|
||||
self.qk_normalization = qk_normalization
|
||||
self.norm_type = norm_type
|
||||
self.initializer_range = initializer_range
|
||||
self.initializer_factor = initializer_factor
|
||||
self.use_mask_token = use_mask_token
|
||||
self.use_mean_pooling = use_mean_pooling
|
||||
|
||||
|
||||
class QianfanOCRConfig(PreTrainedConfig):
|
||||
model_type = "qianfan_ocr"
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
vision_config: dict | None = None,
|
||||
text_config: dict | None = None,
|
||||
downsample_ratio: float = 0.5,
|
||||
dynamic_image_size: bool = True,
|
||||
force_image_size: int = 448,
|
||||
image_token_id: int = 151671,
|
||||
max_dynamic_patch: int = 12,
|
||||
min_dynamic_patch: int = 1,
|
||||
pad2square: bool = False,
|
||||
ps_version: str = "v2",
|
||||
select_layer: int = -1,
|
||||
template: str = "internvl2_5",
|
||||
use_thumbnail: bool = True,
|
||||
tie_word_embeddings: bool = False,
|
||||
**kwargs: Any,
|
||||
):
|
||||
super().__init__(**kwargs)
|
||||
|
||||
if isinstance(vision_config, dict):
|
||||
self.vision_config = QianfanOCRVisionConfig(**vision_config)
|
||||
elif vision_config is None:
|
||||
self.vision_config = QianfanOCRVisionConfig()
|
||||
else:
|
||||
self.vision_config = vision_config
|
||||
|
||||
if isinstance(text_config, dict):
|
||||
model_type = text_config.get("model_type", "qwen3")
|
||||
self.text_config = CONFIG_MAPPING[model_type](**text_config)
|
||||
elif text_config is None:
|
||||
self.text_config = CONFIG_MAPPING["qwen3"]()
|
||||
else:
|
||||
self.text_config = text_config
|
||||
|
||||
self.downsample_ratio = downsample_ratio
|
||||
self.dynamic_image_size = dynamic_image_size
|
||||
self.force_image_size = force_image_size
|
||||
self.image_token_id = image_token_id
|
||||
self.max_dynamic_patch = max_dynamic_patch
|
||||
self.min_dynamic_patch = min_dynamic_patch
|
||||
self.pad2square = pad2square
|
||||
self.ps_version = ps_version
|
||||
self.select_layer = select_layer
|
||||
self.template = template
|
||||
self.use_thumbnail = use_thumbnail
|
||||
self.tie_word_embeddings = tie_word_embeddings
|
||||
@@ -1,455 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
# ruff: noqa
|
||||
# mypy: ignore-errors
|
||||
# coding=utf-8
|
||||
# Copyright 2026 The Qwen team, Alibaba Group and the HuggingFace Inc. team. All rights reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
from transformers.configuration_utils import PreTrainedConfig
|
||||
from transformers.modeling_rope_utils import rope_config_validation
|
||||
from transformers.utils import logging
|
||||
|
||||
logger = logging.get_logger(__name__)
|
||||
|
||||
|
||||
class Qwen3ASRAudioEncoderConfig(PreTrainedConfig):
|
||||
r"""
|
||||
This is the configuration class to store the configuration of a [`Qwen3ASRAudioEncoder`]. It is used to instantiate a
|
||||
Qwen3-ASR audio encoder according to the specified arguments, defining the model architecture. Instantiating a
|
||||
configuration with the defaults will yield a similar configuration to that of the audio encoder of the Qwen2-Audio
|
||||
architecture.
|
||||
|
||||
e.g. [Qwen/Qwen3-ASR-1.7B](https://huggingface.co/Qwen/Qwen3-ASR-1.7B)
|
||||
|
||||
Configuration objects inherit from [`PreTrainedConfig`] and can be used to control the model outputs. Read the
|
||||
documentation from [`PreTrainedConfig`] for more information.
|
||||
|
||||
Args:
|
||||
num_mel_bins (`int`, *optional*, defaults to 128):
|
||||
Number of mel features used per input features. Should correspond to the value used in the
|
||||
`Qwen3ASRProcessor` class.
|
||||
encoder_layers (`int`, *optional*, defaults to 32):
|
||||
Number of encoder layers.
|
||||
encoder_attention_heads (`int`, *optional*, defaults to 20):
|
||||
Number of attention heads for each attention layer in the Transformer encoder.
|
||||
encoder_ffn_dim (`int`, *optional*, defaults to 5120):
|
||||
Dimensionality of the "intermediate" (often named feed-forward) layer in encoder.
|
||||
d_model (`int`, *optional*, defaults to 1280):
|
||||
Dimensionality of the layers.
|
||||
dropout (`float`, *optional*, defaults to 0.0):
|
||||
The dropout probability for all fully connected layers in the embeddings, encoder, and pooler.
|
||||
attention_dropout (`float`, *optional*, defaults to 0.0):
|
||||
The dropout ratio for the attention probabilities.
|
||||
activation_function (`str`, *optional*, defaults to `"gelu"`):
|
||||
The non-linear activation function (function or string) in the encoder and pooler. If string, `"gelu"`,
|
||||
`"relu"`, `"silu"` and `"gelu_new"` are supported.
|
||||
activation_dropout (`float`, *optional*, defaults to 0.0):
|
||||
The dropout ratio for activations inside the fully connected layer.
|
||||
scale_embedding (`bool`, *optional*, defaults to `False`):
|
||||
Scale embeddings by diving by sqrt(d_model).
|
||||
initializer_range (`float`, *optional*, defaults to 0.02):
|
||||
The standard deviation of the truncated_normal_initializer for initializing all weight matrices.
|
||||
max_source_positions (`int`, *optional*, defaults to 1500):
|
||||
The maximum sequence length of log-mel filter-bank features that this model might ever be used with.
|
||||
n_window (`int`, *optional*, defaults to 100):
|
||||
The chunk for conv and flash attn in AudioEncoder.
|
||||
output_dim (`int`, *optional*, defaults to 3584):
|
||||
The output dimension of AudioEncoder.
|
||||
|
||||
Example:
|
||||
|
||||
```python
|
||||
>>> from transformers import Qwen3ASRAudioEncoderConfig, Qwen3ASRAudioEncoder
|
||||
|
||||
>>> # Initializing a Qwen3ASRAudioEncoderConfig
|
||||
>>> configuration = Qwen3ASRAudioEncoderConfig()
|
||||
|
||||
>>> # Initializing a Qwen3ASRAudioEncoder (with random weights)
|
||||
>>> model = Qwen3ASRAudioEncoder(configuration)
|
||||
|
||||
>>> # Accessing the model configuration
|
||||
>>> configuration = model.config
|
||||
```"""
|
||||
|
||||
model_type = "qwen3_asr_audio_encoder"
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
num_mel_bins=128,
|
||||
encoder_layers=32,
|
||||
encoder_attention_heads=20,
|
||||
encoder_ffn_dim=5120,
|
||||
d_model=1280,
|
||||
dropout=0,
|
||||
attention_dropout=0,
|
||||
activation_function="gelu",
|
||||
activation_dropout=0,
|
||||
scale_embedding=False,
|
||||
initializer_range=0.02,
|
||||
max_source_positions=1500,
|
||||
n_window=100,
|
||||
output_dim=3584,
|
||||
n_window_infer=400,
|
||||
conv_chunksize=500,
|
||||
downsample_hidden_size=480,
|
||||
**kwargs,
|
||||
):
|
||||
super().__init__(**kwargs)
|
||||
|
||||
self.num_mel_bins = num_mel_bins
|
||||
self.d_model = d_model
|
||||
self.encoder_layers = encoder_layers
|
||||
self.encoder_attention_heads = encoder_attention_heads
|
||||
self.encoder_ffn_dim = encoder_ffn_dim
|
||||
self.dropout = dropout
|
||||
self.attention_dropout = attention_dropout
|
||||
self.activation_function = activation_function
|
||||
self.activation_dropout = activation_dropout
|
||||
self.num_hidden_layers = encoder_layers
|
||||
self.initializer_range = initializer_range
|
||||
self.scale_embedding = (
|
||||
scale_embedding # scale factor will be sqrt(d_model) if True
|
||||
)
|
||||
self.max_source_positions = max_source_positions
|
||||
self.n_window = n_window
|
||||
self.output_dim = output_dim
|
||||
self.n_window_infer = n_window_infer
|
||||
self.conv_chunksize = conv_chunksize
|
||||
self.downsample_hidden_size = downsample_hidden_size
|
||||
|
||||
|
||||
class Qwen3ASRTextConfig(PreTrainedConfig):
|
||||
r"""
|
||||
This is the configuration class to store the configuration of a [`Qwen3ASRTextModel`]. It is used to instantiate a
|
||||
Qwen3-ASR model according to the specified arguments, defining the model architecture. Instantiating a configuration
|
||||
with the defaults will yield a similar configuration to that of
|
||||
Qwen3-ASR-1.7B [Qwen/Qwen3-ASR-1.7B](https://huggingface.co/Qwen/Qwen3-ASR-1.7B)
|
||||
|
||||
Configuration objects inherit from [`PreTrainedConfig`] and can be used to control the model outputs. Read the
|
||||
documentation from [`PreTrainedConfig`] for more information.
|
||||
|
||||
Args:
|
||||
vocab_size (`int`, *optional*, defaults to 151936):
|
||||
Vocabulary size of the Qwen3ASR model. Defines the number of different tokens that can be represented by the
|
||||
`inputs_ids` passed when calling [`Qwen3ASRModel`]
|
||||
hidden_size (`int`, *optional*, defaults to 4096):
|
||||
Dimension of the hidden representations.
|
||||
intermediate_size (`int`, *optional*, defaults to 22016):
|
||||
Dimension of the MLP representations.
|
||||
num_hidden_layers (`int`, *optional*, defaults to 32):
|
||||
Number of hidden layers in the Transformer encoder.
|
||||
num_attention_heads (`int`, *optional*, defaults to 32):
|
||||
Number of attention heads for each attention layer in the Transformer encoder.
|
||||
num_key_value_heads (`int`, *optional*, defaults to 32):
|
||||
This is the number of key_value heads that should be used to implement Grouped Query Attention. If
|
||||
`num_key_value_heads=num_attention_heads`, the model will use Multi Head Attention (MHA), if
|
||||
`num_key_value_heads=1` the model will use Multi Query Attention (MQA) otherwise GQA is used. When
|
||||
converting a multi-head checkpoint to a GQA checkpoint, each group key and value head should be constructed
|
||||
by meanpooling all the original heads within that group. For more details, check out [this
|
||||
paper](https://huggingface.co/papers/2305.13245). If it is not specified, will default to `32`.
|
||||
head_dim (`int`, *optional*, defaults to 128):
|
||||
The dimension of the head. If not specified, will default to `hidden_size // num_attention_heads`.
|
||||
hidden_act (`str` or `function`, *optional*, defaults to `"silu"`):
|
||||
The non-linear activation function (function or string) in the decoder.
|
||||
max_position_embeddings (`int`, *optional*, defaults to 128000):
|
||||
The maximum sequence length that this model might ever be used with.
|
||||
initializer_range (`float`, *optional*, defaults to 0.02):
|
||||
The standard deviation of the truncated_normal_initializer for initializing all weight matrices.
|
||||
rms_norm_eps (`float`, *optional*, defaults to 1e-06):
|
||||
The epsilon used by the rms normalization layers.
|
||||
use_cache (`bool`, *optional*, defaults to `True`):
|
||||
Whether or not the model should return the last key/values attentions (not used by all models). Only
|
||||
relevant if `config.is_decoder=True`.
|
||||
tie_word_embeddings (`bool`, *optional*, defaults to `False`):
|
||||
Whether the model's input and output word embeddings should be tied.
|
||||
rope_theta (`float`, *optional*, defaults to 5000000.0):
|
||||
The base period of the RoPE embeddings.
|
||||
rope_scaling (`Dict`, *optional*):
|
||||
Dictionary containing the scaling configuration for the RoPE embeddings. NOTE: if you apply new rope type
|
||||
and you expect the model to work on longer `max_position_embeddings`, we recommend you to update this value
|
||||
accordingly.
|
||||
Expected contents:
|
||||
`rope_type` (`str`):
|
||||
The sub-variant of RoPE to use. Can be one of ['default', 'linear', 'dynamic', 'yarn', 'longrope',
|
||||
'llama3'], with 'default' being the original RoPE implementation.
|
||||
`factor` (`float`, *optional*):
|
||||
Used with all rope types except 'default'. The scaling factor to apply to the RoPE embeddings. In
|
||||
most scaling types, a `factor` of x will enable the model to handle sequences of length x *
|
||||
original maximum pre-trained length.
|
||||
`original_max_position_embeddings` (`int`, *optional*):
|
||||
Used with 'dynamic', 'longrope' and 'llama3'. The original max position embeddings used during
|
||||
pretraining.
|
||||
`attention_factor` (`float`, *optional*):
|
||||
Used with 'yarn' and 'longrope'. The scaling factor to be applied on the attention
|
||||
computation. If unspecified, it defaults to value recommended by the implementation, using the
|
||||
`factor` field to infer the suggested value.
|
||||
`beta_fast` (`float`, *optional*):
|
||||
Only used with 'yarn'. Parameter to set the boundary for extrapolation (only) in the linear
|
||||
ramp function. If unspecified, it defaults to 32.
|
||||
`beta_slow` (`float`, *optional*):
|
||||
Only used with 'yarn'. Parameter to set the boundary for interpolation (only) in the linear
|
||||
ramp function. If unspecified, it defaults to 1.
|
||||
`short_factor` (`list[float]`, *optional*):
|
||||
Only used with 'longrope'. The scaling factor to be applied to short contexts (<
|
||||
`original_max_position_embeddings`). Must be a list of numbers with the same length as the hidden
|
||||
size divided by the number of attention heads divided by 2
|
||||
`long_factor` (`list[float]`, *optional*):
|
||||
Only used with 'longrope'. The scaling factor to be applied to long contexts (<
|
||||
`original_max_position_embeddings`). Must be a list of numbers with the same length as the hidden
|
||||
size divided by the number of attention heads divided by 2
|
||||
`low_freq_factor` (`float`, *optional*):
|
||||
Only used with 'llama3'. Scaling factor applied to low frequency components of the RoPE
|
||||
`high_freq_factor` (`float`, *optional*):
|
||||
Only used with 'llama3'. Scaling factor applied to high frequency components of the RoPE
|
||||
attention_bias (`bool`, defaults to `False`, *optional*, defaults to `False`):
|
||||
Whether to use a bias in the query, key, value and output projection layers during self-attention.
|
||||
attention_dropout (`float`, *optional*, defaults to 0.0):
|
||||
The dropout ratio for the attention probabilities.
|
||||
|
||||
```python
|
||||
>>> from transformers import Qwen3ASRTextModel, Qwen3ASRTextConfig
|
||||
|
||||
>>> # Initializing a Qwen3ASR style configuration
|
||||
>>> configuration = Qwen3ASRTextConfig()
|
||||
|
||||
>>> # Initializing a model from the Qwen3-VL-7B style configuration
|
||||
>>> model = Qwen3ASRTextModel(configuration)
|
||||
|
||||
>>> # Accessing the model configuration
|
||||
>>> configuration = model.config
|
||||
```"""
|
||||
|
||||
model_type = "qwen3_asr_text"
|
||||
base_config_key = "text_config"
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
vocab_size=151936,
|
||||
hidden_size=4096,
|
||||
intermediate_size=22016,
|
||||
num_hidden_layers=32,
|
||||
num_attention_heads=32,
|
||||
num_key_value_heads=32,
|
||||
head_dim=128,
|
||||
hidden_act="silu",
|
||||
max_position_embeddings=128000,
|
||||
initializer_range=0.02,
|
||||
rms_norm_eps=1e-6,
|
||||
use_cache=True,
|
||||
tie_word_embeddings=False,
|
||||
rope_theta=5000000.0,
|
||||
rope_scaling=None,
|
||||
attention_bias=False,
|
||||
attention_dropout=0.0,
|
||||
**kwargs,
|
||||
):
|
||||
self.vocab_size = vocab_size
|
||||
self.max_position_embeddings = max_position_embeddings
|
||||
self.hidden_size = hidden_size
|
||||
self.intermediate_size = intermediate_size
|
||||
self.num_hidden_layers = num_hidden_layers
|
||||
self.num_attention_heads = num_attention_heads
|
||||
|
||||
# for backward compatibility
|
||||
if num_key_value_heads is None:
|
||||
num_key_value_heads = num_attention_heads
|
||||
|
||||
self.num_key_value_heads = num_key_value_heads
|
||||
self.head_dim = head_dim
|
||||
self.hidden_act = hidden_act
|
||||
self.initializer_range = initializer_range
|
||||
self.rms_norm_eps = rms_norm_eps
|
||||
self.use_cache = use_cache
|
||||
self.rope_theta = rope_theta
|
||||
self.rope_scaling = rope_scaling
|
||||
self.attention_bias = attention_bias
|
||||
self.attention_dropout = attention_dropout
|
||||
# Validate the correctness of rotary position embeddings parameters
|
||||
# BC: if there is a 'type' field, move it to 'rope_type'.
|
||||
if self.rope_scaling is not None and "type" in self.rope_scaling:
|
||||
self.rope_scaling["rope_type"] = self.rope_scaling["type"]
|
||||
rope_config_validation(self)
|
||||
|
||||
super().__init__(tie_word_embeddings=tie_word_embeddings, **kwargs)
|
||||
|
||||
|
||||
class Qwen3ASRThinkerConfig(PreTrainedConfig):
|
||||
r"""
|
||||
This is the configuration class to store the configuration of a [`Qwen3ASRThinker`]. It is used to instantiate a
|
||||
Qwen3-ASR-Thinker model according to the specified arguments, defining the model architecture. Instantiating a
|
||||
configuration with the defaults will yield a similar configuration to that of the thinker component of the Qwen3-Omni
|
||||
architecture.
|
||||
|
||||
e.g. [Qwen/Qwen3-ASR-1.7B](https://huggingface.co/Qwen/Qwen3-ASR-1.7B)
|
||||
|
||||
Configuration objects inherit from [`PreTrainedConfig`] and can be used to control the model outputs. Read the
|
||||
documentation from [`PreTrainedConfig`] for more information.
|
||||
|
||||
Args:
|
||||
audio_config (`dict`, *optional*):
|
||||
The config dictionary of the audio backbone.
|
||||
text_config (`dict`, *optional*):
|
||||
The config dictionary of the text backbone.
|
||||
audio_token_id (`int`, *optional*, defaults to 151646):
|
||||
The audio token id to encode the audio prompt.
|
||||
audio_start_token_id (`int`, *optional*, defaults to 151647):
|
||||
The audio start token id to encode the audio prompt.
|
||||
user_token_id (`int`, *optional*, defaults to 872):
|
||||
The user token id to encode the user token.
|
||||
initializer_range (`float`, *optional*, defaults to 0.02):
|
||||
The standard deviation of the truncated_normal_initializer for initializing all weight matrices.
|
||||
|
||||
Example:
|
||||
|
||||
```python
|
||||
>>> from transformers import Qwen3ASRThinkerModel, Qwen3ASRThinkerConfig
|
||||
|
||||
>>> # Initializing a default Qwen3ASRThinkerConfig
|
||||
>>> configuration = Qwen3ASRThinkerConfig()
|
||||
|
||||
>>> # Initializing a model (with random weights) from the default configuration
|
||||
>>> model = Qwen3ASRThinkerModel(configuration)
|
||||
|
||||
>>> # Accessing the model configuration
|
||||
>>> configuration = model.config
|
||||
```"""
|
||||
|
||||
model_type = "qwen3_asr_thinker"
|
||||
|
||||
attribute_map = {}
|
||||
sub_configs = {
|
||||
"audio_config": Qwen3ASRAudioEncoderConfig,
|
||||
"text_config": Qwen3ASRTextConfig,
|
||||
}
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
audio_config=None,
|
||||
text_config=None,
|
||||
audio_token_id=151646,
|
||||
audio_start_token_id=151647,
|
||||
user_token_id=872,
|
||||
initializer_range=0.02,
|
||||
classify_num=None,
|
||||
**kwargs,
|
||||
):
|
||||
super().__init__(**kwargs)
|
||||
self.user_token_id = user_token_id
|
||||
self.audio_start_token_id = audio_start_token_id
|
||||
self.initializer_range = initializer_range
|
||||
self.classify_num = classify_num
|
||||
|
||||
if isinstance(audio_config, dict):
|
||||
audio_config = Qwen3ASRAudioEncoderConfig(**audio_config)
|
||||
elif audio_config is None:
|
||||
audio_config = Qwen3ASRAudioEncoderConfig()
|
||||
self.audio_config = audio_config
|
||||
|
||||
if isinstance(text_config, dict):
|
||||
text_config = Qwen3ASRTextConfig(**text_config)
|
||||
elif text_config is None:
|
||||
text_config = Qwen3ASRTextConfig()
|
||||
self.text_config = text_config
|
||||
self.audio_token_id = audio_token_id
|
||||
|
||||
|
||||
class Qwen3ASRConfig(PreTrainedConfig):
|
||||
"""
|
||||
This is the configuration class to store the configuration of a [`Qwen3ASRForConditionalGeneration`]. It is used to instantiate a Qwen3ASR
|
||||
model according to the specified sub-models configurations, defining the model architecture.
|
||||
|
||||
Instantiating a configuration with the defaults will yield a similar configuration to that of the
|
||||
[Qwen/Qwen3-ASR-1.7B](https://huggingface.co/Qwen/Qwen3-ASR-1.7B) architecture.
|
||||
|
||||
Configuration objects inherit from [`PreTrainedConfig`] and can be used to control the model outputs. Read the
|
||||
documentation from [`PreTrainedConfig`] for more information.
|
||||
|
||||
Args:
|
||||
thinker_config (`dict`, *optional*): Configuration of the underlying thinker sub-model.
|
||||
support_languages (`List[str]`, *optional*): The languages supported by the model.
|
||||
|
||||
Example:
|
||||
|
||||
```python
|
||||
>>> from transformers import (
|
||||
... Qwen3ASRThinkerConfig,
|
||||
... Qwen3ASRForConditionalGeneration,
|
||||
... Qwen3ASRConfig,
|
||||
... )
|
||||
|
||||
>>> # Initializing a Qwen3ASR style configuration
|
||||
>>> configuration = Qwen3ASRConfig()
|
||||
|
||||
>>> # Initializing a model from the configuration
|
||||
>>> model = Qwen3ASRForConditionalGeneration(configuration)
|
||||
|
||||
>>> # Accessing the model configuration
|
||||
>>> configuration = model.config
|
||||
```"""
|
||||
|
||||
model_type = "qwen3_asr"
|
||||
sub_configs = {
|
||||
"thinker_config": Qwen3ASRThinkerConfig,
|
||||
}
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
thinker_config=None,
|
||||
audio_config=None,
|
||||
text_config=None,
|
||||
audio_token_id=None,
|
||||
support_languages=None,
|
||||
timestamp_token_id=None,
|
||||
timestamp_segment_time=None,
|
||||
**kwargs,
|
||||
):
|
||||
if thinker_config is None:
|
||||
thinker_config = {
|
||||
key: value
|
||||
for key, value in (
|
||||
("audio_config", audio_config),
|
||||
("text_config", text_config),
|
||||
("audio_token_id", audio_token_id),
|
||||
)
|
||||
if value is not None
|
||||
}
|
||||
if not thinker_config:
|
||||
logger.info(
|
||||
"thinker_config is None. Initializing thinker model with "
|
||||
"default values"
|
||||
)
|
||||
|
||||
self.thinker_config = Qwen3ASRThinkerConfig(**thinker_config)
|
||||
self.support_languages = support_languages
|
||||
self.timestamp_token_id = timestamp_token_id
|
||||
self.timestamp_segment_time = timestamp_segment_time
|
||||
super().__init__(**kwargs)
|
||||
|
||||
def get_text_config(self, decoder=False) -> "PreTrainedConfig":
|
||||
"""
|
||||
Returns the config that is meant to be used with text IO. On most models, it is the original config instance
|
||||
itself. On specific composite models, it is under a set of valid names.
|
||||
|
||||
Args:
|
||||
decoder (`Optional[bool]`, *optional*, defaults to `False`):
|
||||
If set to `True`, then only search for decoder config names.
|
||||
"""
|
||||
# Overridden for deeply nested config like Qwen2.5-Omni. We don't have any omni model
|
||||
# except for Qwen yet. This has to be generalized if more deeply nested configs are
|
||||
# added. NOTE: currently method used only by vLLM
|
||||
return self.thinker_config.get_text_config()
|
||||
|
||||
|
||||
__all__ = ["Qwen3ASRConfig", "Qwen3ASRThinkerConfig", "Qwen3ASRAudioEncoderConfig"]
|
||||
@@ -1,110 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Radio vision model configuration."""
|
||||
|
||||
from typing import Any
|
||||
|
||||
from transformers.configuration_utils import PreTrainedConfig
|
||||
from transformers.utils import logging
|
||||
|
||||
logger = logging.get_logger(__name__)
|
||||
|
||||
VIT_TIMM_DIM_BY_NAME: dict[str, tuple[int, int, int, int]] = {
|
||||
"vit_small_patch16_224": (384, 12, 6, 1536),
|
||||
"vit_base_patch16_224": (768, 12, 12, 3072),
|
||||
"vit_large_patch16_224": (1024, 24, 16, 4096),
|
||||
"vit_huge_patch16_224": (1280, 32, 16, 5120),
|
||||
}
|
||||
|
||||
OPENAI_CLIP_MEAN = (0.48145466, 0.4578275, 0.40821073)
|
||||
OPENAI_CLIP_STD = (0.26862954, 0.26130258, 0.27577711)
|
||||
|
||||
|
||||
class RadioConfig(PreTrainedConfig):
|
||||
r"""This is the configuration class to store the configuration of a Radio
|
||||
vision model. It is used to instantiate a Radio model according to the
|
||||
specified arguments, defining the model architecture.
|
||||
|
||||
Args:
|
||||
model_name: Name of the vision transformer model
|
||||
(e.g., "vit_base_patch16_224"). Used to determine architecture
|
||||
dimensions from `VIT_TIMM_DIM_BY_NAME`.
|
||||
image_size: The size (resolution) of each image.
|
||||
patch_size: The size (resolution) of each patch.
|
||||
qkv_bias: Whether to add a bias to the queries, keys and values.
|
||||
qk_normalization: Whether to apply normalization to queries and keys.
|
||||
norm_type: The normalization type to use.
|
||||
layer_norm_eps: The epsilon used by the layer normalization layers.
|
||||
initializer_factor: A factor for initializing all weight matrices.
|
||||
hidden_act: The non-linear activation function in the encoder.
|
||||
cpe_max_size: Maximum image size for position embeddings.
|
||||
norm_mean: Mean values for image normalization (RGB channels).
|
||||
Defaults to (0.48145466, 0.4578275, 0.40821073)).
|
||||
norm_std: Standard deviation values for image normalization
|
||||
(RGB channels). Defaults to (0.26862954, 0.26130258, 0.27577711)).
|
||||
register_multiple: Number of register tokens to use.
|
||||
teachers: A list of teacher model configurations. Each teacher configuration is
|
||||
a dict with keys like "name" and some may have "use_summary".
|
||||
cls_token_per_teacher: Whether to use a separate CLS token for each teacher.
|
||||
video_temporal_patch_size: Number of consecutive video frames grouped into
|
||||
a single tubelet for temporal compression. Default 1 (no compression).
|
||||
When > 1, a dedicated video_embedder (3*T*P*P -> hidden) is created
|
||||
alongside the image embedder (3*P*P -> hidden).
|
||||
separate_video_embedder: When True and video_temporal_patch_size > 1, use a
|
||||
dedicated video patch embedder (3*T*P*P -> hidden) separate from the
|
||||
image embedder (3*P*P -> hidden). When False, a single embedder with
|
||||
input size 3*T*P*P is used for both (images are duplicated T times).
|
||||
|
||||
"""
|
||||
|
||||
model_type = "radio"
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
model_name: str,
|
||||
image_size: int = 224,
|
||||
patch_size: int = 16,
|
||||
qkv_bias: bool = True,
|
||||
qk_normalization: bool = False,
|
||||
norm_type: str = "layer_norm",
|
||||
layer_norm_eps: float = 1e-6,
|
||||
initializer_factor: float = 1.0,
|
||||
hidden_act: str = "gelu",
|
||||
cpe_max_size: int = 2048,
|
||||
norm_mean: tuple[float, float, float] | list = OPENAI_CLIP_MEAN,
|
||||
norm_std: tuple[float, float, float] | list = OPENAI_CLIP_STD,
|
||||
register_multiple: int | None = None,
|
||||
teachers: list[dict[str, Any]] | None = None,
|
||||
cls_token_per_teacher: bool = False,
|
||||
video_temporal_patch_size: int = 1,
|
||||
separate_video_embedder: bool = True,
|
||||
**kwargs,
|
||||
):
|
||||
self.model_name = model_name
|
||||
(
|
||||
self.hidden_size,
|
||||
self.num_hidden_layers,
|
||||
self.num_attention_heads,
|
||||
self.intermediate_size,
|
||||
) = VIT_TIMM_DIM_BY_NAME[model_name]
|
||||
self.image_size = image_size
|
||||
self.patch_size = patch_size
|
||||
self.qkv_bias = qkv_bias
|
||||
self.qk_normalization = qk_normalization
|
||||
self.norm_type = norm_type
|
||||
self.layer_norm_eps = layer_norm_eps
|
||||
self.initializer_factor = initializer_factor
|
||||
self.hidden_act = hidden_act
|
||||
self.cpe_max_size = cpe_max_size
|
||||
self.norm_mean = (
|
||||
list(norm_mean) if isinstance(norm_mean, (tuple, list)) else norm_mean
|
||||
)
|
||||
self.norm_std = (
|
||||
list(norm_std) if isinstance(norm_std, (tuple, list)) else norm_std
|
||||
)
|
||||
self.register_multiple = register_multiple
|
||||
self.teachers = teachers if teachers is not None else []
|
||||
self.cls_token_per_teacher = cls_token_per_teacher
|
||||
self.video_temporal_patch_size = video_temporal_patch_size
|
||||
self.separate_video_embedder = separate_video_embedder
|
||||
super().__init__(**kwargs)
|
||||
@@ -1,178 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from typing import Any
|
||||
|
||||
from transformers.configuration_utils import PreTrainedConfig
|
||||
|
||||
|
||||
class Step3VisionEncoderConfig(PreTrainedConfig):
|
||||
model_type = "step3_vision_encoder"
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
hidden_size=1792,
|
||||
intermediate_size=3072,
|
||||
output_hidden_size=4096,
|
||||
num_hidden_layers=63,
|
||||
num_attention_heads=16,
|
||||
num_channels=3,
|
||||
image_size=728,
|
||||
patch_size=14,
|
||||
hidden_act="quick_gelu",
|
||||
layer_norm_eps=1e-5,
|
||||
**kwargs,
|
||||
):
|
||||
self.hidden_size = hidden_size
|
||||
self.intermediate_size = intermediate_size
|
||||
self.output_hidden_size = output_hidden_size
|
||||
self.num_hidden_layers = num_hidden_layers
|
||||
self.num_attention_heads = num_attention_heads
|
||||
self.num_channels = num_channels
|
||||
self.patch_size = patch_size
|
||||
self.image_size = image_size
|
||||
self.layer_norm_eps = layer_norm_eps
|
||||
self.hidden_act = hidden_act
|
||||
super().__init__(**kwargs)
|
||||
|
||||
|
||||
class Step3TextConfig(PreTrainedConfig):
|
||||
model_type = "step3_text"
|
||||
architectures = ["Step3TextForCausalLM"]
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
hidden_size: int = 7168,
|
||||
intermediate_size: int = 18432,
|
||||
num_attention_heads: int = 64,
|
||||
num_attention_groups: int = 1,
|
||||
num_hidden_layers: int = 61,
|
||||
max_seq_len: int = 65536,
|
||||
vocab_size: int = 128815,
|
||||
rms_norm_eps: float = 1e-5,
|
||||
moe_intermediate_size: int = 5120,
|
||||
moe_num_experts: int = 48,
|
||||
moe_top_k: int = 3,
|
||||
rope_parameters: dict[str, Any] | None = None,
|
||||
max_position_embedding: int = 65536,
|
||||
share_expert_dim: int = 5120,
|
||||
share_q_dim: int = 2048,
|
||||
head_dim: int = 256,
|
||||
norm_expert_weight: bool = False,
|
||||
moe_layers_enum: tuple[int, ...] = (
|
||||
4,
|
||||
5,
|
||||
6,
|
||||
7,
|
||||
8,
|
||||
9,
|
||||
10,
|
||||
11,
|
||||
12,
|
||||
13,
|
||||
14,
|
||||
15,
|
||||
16,
|
||||
17,
|
||||
18,
|
||||
19,
|
||||
20,
|
||||
21,
|
||||
22,
|
||||
23,
|
||||
24,
|
||||
25,
|
||||
26,
|
||||
27,
|
||||
28,
|
||||
29,
|
||||
30,
|
||||
31,
|
||||
32,
|
||||
33,
|
||||
34,
|
||||
35,
|
||||
36,
|
||||
37,
|
||||
38,
|
||||
39,
|
||||
40,
|
||||
41,
|
||||
42,
|
||||
43,
|
||||
44,
|
||||
45,
|
||||
46,
|
||||
47,
|
||||
48,
|
||||
49,
|
||||
50,
|
||||
51,
|
||||
52,
|
||||
53,
|
||||
54,
|
||||
55,
|
||||
56,
|
||||
57,
|
||||
58,
|
||||
59,
|
||||
),
|
||||
**kwargs,
|
||||
) -> None:
|
||||
self.hidden_size = hidden_size
|
||||
self.intermediate_size = intermediate_size
|
||||
self.num_attention_heads = num_attention_heads
|
||||
self.num_attention_groups = num_attention_groups
|
||||
self.num_hidden_layers = num_hidden_layers
|
||||
self.max_seq_len = max_seq_len
|
||||
self.vocab_size = vocab_size
|
||||
self.rms_norm_eps = rms_norm_eps
|
||||
self.moe_intermediate_size = moe_intermediate_size
|
||||
self.moe_num_experts = moe_num_experts
|
||||
self.moe_top_k = moe_top_k
|
||||
# Try to set `rope_scaling` if available, otherwise use `rope_parameters`
|
||||
rope_scaling = kwargs.pop("rope_scaling", None)
|
||||
rope_parameters = rope_scaling or rope_parameters or {"rope_type": "default"}
|
||||
rope_theta = kwargs.pop("rope_theta", 500000.0)
|
||||
if "rope_theta" not in rope_parameters:
|
||||
rope_parameters["rope_theta"] = rope_theta
|
||||
self.rope_parameters = rope_parameters
|
||||
self.max_position_embedding = max_position_embedding
|
||||
self.share_expert_dim = share_expert_dim
|
||||
self.share_q_dim = share_q_dim
|
||||
self.head_dim = head_dim
|
||||
self.norm_expert_weight = norm_expert_weight
|
||||
self.moe_layers_enum = moe_layers_enum
|
||||
|
||||
super().__init__(**kwargs)
|
||||
|
||||
|
||||
class Step3VLConfig(PreTrainedConfig):
|
||||
model_type = "step3_vl"
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
vision_config: dict | Step3VisionEncoderConfig | None = None,
|
||||
text_config: dict | Step3TextConfig | None = None,
|
||||
understand_projector_stride: int = 1,
|
||||
projector_bias: bool = True,
|
||||
image_token_id: int = 128001,
|
||||
**kwargs,
|
||||
) -> None:
|
||||
if vision_config is None:
|
||||
vision_config = Step3VisionEncoderConfig()
|
||||
elif isinstance(vision_config, dict):
|
||||
vision_config = Step3VisionEncoderConfig(**vision_config)
|
||||
self.vision_config = vision_config
|
||||
|
||||
if text_config is None:
|
||||
text_config = Step3TextConfig()
|
||||
elif isinstance(text_config, dict):
|
||||
text_config = Step3TextConfig(**text_config)
|
||||
self.text_config = text_config
|
||||
|
||||
self.understand_projector_stride = understand_projector_stride
|
||||
self.projector_bias = projector_bias
|
||||
self.hidden_size = text_config.hidden_size
|
||||
self.image_token_id = image_token_id
|
||||
|
||||
super().__init__(**kwargs)
|
||||
@@ -1,103 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from typing import Any
|
||||
|
||||
from transformers.configuration_utils import PreTrainedConfig
|
||||
|
||||
|
||||
class Step3p5Config(PreTrainedConfig):
|
||||
model_type = "step3p5"
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
hidden_size: int = 5120,
|
||||
intermediate_size: int = 13312,
|
||||
num_attention_heads: int = 40,
|
||||
num_attention_groups: int = 8,
|
||||
num_hidden_layers: int = 48,
|
||||
max_seq_len: int = 4096,
|
||||
vocab_size: int = 65536,
|
||||
rms_norm_eps: float = 1e-5,
|
||||
moe_every_n_layer: int = 2,
|
||||
use_moe: bool = False,
|
||||
moe_intermediate_size: int = 10240,
|
||||
moe_num_experts: int = 16,
|
||||
moe_top_k: int = 4,
|
||||
moe_layer_offset: int = 0,
|
||||
rope_theta: float | list[float] | None = 500000,
|
||||
rope_scaling: dict[str, Any] | None = None,
|
||||
head_dim: int | None = None,
|
||||
share_expert_dim: int | None = None,
|
||||
norm_expert_weight: bool = True,
|
||||
bos_token_id: list[int] | int | None = None,
|
||||
eos_token_id: list[int] | int | None = None,
|
||||
moe_router_activation: str = "softmax",
|
||||
moe_router_scaling_factor: float = 1.0,
|
||||
att_impl_type: str = "GQA",
|
||||
use_head_wise_attn_gate: bool = False,
|
||||
use_moe_router_bias: bool = True,
|
||||
need_fp32_gate: bool = True,
|
||||
layer_types: list[str] | None = None,
|
||||
use_rope_layers: list[bool] | None = None,
|
||||
yarn_only_types: list[str] | None = None,
|
||||
attention_other_setting: dict[str, Any] | None = None,
|
||||
num_nextn_predict_layers: int = 0,
|
||||
swiglu_limits: list[float] | None = None,
|
||||
swiglu_limits_shared: list[float] | None = None,
|
||||
max_position_embeddings: int | None = None,
|
||||
**kwargs,
|
||||
):
|
||||
self.hidden_size = hidden_size
|
||||
self.intermediate_size = intermediate_size
|
||||
self.num_attention_heads = num_attention_heads
|
||||
self.num_attention_groups = num_attention_groups
|
||||
self.num_hidden_layers = num_hidden_layers
|
||||
self.max_seq_len = max_seq_len
|
||||
self.vocab_size = vocab_size
|
||||
self.rms_norm_eps = rms_norm_eps
|
||||
self.use_moe = use_moe
|
||||
self.moe_intermediate_size = moe_intermediate_size
|
||||
self.moe_every_n_layer = moe_every_n_layer
|
||||
self.moe_num_experts = moe_num_experts
|
||||
self.num_experts_per_tok = moe_top_k
|
||||
self.moe_top_k = moe_top_k
|
||||
self.moe_layer_offset = moe_layer_offset
|
||||
|
||||
self.rope_theta = rope_theta
|
||||
self.rope_scaling = rope_scaling
|
||||
self.head_dim = head_dim
|
||||
if share_expert_dim is None:
|
||||
self.share_expert_dim = self.moe_intermediate_size * self.moe_top_k
|
||||
else:
|
||||
self.share_expert_dim = share_expert_dim
|
||||
self.norm_expert_weight = norm_expert_weight
|
||||
|
||||
self.max_position_embeddings = max_position_embeddings
|
||||
self.moe_router_activation = moe_router_activation
|
||||
self.moe_router_scaling_factor = moe_router_scaling_factor
|
||||
self.use_moe_router_bias = use_moe_router_bias
|
||||
self.need_fp32_gate = need_fp32_gate
|
||||
|
||||
self.att_impl_type = att_impl_type
|
||||
self.use_head_wise_attn_gate = use_head_wise_attn_gate
|
||||
# For some reason the checkpoint has longer layer_types than num_hidden_layers
|
||||
if layer_types is not None:
|
||||
layer_types = layer_types[: self.num_hidden_layers]
|
||||
self.layer_types = layer_types
|
||||
self.use_rope_layers = use_rope_layers
|
||||
self.yarn_only_types = yarn_only_types
|
||||
self.attention_other_setting = attention_other_setting
|
||||
self.num_nextn_predict_layers = num_nextn_predict_layers
|
||||
self.swiglu_limits = swiglu_limits
|
||||
self.swiglu_limits_shared = swiglu_limits_shared
|
||||
|
||||
resolved_bos_token_id = 1 if bos_token_id is None else bos_token_id
|
||||
resolved_eos_token_id = [2, 3] if eos_token_id is None else eos_token_id
|
||||
self.bos_token_id = resolved_bos_token_id
|
||||
self.eos_token_id = resolved_eos_token_id
|
||||
|
||||
super().__init__(
|
||||
bos_token_id=resolved_bos_token_id,
|
||||
eos_token_id=resolved_eos_token_id,
|
||||
**kwargs,
|
||||
)
|
||||
@@ -1,120 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
# Adapted from https://github.com/fixie-ai/ultravox/blob/ecd58c4041030bae2ad15aa6bcf04ab43199ea02/ultravox/model/ultravox_config.py
|
||||
from typing import Any
|
||||
|
||||
import transformers
|
||||
|
||||
|
||||
class UltravoxConfig(transformers.PreTrainedConfig):
|
||||
r"""This is the configuration class to store the configuration of a
|
||||
[`UltravoxForConditionalGeneration`]. It is used to instantiate an
|
||||
Ultravox model according to the specified arguments, defining the model
|
||||
architecture.
|
||||
|
||||
Configuration objects inherit from [`PreTrainedConfig`] and can be used to
|
||||
control the model outputs. Read the documentation from [`PreTrainedConfig`]
|
||||
for more information.
|
||||
|
||||
Args:
|
||||
audio_config (`Union[AutoConfig, dict]`, *optional*):
|
||||
Custom audio config or dict.
|
||||
text_config (`Union[AutoConfig, dict]`, *optional*):
|
||||
The config object of the text backbone.
|
||||
audio_model_id (`str`, *optional*):
|
||||
The model ID of the audio backbone.
|
||||
text_model_id (`str`, *optional*):
|
||||
The model ID of the text backbone.
|
||||
ignore_index (`int`, *optional*, defaults to -100):
|
||||
The ignore index for the loss function.
|
||||
audio_token_index (`int`, *optional*, defaults to 32000):
|
||||
The audio token index to encode the audio prompt.
|
||||
stack_factor (`int`, *optional*, defaults to 8):
|
||||
Audio downsampling factor for the multimodal projector.
|
||||
norm_init (`float`, *optional*, defaults to 0.4):
|
||||
The initialization value for the layer normalization.
|
||||
projector_act (`str`, *optional*, defaults to `"swiglu"`):
|
||||
The activation function used by the multimodal projector.
|
||||
projector_ln_mid (`bool`, *optional*, defaults to `False`):
|
||||
Whether to apply layer normalization at the middle of the
|
||||
projector or at the end. Versions v0.4.1 and below
|
||||
use `False`, but v0.5 and above use `True`.
|
||||
|
||||
"""
|
||||
|
||||
model_type = "ultravox"
|
||||
audio_token = "<|audio|>"
|
||||
is_composition = False
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
audio_config: dict[str, Any] | None = None,
|
||||
text_config: dict[str, Any] | None = None,
|
||||
audio_model_id: str | None = None,
|
||||
text_model_id: str | None = None,
|
||||
ignore_index: int = -100,
|
||||
audio_token_index: int = 32000,
|
||||
hidden_size: int = 4096,
|
||||
stack_factor: int = 8,
|
||||
norm_init: float = 0.4,
|
||||
projector_act: str = "swiglu",
|
||||
projector_ln_mid: bool = False,
|
||||
num_projector_layers: int = 0,
|
||||
**kwargs,
|
||||
):
|
||||
self.ignore_index = ignore_index
|
||||
self.audio_token_index = audio_token_index
|
||||
|
||||
self.hidden_size = hidden_size
|
||||
self.stack_factor = stack_factor
|
||||
self.norm_init = norm_init
|
||||
self.projector_act = projector_act
|
||||
self.projector_ln_mid = projector_ln_mid
|
||||
self.num_projector_layers = num_projector_layers
|
||||
|
||||
# N.B. May set the wrapped_model_config below.
|
||||
self.wrapped_model_config: transformers.PreTrainedConfig
|
||||
self.text_model_id = text_model_id
|
||||
if text_model_id is None:
|
||||
text_config = text_config or {}
|
||||
self.wrapped_model_config = transformers.CONFIG_MAPPING[
|
||||
text_config.get("model_type", "llama")
|
||||
](**text_config)
|
||||
|
||||
# N.B. May set the audio_config below.
|
||||
self.audio_model_id = audio_model_id
|
||||
if audio_model_id is None:
|
||||
self.audio_model_id = None
|
||||
audio_config = audio_config or {}
|
||||
self.audio_config = transformers.CONFIG_MAPPING[
|
||||
audio_config.get("model_type", "whisper")
|
||||
](**audio_config)
|
||||
|
||||
super().__init__(**kwargs)
|
||||
|
||||
def __setattr__(self, key, value):
|
||||
# Since --hf-overrides are applied _after_ the UltravoxConfig is
|
||||
# instantiated, load the configs implicitly when assigning text_model_id
|
||||
# or audio_model_id. This allows:
|
||||
#
|
||||
# --hf-overrides.text_model_id=<quantized variant>
|
||||
#
|
||||
# to behave as intended.
|
||||
if key == "text_model_id" and value is not None:
|
||||
from vllm.transformers_utils.config import get_config
|
||||
|
||||
self.wrapped_model_config = get_config(value, trust_remote_code=False)
|
||||
elif key == "audio_model_id" and value is not None:
|
||||
from vllm.transformers_utils.config import get_config
|
||||
|
||||
self.audio_config = get_config(value, trust_remote_code=False)
|
||||
|
||||
return super().__setattr__(key, value)
|
||||
|
||||
@property
|
||||
def text_config(self) -> transformers.PreTrainedConfig:
|
||||
# When Ultravox wraps a multi-modal model (e.g. Gemma), we instantiate
|
||||
# the full model, but the text config is the text config of the inner
|
||||
# model.
|
||||
return self.wrapped_model_config.get_text_config()
|
||||
@@ -1,35 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
# Unlimited-OCR (baidu/Unlimited-OCR) reuses
|
||||
# the DeepSeek-OCR multimodal layout (DeepEncoder = SAM-ViT-B + CLIP-L, a linear
|
||||
# MLP projector and a DeepSeek-V2 text backbone). The only architectural
|
||||
# difference is the language model, which is a DeepSeek-V2 *MoE* with plain
|
||||
# multi-head attention (``use_mla=False``) instead of the dense MLA backbone.
|
||||
# We therefore reuse ``DeepseekVLV2Config`` for parsing the nested config.
|
||||
|
||||
from vllm.transformers_utils.configs.deepseek_vl2 import DeepseekVLV2Config
|
||||
|
||||
|
||||
class UnlimitedOCRConfig(DeepseekVLV2Config):
|
||||
model_type = "unlimited-ocr"
|
||||
|
||||
# An explicit ``__init__`` is required: Transformers v5 processes each
|
||||
# concrete config class' ``__init__`` signature to build nested sub-configs,
|
||||
# and an empty subclass (only overriding ``model_type``) would skip
|
||||
# ``DeepseekVLV2Config.__init__``, leaving ``text_config`` unset.
|
||||
def __init__(
|
||||
self,
|
||||
tile_tag: str = "2D",
|
||||
global_view_pos: str = "head",
|
||||
candidate_resolutions: tuple[tuple[int, int]] = ((384, 384),),
|
||||
rswa_window: int = 128,
|
||||
**kwargs,
|
||||
):
|
||||
super().__init__(
|
||||
tile_tag=tile_tag,
|
||||
global_view_pos=global_view_pos,
|
||||
candidate_resolutions=candidate_resolutions,
|
||||
**kwargs,
|
||||
)
|
||||
self.rswa_window = rswa_window
|
||||
@@ -41,37 +41,7 @@ __all__ = [
|
||||
"Step3VLProcessor",
|
||||
]
|
||||
|
||||
_CLASS_TO_MODULE: dict[str, str] = {
|
||||
"BagelProcessor": "vllm.transformers_utils.processors.bagel",
|
||||
"CohereASRProcessor": "vllm.transformers_utils.processors.cohere_asr",
|
||||
"Cosmos3EdgeProcessor": "vllm.transformers_utils.processors.cosmos3_edge",
|
||||
"DeepseekVLV2Processor": "vllm.transformers_utils.processors.deepseek_vl2",
|
||||
"FireRedASR2Processor": "vllm.transformers_utils.processors.fireredasr2",
|
||||
"FunASRProcessor": "vllm.transformers_utils.processors.funasr",
|
||||
"GLM4VProcessor": "vllm.transformers_utils.processors.glm4v",
|
||||
"H2OVLProcessor": "vllm.transformers_utils.processors.h2ovl",
|
||||
"IsaacProcessor": "vllm.transformers_utils.processors.isaac",
|
||||
"KimiAudioProcessor": "vllm.transformers_utils.processors.kimi_audio",
|
||||
"KimiK25Processor": "vllm.transformers_utils.processors.kimi_k25",
|
||||
"KimiK3Processor": "vllm.transformers_utils.processors.kimi_k3",
|
||||
"MiMoOmniProcessor": "vllm.transformers_utils.processors.mimo_v2_omni",
|
||||
"MiniCPMOProcessor": "vllm.transformers_utils.processors.minicpmo",
|
||||
"MiniCPMVProcessor": "vllm.transformers_utils.processors.minicpmv",
|
||||
"MiniMaxM3VLImageProcessor": "vllm.transformers_utils.processors.minimax_m3",
|
||||
"MiniMaxM3VLVideoProcessor": "vllm.transformers_utils.processors.minimax_m3",
|
||||
"MiniMaxVLProcessor": "vllm.transformers_utils.processors.minimax_m3",
|
||||
"MistralCommonPixtralProcessor": "vllm.transformers_utils.processors.pixtral",
|
||||
"MistralCommonVoxtralProcessor": "vllm.transformers_utils.processors.voxtral",
|
||||
"Moondream3Processor": "vllm.transformers_utils.processors.moondream3",
|
||||
"NanoNemotronVLProcessor": "vllm.transformers_utils.processors.nano_nemotron_vl",
|
||||
"NemotronVLProcessor": "vllm.transformers_utils.processors.nemotron_vl",
|
||||
"LlamaNemotronVLEmbedProcessor": "vllm.transformers_utils.processors.nemotron_vl",
|
||||
"NVLMProcessor": "vllm.transformers_utils.processors.nvlm_d",
|
||||
"OpenVLAProcessor": "vllm.transformers_utils.processors.openvla",
|
||||
"OvisProcessor": "vllm.transformers_utils.processors.ovis",
|
||||
"Ovis2_5Processor": "vllm.transformers_utils.processors.ovis2_5",
|
||||
"Step3VLProcessor": "vllm.transformers_utils.processors.step3_vl",
|
||||
}
|
||||
_CLASS_TO_MODULE: dict[str, str] = {}
|
||||
|
||||
|
||||
def __getattr__(name: str):
|
||||
|
||||
@@ -1,73 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
# Copyright 2025 Bytedance Ltd. and/or its affiliates.
|
||||
"""BAGEL processor for image and text inputs."""
|
||||
|
||||
from transformers.feature_extraction_utils import BatchFeature
|
||||
from transformers.image_utils import ImageInput
|
||||
from transformers.processing_utils import ProcessingKwargs, ProcessorMixin, Unpack
|
||||
from transformers.tokenization_utils_base import PreTokenizedInput, TextInput
|
||||
|
||||
|
||||
class BagelProcessorKwargs(ProcessingKwargs, total=False): # type: ignore[call-arg]
|
||||
_defaults = {
|
||||
"images_kwargs": {
|
||||
"return_tensors": "pt",
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
class BagelProcessor(ProcessorMixin):
|
||||
"""Constructs a BAGEL processor which wraps a
|
||||
SigLIP image processor and a Qwen2 tokenizer.
|
||||
"""
|
||||
|
||||
attributes = ["image_processor", "tokenizer"]
|
||||
image_processor_class = "SiglipImageProcessor"
|
||||
tokenizer_class = "AutoTokenizer"
|
||||
|
||||
def __call__(
|
||||
self,
|
||||
text: TextInput
|
||||
| PreTokenizedInput
|
||||
| list[TextInput]
|
||||
| list[PreTokenizedInput] = None,
|
||||
images: ImageInput = None,
|
||||
**kwargs: Unpack[BagelProcessorKwargs],
|
||||
):
|
||||
"""Prepare one or more sequences and images for the model."""
|
||||
output_kwargs = self._merge_kwargs(
|
||||
BagelProcessorKwargs,
|
||||
tokenizer_init_kwargs=self.tokenizer.init_kwargs,
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
if images is not None:
|
||||
# Process images with the image processor
|
||||
pixel_values = self.image_processor(
|
||||
images, **output_kwargs["images_kwargs"]
|
||||
)
|
||||
else:
|
||||
pixel_values = {}
|
||||
|
||||
text_inputs = (
|
||||
self.tokenizer(text, **output_kwargs["text_kwargs"])
|
||||
if text is not None
|
||||
else {}
|
||||
)
|
||||
|
||||
return BatchFeature(data={**pixel_values, **text_inputs})
|
||||
|
||||
def batch_decode(self, *args, **kwargs):
|
||||
"""Forward all arguments to `Qwen2TokenizerFast.batch_decode`."""
|
||||
return self.tokenizer.batch_decode(*args, **kwargs)
|
||||
|
||||
def decode(self, *args, **kwargs):
|
||||
"""This method forwards all its arguments to Qwen2TokenizerFast's decode."""
|
||||
return self.tokenizer.decode(*args, **kwargs)
|
||||
|
||||
@property
|
||||
def model_input_names(self):
|
||||
tokenizer_input_names = self.tokenizer.model_input_names
|
||||
image_processor_input_names = self.image_processor.model_input_names
|
||||
return list(dict.fromkeys(tokenizer_input_names + image_processor_input_names))
|
||||
@@ -1,573 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import logging
|
||||
import math
|
||||
import random
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
from torch import nn
|
||||
from torchaudio.functional import melscale_fbanks
|
||||
from transformers import AutoFeatureExtractor, AutoProcessor, BatchFeature
|
||||
from transformers.feature_extraction_sequence_utils import (
|
||||
SequenceFeatureExtractor,
|
||||
)
|
||||
from transformers.processing_utils import ProcessorMixin
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
CONSTANT = 1e-5
|
||||
INF_VAL = 10000.0
|
||||
|
||||
|
||||
class FilterbankFeatures(nn.Module):
|
||||
"""Featurizer that converts wavs to Mel Spectrograms.
|
||||
See AudioToMelSpectrogramPreprocessor for args.
|
||||
"""
|
||||
|
||||
window: torch.Tensor
|
||||
fb: torch.Tensor
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
sample_rate=16000,
|
||||
n_window_size=320,
|
||||
n_window_stride=160,
|
||||
window="hann",
|
||||
normalize="per_feature",
|
||||
n_fft=None,
|
||||
preemph=0.97,
|
||||
nfilt=64,
|
||||
lowfreq=0,
|
||||
highfreq=None,
|
||||
log=True,
|
||||
log_zero_guard_type="add",
|
||||
log_zero_guard_value=2**-24,
|
||||
dither=CONSTANT,
|
||||
pad_to=16,
|
||||
max_duration=30,
|
||||
frame_splicing=1,
|
||||
exact_pad=False,
|
||||
pad_value=0,
|
||||
mag_power=2.0,
|
||||
use_grads=False,
|
||||
rng=None,
|
||||
nb_augmentation_prob=0.0,
|
||||
nb_max_freq=4000,
|
||||
mel_norm="slaney",
|
||||
stft_exact_pad=False,
|
||||
stft_conv=False,
|
||||
device="cpu",
|
||||
):
|
||||
super().__init__()
|
||||
if stft_conv or stft_exact_pad:
|
||||
logger.warning(
|
||||
"Using torch_stft is deprecated and has been removed. "
|
||||
"The values have been forcibly set to False for "
|
||||
"FilterbankFeatures and AudioToMelSpectrogramPreprocessor. "
|
||||
"Please set exact_pad to True as needed."
|
||||
)
|
||||
if exact_pad and n_window_stride % 2 == 1:
|
||||
raise NotImplementedError(
|
||||
f"{self} received exact_pad == True, but hop_size was odd. "
|
||||
"If audio_length % hop_size == 0, the returned spectrogram "
|
||||
"would not be of length audio_length // hop_size. "
|
||||
"Please use an even hop_size."
|
||||
)
|
||||
self.log_zero_guard_value = log_zero_guard_value
|
||||
if (
|
||||
n_window_size is None
|
||||
or n_window_stride is None
|
||||
or not isinstance(n_window_size, int)
|
||||
or not isinstance(n_window_stride, int)
|
||||
or n_window_size <= 0
|
||||
or n_window_stride <= 0
|
||||
):
|
||||
raise ValueError(
|
||||
f"{self} got an invalid value for either n_window_size or "
|
||||
f"n_window_stride. Both must be positive ints."
|
||||
)
|
||||
|
||||
self.sample_rate = sample_rate
|
||||
self.win_length = n_window_size
|
||||
self.hop_length = n_window_stride
|
||||
self.n_fft = n_fft or 2 ** math.ceil(math.log2(self.win_length))
|
||||
self.stft_pad_amount = (
|
||||
(self.n_fft - self.hop_length) // 2 if exact_pad else None
|
||||
)
|
||||
self.exact_pad = exact_pad
|
||||
self.sample_rate = sample_rate
|
||||
self.max_duration = max_duration
|
||||
|
||||
if exact_pad:
|
||||
logger.info("STFT using exact pad")
|
||||
torch_windows = {
|
||||
"hann": torch.hann_window,
|
||||
"hamming": torch.hamming_window,
|
||||
"blackman": torch.blackman_window,
|
||||
"bartlett": torch.bartlett_window,
|
||||
"none": None,
|
||||
}
|
||||
window_fn = torch_windows.get(window)
|
||||
window_tensor = (
|
||||
window_fn(self.win_length, periodic=False) if window_fn else None
|
||||
)
|
||||
self.register_buffer("window", window_tensor)
|
||||
|
||||
self.normalize = normalize
|
||||
self.log = log
|
||||
self.dither = dither
|
||||
self.frame_splicing = frame_splicing
|
||||
self.nfilt = nfilt
|
||||
self.preemph = preemph
|
||||
self.pad_to = pad_to
|
||||
highfreq = highfreq or sample_rate / 2
|
||||
self.sample_rate = sample_rate
|
||||
# disable pad min duration
|
||||
# self.pad_min_duration = 1.0
|
||||
self.pad_min_duration = 0.0
|
||||
self.pad_direction = "both"
|
||||
|
||||
filterbanks = melscale_fbanks(
|
||||
n_freqs=self.n_fft // 2 + 1,
|
||||
f_min=lowfreq,
|
||||
f_max=highfreq,
|
||||
n_mels=nfilt,
|
||||
sample_rate=sample_rate,
|
||||
norm=mel_norm,
|
||||
mel_scale="slaney",
|
||||
).T.unsqueeze(0)
|
||||
self.register_buffer("fb", filterbanks)
|
||||
|
||||
# Calculate maximum sequence length
|
||||
max_length = self.get_seq_len(
|
||||
torch.tensor(max_duration * sample_rate, dtype=torch.float)
|
||||
)
|
||||
max_pad = pad_to - (max_length % pad_to) if pad_to > 0 else 0
|
||||
self.max_length = max_length + max_pad
|
||||
self.pad_value = pad_value
|
||||
self.mag_power = mag_power
|
||||
|
||||
# We want to avoid taking the log of zero
|
||||
# There are two options: either adding or clamping to a small value
|
||||
if log_zero_guard_type not in ["add", "clamp"]:
|
||||
raise ValueError(
|
||||
f"{self} received {log_zero_guard_type} for the "
|
||||
f"log_zero_guard_type parameter. It must be either 'add' or "
|
||||
f"'clamp'."
|
||||
)
|
||||
|
||||
self.use_grads = use_grads
|
||||
if not use_grads:
|
||||
self.forward = torch.no_grad()(self.forward)
|
||||
self._rng = random.Random() if rng is None else rng
|
||||
self.nb_augmentation_prob = nb_augmentation_prob
|
||||
if self.nb_augmentation_prob > 0.0:
|
||||
if nb_max_freq >= sample_rate / 2:
|
||||
self.nb_augmentation_prob = 0.0
|
||||
else:
|
||||
self._nb_max_fft_bin = int((nb_max_freq / sample_rate) * n_fft)
|
||||
|
||||
# log_zero_guard_value is the the small we want to use, we support
|
||||
# an actual number, or "tiny", or "eps"
|
||||
self.log_zero_guard_type = log_zero_guard_type
|
||||
|
||||
assert self.window is not None
|
||||
assert self.fb is not None
|
||||
self.window = self.window.to(dtype=torch.bfloat16)
|
||||
self.fb = self.fb.to(dtype=torch.bfloat16)
|
||||
|
||||
self.generator = torch.Generator(device=device)
|
||||
self.generator.manual_seed(0)
|
||||
|
||||
@torch._dynamo.disable
|
||||
def stft(self, x):
|
||||
# disable autocast to get full range of stft values
|
||||
with torch.amp.autocast(x.device.type, enabled=False):
|
||||
return torch.stft(
|
||||
x,
|
||||
n_fft=self.n_fft,
|
||||
hop_length=self.hop_length,
|
||||
win_length=self.win_length,
|
||||
center=not self.exact_pad,
|
||||
window=self.window.to(dtype=torch.float, device=x.device),
|
||||
return_complex=True,
|
||||
pad_mode="constant",
|
||||
)
|
||||
|
||||
def log_zero_guard_value_fn(self, x):
|
||||
if isinstance(self.log_zero_guard_value, str):
|
||||
if self.log_zero_guard_value == "tiny":
|
||||
return torch.finfo(x.dtype).tiny
|
||||
elif self.log_zero_guard_value == "eps":
|
||||
return torch.finfo(x.dtype).eps
|
||||
else:
|
||||
raise ValueError(
|
||||
f"{self} received {self.log_zero_guard_value} for the "
|
||||
f"log_zero_guard_type parameter. It must be either a "
|
||||
f"number, 'tiny', or 'eps'"
|
||||
)
|
||||
else:
|
||||
return self.log_zero_guard_value
|
||||
|
||||
def get_seq_len(self, seq_len):
|
||||
# Assuming that center is True is stft_pad_amount = 0
|
||||
pad_amount = (
|
||||
self.stft_pad_amount * 2
|
||||
if self.stft_pad_amount is not None
|
||||
else self.n_fft // 2 * 2
|
||||
)
|
||||
seq_len = torch.floor_divide(
|
||||
(seq_len + pad_amount - self.n_fft), self.hop_length
|
||||
)
|
||||
return seq_len.to(dtype=torch.long)
|
||||
|
||||
@property
|
||||
def filter_banks(self):
|
||||
return self.fb
|
||||
|
||||
def splice_frames(self, x, frame_splicing):
|
||||
"""Stacks frames together across feature dim.
|
||||
|
||||
input is batch_size, feature_dim, num_frames
|
||||
output is batch_size, feature_dim*frame_splicing, num_frames
|
||||
|
||||
"""
|
||||
seq = [x]
|
||||
for n in range(1, frame_splicing):
|
||||
seq.append(torch.cat([x[:, :, :n], x[:, :, n:]], dim=2))
|
||||
return torch.cat(seq, dim=1)
|
||||
|
||||
def normalize_batch(self, x, seq_len, normalize_type):
|
||||
x_mean = None
|
||||
x_std = None
|
||||
if normalize_type == "per_feature":
|
||||
batch_size = x.shape[0]
|
||||
max_time = x.shape[2]
|
||||
|
||||
# When doing stream capture to a graph, item() is not allowed
|
||||
# because it calls cudaStreamSynchronize(). Therefore, we are
|
||||
# sacrificing some error checking when running with cuda graphs.
|
||||
# if (
|
||||
# torch.cuda.is_available()
|
||||
# and not torch.cuda.is_current_stream_capturing()
|
||||
# and torch.any(seq_len == 1).item()
|
||||
# ):
|
||||
# raise ValueError(
|
||||
# "normalize_batch with `per_feature` normalize_type "
|
||||
# "received a tensor of length 1. This will result in "
|
||||
# "torch.std() returning nan. Make sure your audio length "
|
||||
# "has enough samples for a single feature (ex. at least "
|
||||
# "`hop_length` for Mel Spectrograms)."
|
||||
# )
|
||||
time_steps = (
|
||||
torch.arange(max_time, device=x.device)
|
||||
.unsqueeze(0)
|
||||
.expand(batch_size, max_time)
|
||||
)
|
||||
valid_mask = time_steps < seq_len.unsqueeze(1)
|
||||
x_mean_numerator = torch.where(valid_mask.unsqueeze(1), x, 0.0).sum(axis=2)
|
||||
x_mean_denominator = valid_mask.sum(axis=1)
|
||||
x_mean = x_mean_numerator / x_mean_denominator.unsqueeze(1)
|
||||
|
||||
# Subtract 1 in the denominator to correct for the bias.
|
||||
x_std = torch.sqrt(
|
||||
torch.sum(
|
||||
torch.where(valid_mask.unsqueeze(1), x - x_mean.unsqueeze(2), 0.0)
|
||||
** 2,
|
||||
axis=2,
|
||||
)
|
||||
/ (x_mean_denominator.unsqueeze(1) - 1.0)
|
||||
)
|
||||
x_std = x_std.masked_fill(
|
||||
x_std.isnan(), 0.0
|
||||
) # edge case: only 1 frame in denominator
|
||||
# make sure x_std is not zero
|
||||
x_std += CONSTANT
|
||||
return (x - x_mean.unsqueeze(2)) / x_std.unsqueeze(2), x_mean, x_std
|
||||
elif normalize_type == "all_features":
|
||||
x_mean = torch.zeros(seq_len.shape, dtype=x.dtype, device=x.device)
|
||||
x_std = torch.zeros(seq_len.shape, dtype=x.dtype, device=x.device)
|
||||
for i in range(x.shape[0]):
|
||||
x_mean[i] = x[i, :, : seq_len[i].item()].mean()
|
||||
x_std[i] = x[i, :, : seq_len[i].item()].std()
|
||||
# make sure x_std is not zero
|
||||
x_std += CONSTANT
|
||||
return (x - x_mean.view(-1, 1, 1)) / x_std.view(-1, 1, 1), x_mean, x_std
|
||||
elif "fixed_mean" in normalize_type and "fixed_std" in normalize_type:
|
||||
x_mean = torch.tensor(normalize_type["fixed_mean"], device=x.device)
|
||||
x_std = torch.tensor(normalize_type["fixed_std"], device=x.device)
|
||||
return (
|
||||
(x - x_mean.view(x.shape[0], x.shape[1]).unsqueeze(2))
|
||||
/ x_std.view(x.shape[0], x.shape[1]).unsqueeze(2),
|
||||
x_mean,
|
||||
x_std,
|
||||
)
|
||||
else:
|
||||
return x, x_mean, x_std
|
||||
|
||||
@torch.compile
|
||||
def forward(self, x, seq_len, linear_spec=False):
|
||||
if x.shape[1] < self.sample_rate * self.pad_min_duration:
|
||||
pad_amount = int(self.sample_rate * self.pad_min_duration) - x.shape[1]
|
||||
if self.pad_direction == "right":
|
||||
x = F.pad(x, (0, pad_amount), value=self.pad_value)
|
||||
elif self.pad_direction == "left":
|
||||
x = F.pad(x, (pad_amount, 0), value=self.pad_value)
|
||||
elif self.pad_direction == "both":
|
||||
left_pad = pad_amount // 2
|
||||
right_pad = pad_amount - left_pad
|
||||
x = F.pad(x, (left_pad, right_pad), value=self.pad_value)
|
||||
else:
|
||||
raise ValueError(
|
||||
f"{self} received an invalid pad_direction: {self.pad_direction}. "
|
||||
f"It must be one of 'left', 'right', or 'both'."
|
||||
)
|
||||
seq_len = torch.tensor([x.shape[1]], dtype=torch.float, device=x.device)
|
||||
|
||||
seq_len_time = seq_len
|
||||
seq_len_unfixed = self.get_seq_len(seq_len)
|
||||
|
||||
# fix for seq_len = 0 for streaming; if size was 0, it is always padded
|
||||
# to 1, and normalizer fails
|
||||
seq_len = torch.where(
|
||||
seq_len == 0, torch.zeros_like(seq_len_unfixed), seq_len_unfixed
|
||||
)
|
||||
|
||||
if self.stft_pad_amount is not None:
|
||||
x = torch.nn.functional.pad(
|
||||
x.unsqueeze(1), (self.stft_pad_amount, self.stft_pad_amount), "constant"
|
||||
).squeeze(1)
|
||||
|
||||
# use dither for inference as well
|
||||
if self.dither > 0:
|
||||
x += self.dither * torch.randn(
|
||||
x.shape, dtype=x.dtype, device=x.device, generator=self.generator
|
||||
)
|
||||
|
||||
# do preemphasis
|
||||
if self.preemph is not None:
|
||||
timemask = torch.arange(x.shape[1], device=x.device).unsqueeze(
|
||||
0
|
||||
) < seq_len_time.unsqueeze(1)
|
||||
x = torch.cat(
|
||||
(x[:, 0].unsqueeze(1), x[:, 1:] - self.preemph * x[:, :-1]), dim=1
|
||||
)
|
||||
|
||||
x = x.masked_fill(~timemask, 0.0)
|
||||
|
||||
x = self.stft(x)
|
||||
|
||||
# torch stft returns complex tensor (of shape [B,N,T]); so convert to magnitude
|
||||
# guard is needed for sqrt if grads are passed through
|
||||
guard = 0 if not self.use_grads else CONSTANT
|
||||
x = torch.view_as_real(x)
|
||||
x = torch.sqrt(x.pow(2).sum(-1) + guard)
|
||||
|
||||
# get power spectrum
|
||||
if self.mag_power != 1.0:
|
||||
x = x.pow(self.mag_power)
|
||||
|
||||
# return plain spectrogram if required
|
||||
if linear_spec:
|
||||
return x, seq_len
|
||||
|
||||
# disable autocast, otherwise it might be automatically casted to fp16
|
||||
# on fp16 compatible GPUs and get NaN values for input value of 65520
|
||||
with torch.amp.autocast(x.device.type, enabled=False):
|
||||
# dot with filterbank energies
|
||||
x = torch.matmul(self.fb.to(x.dtype), x)
|
||||
|
||||
# log features if required
|
||||
if self.log:
|
||||
if self.log_zero_guard_type == "add":
|
||||
x = torch.log(x + self.log_zero_guard_value_fn(x))
|
||||
elif self.log_zero_guard_type == "clamp":
|
||||
x = torch.log(torch.clamp(x, min=self.log_zero_guard_value_fn(x)))
|
||||
else:
|
||||
raise ValueError("log_zero_guard_type was not understood")
|
||||
|
||||
# frame splicing if required
|
||||
if self.frame_splicing > 1:
|
||||
x = self.splice_frames(x, self.frame_splicing)
|
||||
|
||||
# normalize if required
|
||||
if self.normalize:
|
||||
x, _, _ = self.normalize_batch(x, seq_len, normalize_type=self.normalize)
|
||||
|
||||
# mask to zero any values beyond seq_len in batch, pad to multiple of
|
||||
# `pad_to` (for efficiency)
|
||||
max_len = x.size(-1)
|
||||
mask = torch.arange(max_len, device=x.device)
|
||||
mask = mask.repeat(x.size(0), 1) >= seq_len.unsqueeze(1)
|
||||
x = x.masked_fill(
|
||||
mask.unsqueeze(1).type(torch.bool).to(device=x.device), self.pad_value
|
||||
)
|
||||
|
||||
del mask
|
||||
pad_to = self.pad_to
|
||||
if pad_to == "max":
|
||||
x = nn.functional.pad(
|
||||
x, (0, self.max_length - x.size(-1)), value=self.pad_value
|
||||
)
|
||||
elif pad_to > 0:
|
||||
pad_amt = x.size(-1) % pad_to
|
||||
if pad_amt != 0:
|
||||
x = nn.functional.pad(x, (0, pad_to - pad_amt), value=self.pad_value)
|
||||
|
||||
return x, seq_len
|
||||
|
||||
|
||||
class CohereASRFeatureExtractor(SequenceFeatureExtractor):
|
||||
"""HF-compatible feature extractor wrapping FilterbankFeatures."""
|
||||
|
||||
model_input_names = ["input_features"]
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
feature_size=64,
|
||||
sampling_rate=16000,
|
||||
padding_value=0.0,
|
||||
max_duration=30,
|
||||
n_window_size=320,
|
||||
n_window_stride=160,
|
||||
window="hann",
|
||||
normalize="per_feature",
|
||||
n_fft=None,
|
||||
preemph=0.97,
|
||||
lowfreq=0,
|
||||
highfreq=None,
|
||||
log=True,
|
||||
log_zero_guard_type="add",
|
||||
log_zero_guard_value=2**-24,
|
||||
dither=CONSTANT,
|
||||
pad_to=16,
|
||||
frame_splicing=1,
|
||||
exact_pad=False,
|
||||
mag_power=2.0,
|
||||
nb_augmentation_prob=0.0,
|
||||
nb_max_freq=4000,
|
||||
mel_norm="slaney",
|
||||
stft_exact_pad=False,
|
||||
stft_conv=False,
|
||||
device="cpu",
|
||||
**kwargs,
|
||||
):
|
||||
super().__init__(
|
||||
feature_size=feature_size,
|
||||
sampling_rate=sampling_rate,
|
||||
padding_value=padding_value,
|
||||
**kwargs,
|
||||
)
|
||||
self.max_duration = max_duration
|
||||
self.hop_length = n_window_stride
|
||||
self._device = torch.device(device)
|
||||
self._fb_config = dict(
|
||||
sample_rate=sampling_rate,
|
||||
n_window_size=n_window_size,
|
||||
n_window_stride=n_window_stride,
|
||||
window=window,
|
||||
normalize=normalize,
|
||||
n_fft=n_fft,
|
||||
preemph=preemph,
|
||||
nfilt=feature_size,
|
||||
lowfreq=lowfreq,
|
||||
highfreq=highfreq,
|
||||
log=log,
|
||||
log_zero_guard_type=log_zero_guard_type,
|
||||
log_zero_guard_value=log_zero_guard_value,
|
||||
dither=dither,
|
||||
pad_to=pad_to,
|
||||
max_duration=max_duration,
|
||||
frame_splicing=frame_splicing,
|
||||
exact_pad=exact_pad,
|
||||
pad_value=padding_value,
|
||||
mag_power=mag_power,
|
||||
nb_augmentation_prob=nb_augmentation_prob,
|
||||
nb_max_freq=nb_max_freq,
|
||||
mel_norm=mel_norm,
|
||||
stft_exact_pad=stft_exact_pad,
|
||||
stft_conv=stft_conv,
|
||||
device=device,
|
||||
)
|
||||
self._filterbank: FilterbankFeatures | None = None
|
||||
|
||||
@property
|
||||
def filterbank(self) -> FilterbankFeatures:
|
||||
if self._filterbank is None:
|
||||
fb = FilterbankFeatures(**self._fb_config)
|
||||
fb.eval()
|
||||
self._filterbank = fb.to(self._device)
|
||||
return self._filterbank
|
||||
|
||||
def get_seq_len(self, seq_len):
|
||||
return self.filterbank.get_seq_len(seq_len)
|
||||
|
||||
def __call__(
|
||||
self,
|
||||
raw_speech,
|
||||
sampling_rate=None,
|
||||
return_tensors=None,
|
||||
**kwargs,
|
||||
) -> BatchFeature:
|
||||
if isinstance(raw_speech, np.ndarray):
|
||||
raw_speech = [raw_speech]
|
||||
|
||||
seq_len = torch.tensor([s.shape[0] for s in raw_speech])
|
||||
|
||||
max_len = max(s.shape[0] for s in raw_speech)
|
||||
padded = np.zeros((len(raw_speech), max_len), dtype=np.float32)
|
||||
for i, s in enumerate(raw_speech):
|
||||
padded[i, : s.shape[0]] = s
|
||||
|
||||
audio_tensor = torch.from_numpy(padded).to(self._device)
|
||||
seq_len = seq_len.to(self._device)
|
||||
|
||||
with torch.no_grad():
|
||||
input_features, length = self.filterbank(audio_tensor, seq_len)
|
||||
|
||||
result = BatchFeature(
|
||||
{"input_features": input_features.cpu(), "length": length.cpu()}
|
||||
)
|
||||
if return_tensors is not None:
|
||||
result = result.convert_to_tensors(return_tensors)
|
||||
return result
|
||||
|
||||
|
||||
class CohereASRProcessor(ProcessorMixin):
|
||||
"""HF-compatible processor combining CohereASRFeatureExtractor and a
|
||||
tokenizer."""
|
||||
|
||||
feature_extractor_class = "CohereASRFeatureExtractor"
|
||||
tokenizer_class = "AutoTokenizer"
|
||||
|
||||
def __init__(self, feature_extractor, tokenizer):
|
||||
super().__init__(feature_extractor, tokenizer)
|
||||
|
||||
def __call__(
|
||||
self,
|
||||
text=None,
|
||||
audio=None,
|
||||
sampling_rate=None,
|
||||
return_tensors=None,
|
||||
**kwargs,
|
||||
):
|
||||
if audio is not None:
|
||||
result = self.feature_extractor(
|
||||
audio,
|
||||
sampling_rate=sampling_rate,
|
||||
return_tensors=return_tensors,
|
||||
)
|
||||
else:
|
||||
result = BatchFeature()
|
||||
|
||||
if text is not None:
|
||||
text_inputs = self.tokenizer(text, return_tensors=return_tensors, **kwargs)
|
||||
result["input_ids"] = text_inputs["input_ids"]
|
||||
|
||||
return result
|
||||
|
||||
|
||||
AutoFeatureExtractor.register("CohereASRFeatureExtractor", CohereASRFeatureExtractor)
|
||||
AutoProcessor.register("CohereASRProcessor", CohereASRProcessor)
|
||||
@@ -1,404 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
# ruff: noqa: E501
|
||||
# coding=utf-8
|
||||
# adapted from https://github.com/deepseek-ai/DeepSeek-VL2/blob/ff23960c5cf9e6874b44be38af930cfb0ccbb620/deepseek_vl2/models/processing_deepseek_vl_v2.py
|
||||
# Copyright (c) 2023-2024 DeepSeek.
|
||||
#
|
||||
# Permission is hereby granted, free of charge, to any person obtaining a copy of
|
||||
# this software and associated documentation files (the "Software"), to deal in
|
||||
# the Software without restriction, including without limitation the rights to
|
||||
# use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
|
||||
# the Software, and to permit persons to whom the Software is furnished to do so,
|
||||
# subject to the following conditions:
|
||||
#
|
||||
# The above copyright notice and this permission notice shall be included in all
|
||||
# copies or substantial portions of the Software.
|
||||
#
|
||||
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
|
||||
# FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
|
||||
# COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
|
||||
# IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
|
||||
# CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
|
||||
import math
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
import torchvision.transforms as T
|
||||
from PIL import Image, ImageOps
|
||||
from transformers import BatchFeature, LlamaTokenizerFast
|
||||
from transformers.processing_utils import ProcessorMixin
|
||||
|
||||
|
||||
class ImageTransform:
|
||||
def __init__(
|
||||
self,
|
||||
mean: tuple[float, float, float] = (0.5, 0.5, 0.5),
|
||||
std: tuple[float, float, float] = (0.5, 0.5, 0.5),
|
||||
normalize: bool = True,
|
||||
):
|
||||
self.mean = mean
|
||||
self.std = std
|
||||
self.normalize = normalize
|
||||
|
||||
transform_pipelines = [T.ToTensor()]
|
||||
|
||||
if normalize:
|
||||
transform_pipelines.append(T.Normalize(mean, std))
|
||||
|
||||
self.transform = T.Compose(transform_pipelines)
|
||||
|
||||
def __call__(self, pil_img: Image.Image):
|
||||
x = self.transform(pil_img)
|
||||
return x
|
||||
|
||||
|
||||
class DeepseekVLV2Processor(ProcessorMixin):
|
||||
tokenizer_class = ("LlamaTokenizer", "LlamaTokenizerFast")
|
||||
attributes = ["tokenizer"]
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
tokenizer: LlamaTokenizerFast,
|
||||
candidate_resolutions: tuple[tuple[int, int]],
|
||||
patch_size: int,
|
||||
downsample_ratio: int,
|
||||
image_mean: tuple[float, float, float] = (0.5, 0.5, 0.5),
|
||||
image_std: tuple[float, float, float] = (0.5, 0.5, 0.5),
|
||||
normalize: bool = True,
|
||||
image_token: str = "<image>",
|
||||
pad_token: str = "<|▁pad▁|>",
|
||||
add_special_token: bool = False,
|
||||
sft_format: str = "deepseek",
|
||||
mask_prompt: bool = True,
|
||||
ignore_id: int = -100,
|
||||
**kwargs,
|
||||
):
|
||||
self.candidate_resolutions = candidate_resolutions
|
||||
self.image_size = candidate_resolutions[0][0]
|
||||
self.patch_size = patch_size
|
||||
self.image_mean = image_mean
|
||||
self.image_std = image_std
|
||||
self.normalize = normalize
|
||||
self.downsample_ratio = downsample_ratio
|
||||
|
||||
self.image_transform = ImageTransform(
|
||||
mean=image_mean, std=image_std, normalize=normalize
|
||||
)
|
||||
self.tokenizer = tokenizer
|
||||
self.tokenizer.padding_side = "left" # must set this,padding side with make a difference in batch inference
|
||||
|
||||
# add the pad_token as special token to use 'tokenizer.pad_token' and 'tokenizer.pad_token_id'
|
||||
if tokenizer.pad_token is None:
|
||||
self.tokenizer.add_special_tokens({"pad_token": pad_token})
|
||||
|
||||
# add image token
|
||||
image_token_id = self.tokenizer.vocab.get(image_token)
|
||||
if image_token_id is None:
|
||||
special_tokens = [image_token]
|
||||
special_tokens_dict = {"additional_special_tokens": special_tokens}
|
||||
self.tokenizer.add_special_tokens(special_tokens_dict)
|
||||
self.image_token_id = self.tokenizer.vocab.get(image_token)
|
||||
|
||||
# add five special tokens for grounding-related tasks
|
||||
# <|ref|>, <|/ref|>, <|det|>, <|/det|>, <|grounding|>
|
||||
special_tokens = ["<|ref|>", "<|/ref|>", "<|det|>", "<|/det|>", "<|grounding|>"]
|
||||
special_tokens_dict = {"additional_special_tokens": special_tokens}
|
||||
self.tokenizer.add_special_tokens(special_tokens_dict)
|
||||
|
||||
# add special tokens for SFT data
|
||||
special_tokens = ["<|User|>", "<|Assistant|>"]
|
||||
special_tokens_dict = {"additional_special_tokens": special_tokens}
|
||||
self.tokenizer.add_special_tokens(special_tokens_dict)
|
||||
|
||||
self.image_token = image_token
|
||||
self.pad_token = pad_token
|
||||
self.add_special_token = add_special_token
|
||||
self.sft_format = sft_format
|
||||
self.mask_prompt = mask_prompt
|
||||
self.ignore_id = ignore_id
|
||||
|
||||
super().__init__(
|
||||
tokenizer,
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
def select_best_resolution(self, image_size):
|
||||
# used for cropping
|
||||
original_width, original_height = image_size
|
||||
best_fit = None
|
||||
max_effective_resolution = 0
|
||||
min_wasted_resolution = float("inf")
|
||||
|
||||
for width, height in self.candidate_resolutions:
|
||||
scale = min(width / original_width, height / original_height)
|
||||
downscaled_width, downscaled_height = (
|
||||
int(original_width * scale),
|
||||
int(original_height * scale),
|
||||
)
|
||||
effective_resolution = min(
|
||||
downscaled_width * downscaled_height, original_width * original_height
|
||||
)
|
||||
wasted_resolution = (width * height) - effective_resolution
|
||||
|
||||
if effective_resolution > max_effective_resolution or (
|
||||
effective_resolution == max_effective_resolution
|
||||
and wasted_resolution < min_wasted_resolution
|
||||
):
|
||||
max_effective_resolution = effective_resolution
|
||||
min_wasted_resolution = wasted_resolution
|
||||
best_fit = (width, height)
|
||||
|
||||
return best_fit
|
||||
|
||||
@property
|
||||
def bos_id(self):
|
||||
return self.tokenizer.bos_token_id
|
||||
|
||||
@property
|
||||
def eos_id(self):
|
||||
return self.tokenizer.eos_token_id
|
||||
|
||||
@property
|
||||
def pad_id(self):
|
||||
return self.tokenizer.pad_token_id
|
||||
|
||||
def encode(self, text: str, bos: bool = True, eos: bool = False):
|
||||
t = self.tokenizer.encode(text, add_special_tokens=False)
|
||||
|
||||
if bos:
|
||||
t = [self.bos_id] + t
|
||||
if eos:
|
||||
t = t + [self.eos_id]
|
||||
|
||||
return t
|
||||
|
||||
def decode(self, t: list[int], **kwargs) -> str:
|
||||
return self.tokenizer.decode(t, **kwargs)
|
||||
|
||||
def process_one(
|
||||
self,
|
||||
prompt: str,
|
||||
images: list[Image.Image],
|
||||
inference_mode: bool = True,
|
||||
**kwargs: Any,
|
||||
):
|
||||
"""Args:
|
||||
prompt (str): the formatted prompt;
|
||||
images (list[ImageType]): the list of images;
|
||||
inference_mode (bool): if True, then remove the last eos token;
|
||||
**kwargs: Additional keyword arguments.
|
||||
|
||||
Returns:
|
||||
outputs (BaseProcessorOutput): the output of the processor,
|
||||
- input_ids (torch.LongTensor): [N + image tokens]
|
||||
- target_ids (torch.LongTensor): [N + image tokens]
|
||||
- pixel_values (torch.FloatTensor): [n_patches, 3, H, W]
|
||||
- image_id (int): the id of the image token
|
||||
- num_image_tokens (list[int]): the number of image tokens
|
||||
|
||||
"""
|
||||
if prompt is None or images is None:
|
||||
raise ValueError("prompt and images must be used at the same time.")
|
||||
|
||||
sft_format = prompt
|
||||
(
|
||||
tokenized_str,
|
||||
images_list,
|
||||
images_seq_mask,
|
||||
images_spatial_crop,
|
||||
num_image_tokens,
|
||||
) = self.tokenize_with_images(
|
||||
sft_format, images, bos=True, eos=True, cropping=len(images) <= 2
|
||||
)
|
||||
masked_tokenized_str = []
|
||||
for token_index in tokenized_str:
|
||||
if token_index != self.image_token_id:
|
||||
masked_tokenized_str.append(token_index)
|
||||
else:
|
||||
masked_tokenized_str.append(self.ignore_id)
|
||||
|
||||
assert (
|
||||
len(tokenized_str) == len(images_seq_mask) == len(masked_tokenized_str)
|
||||
), (
|
||||
f"tokenized_str's length {len(tokenized_str)}, input_ids' length {len(masked_tokenized_str)}, "
|
||||
f"imags_seq_mask's length {len(images_seq_mask)}, are not equal"
|
||||
)
|
||||
|
||||
input_ids = torch.LongTensor(tokenized_str)
|
||||
target_ids = torch.LongTensor(masked_tokenized_str)
|
||||
images_seq_mask = torch.tensor(images_seq_mask, dtype=torch.bool)
|
||||
|
||||
# set input_ids < 0 | input_ids == self.image_token_id as ignore_id
|
||||
target_ids[(input_ids < 0) | (input_ids == self.image_token_id)] = (
|
||||
self.ignore_id
|
||||
)
|
||||
input_ids[input_ids < 0] = self.pad_id
|
||||
|
||||
if inference_mode:
|
||||
# Remove the ending eos token
|
||||
assert input_ids[-1] == self.eos_id
|
||||
input_ids = input_ids[:-1]
|
||||
target_ids = target_ids[:-1]
|
||||
images_seq_mask = images_seq_mask[:-1]
|
||||
|
||||
if len(images_list) == 0:
|
||||
pixel_values = torch.zeros((1, 3, self.image_size, self.image_size))
|
||||
images_spatial_crop = torch.zeros((1, 2), dtype=torch.long)
|
||||
else:
|
||||
pixel_values = torch.stack(images_list, dim=0)
|
||||
images_spatial_crop = torch.tensor(images_spatial_crop, dtype=torch.long)
|
||||
|
||||
input_ids = input_ids.unsqueeze(0)
|
||||
|
||||
prepare = BatchFeature(
|
||||
data=dict(
|
||||
input_ids=input_ids,
|
||||
pixel_values=pixel_values,
|
||||
images_seq_mask=images_seq_mask,
|
||||
images_spatial_crop=images_spatial_crop,
|
||||
num_image_tokens=num_image_tokens,
|
||||
),
|
||||
tensor_type="pt",
|
||||
)
|
||||
return prepare
|
||||
|
||||
def __call__(
|
||||
self,
|
||||
*,
|
||||
text: str,
|
||||
images: list[Image.Image],
|
||||
inference_mode: bool = True,
|
||||
**kwargs: Any,
|
||||
):
|
||||
"""Args:
|
||||
text (str): the formatted prompt;
|
||||
images (list[ImageType]): the list of images;
|
||||
inference_mode (bool): if True, then remove the last eos token;
|
||||
**kwargs:
|
||||
|
||||
Returns:
|
||||
outputs (BaseProcessorOutput): the output of the processor,
|
||||
- input_ids (torch.LongTensor): [N + image tokens]
|
||||
- images (torch.FloatTensor): [n_images, 3, H, W]
|
||||
- image_id (int): the id of the image token
|
||||
- num_image_tokens (list[int]): the number of image tokens
|
||||
|
||||
"""
|
||||
prepare = self.process_one(
|
||||
prompt=text,
|
||||
images=images,
|
||||
inference_mode=inference_mode,
|
||||
)
|
||||
|
||||
return prepare
|
||||
|
||||
def tokenize_with_images(
|
||||
self,
|
||||
conversation: str,
|
||||
images: list[Image.Image],
|
||||
bos: bool = True,
|
||||
eos: bool = True,
|
||||
cropping: bool = True,
|
||||
):
|
||||
"""Tokenize text with <image> tags."""
|
||||
num_image_tags = conversation.count(self.image_token)
|
||||
if num_image_tags != len(images):
|
||||
raise ValueError(
|
||||
f"Number of {self.image_token!r} tokens in prompt "
|
||||
f"({num_image_tags}) does not match number of images "
|
||||
f"({len(images)})."
|
||||
)
|
||||
text_splits = conversation.split(self.image_token)
|
||||
images_list, images_seq_mask, images_spatial_crop = [], [], []
|
||||
num_image_tokens = []
|
||||
tokenized_str = []
|
||||
for text_sep, image in zip(text_splits, images):
|
||||
"""encode text_sep"""
|
||||
tokenized_sep = self.encode(text_sep, bos=False, eos=False)
|
||||
tokenized_str += tokenized_sep
|
||||
images_seq_mask += [False] * len(tokenized_sep)
|
||||
|
||||
"""select best resolution for anyres"""
|
||||
if cropping:
|
||||
best_width, best_height = self.select_best_resolution(image.size)
|
||||
else:
|
||||
best_width, best_height = self.image_size, self.image_size
|
||||
|
||||
"""process the global view"""
|
||||
global_view = ImageOps.pad(
|
||||
image,
|
||||
(self.image_size, self.image_size),
|
||||
color=tuple(int(x * 255) for x in self.image_transform.mean),
|
||||
)
|
||||
images_list.append(self.image_transform(global_view))
|
||||
|
||||
"""process the local views"""
|
||||
local_view = ImageOps.pad(
|
||||
image,
|
||||
(best_width, best_height),
|
||||
color=tuple(int(x * 255) for x in self.image_transform.mean),
|
||||
)
|
||||
for i in range(0, best_height, self.image_size):
|
||||
for j in range(0, best_width, self.image_size):
|
||||
images_list.append(
|
||||
self.image_transform(
|
||||
local_view.crop(
|
||||
(j, i, j + self.image_size, i + self.image_size)
|
||||
)
|
||||
)
|
||||
)
|
||||
|
||||
"""record height / width crop num"""
|
||||
num_width_tiles, num_height_tiles = (
|
||||
best_width // self.image_size,
|
||||
best_height // self.image_size,
|
||||
)
|
||||
images_spatial_crop.append([num_width_tiles, num_height_tiles])
|
||||
|
||||
"""add image tokens"""
|
||||
h = w = math.ceil(
|
||||
(self.image_size // self.patch_size) / self.downsample_ratio
|
||||
)
|
||||
# global views tokens h * (w + 1), 1 is for line separator
|
||||
tokenized_image = [self.image_token_id] * h * (w + 1)
|
||||
# add a separator between global and local views
|
||||
tokenized_image += [self.image_token_id]
|
||||
# local views tokens, (num_height_tiles * h) * (num_width_tiles * w + 1)
|
||||
tokenized_image += (
|
||||
[self.image_token_id]
|
||||
* (num_height_tiles * h)
|
||||
* (num_width_tiles * w + 1)
|
||||
)
|
||||
|
||||
tokenized_str += tokenized_image
|
||||
images_seq_mask += [True] * len(tokenized_image)
|
||||
num_image_tokens.append(len(tokenized_image))
|
||||
|
||||
"""process the last text split"""
|
||||
tokenized_sep = self.encode(text_splits[-1], bos=False, eos=False)
|
||||
tokenized_str += tokenized_sep
|
||||
images_seq_mask += [False] * len(tokenized_sep)
|
||||
|
||||
"""add the bos and eos tokens"""
|
||||
if bos:
|
||||
tokenized_str = [self.bos_id] + tokenized_str
|
||||
images_seq_mask = [False] + images_seq_mask
|
||||
if eos:
|
||||
tokenized_str = tokenized_str + [self.eos_id]
|
||||
images_seq_mask = images_seq_mask + [False]
|
||||
|
||||
assert len(tokenized_str) == len(images_seq_mask), (
|
||||
f"tokenize_with_images func: tokenized_str's length {len(tokenized_str)} is not equal to imags_seq_mask's length {len(images_seq_mask)}"
|
||||
)
|
||||
|
||||
return (
|
||||
tokenized_str,
|
||||
images_list,
|
||||
images_seq_mask,
|
||||
images_spatial_crop,
|
||||
num_image_tokens,
|
||||
)
|
||||
@@ -1,345 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
from transformers import (
|
||||
AutoFeatureExtractor,
|
||||
BatchFeature,
|
||||
)
|
||||
from transformers.feature_extraction_sequence_utils import SequenceFeatureExtractor
|
||||
from transformers.processing_utils import ProcessorMixin
|
||||
from transformers.utils import TensorType
|
||||
|
||||
from vllm.logger import init_logger
|
||||
from vllm.utils.import_utils import LazyLoader
|
||||
|
||||
if TYPE_CHECKING:
|
||||
import kaldi_native_fbank as knf
|
||||
else:
|
||||
knf = LazyLoader("knf", globals(), "kaldi_native_fbank")
|
||||
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
class CMVN:
|
||||
def __init__(self, dim, means, inverse_std_variences):
|
||||
self.dim, self.means, self.inverse_std_variences = (
|
||||
dim,
|
||||
np.array(means),
|
||||
np.array(inverse_std_variences),
|
||||
)
|
||||
|
||||
def __call__(self, x):
|
||||
assert x.shape[-1] == self.dim, "CMVN dim mismatch"
|
||||
out = x - self.means
|
||||
out = out * self.inverse_std_variences
|
||||
return out
|
||||
|
||||
|
||||
class KaldifeatFbank:
|
||||
def __init__(self, num_mel_bins=80, frame_length=25, frame_shift=10, dither=1.0):
|
||||
self.dither = dither
|
||||
opts = knf.FbankOptions()
|
||||
opts.frame_opts.dither = dither
|
||||
opts.mel_opts.num_bins = num_mel_bins
|
||||
opts.frame_opts.snip_edges = True
|
||||
opts.mel_opts.debug_mel = False
|
||||
self.opts = opts
|
||||
|
||||
def __call__(self, sample_rate, wav_np, is_train=False):
|
||||
dither = self.dither if is_train else 0.0
|
||||
self.opts.frame_opts.dither = dither
|
||||
fbank = knf.OnlineFbank(self.opts)
|
||||
|
||||
fbank.accept_waveform(sample_rate, wav_np.tolist())
|
||||
feat = []
|
||||
for i in range(fbank.num_frames_ready):
|
||||
feat.append(fbank.get_frame(i))
|
||||
if len(feat) == 0:
|
||||
print("Check data, len(feat) == 0", wav_np, flush=True)
|
||||
return np.zeros((0, self.opts.mel_opts.num_bins))
|
||||
feat = np.vstack(feat)
|
||||
return feat
|
||||
|
||||
|
||||
class FireRedASR2FeatureExtractor(SequenceFeatureExtractor):
|
||||
r"""Constructs a FireRedASR2 feature extractor.
|
||||
|
||||
This feature extractor inherits from [`~feature_extraction_sequence_
|
||||
utils.SequenceFeatureExtractor`] which contains most of the main
|
||||
methods. Users should refer to this superclass for more information
|
||||
regarding those methods.
|
||||
|
||||
This class extracts mel-filter bank features from raw speech using a custom
|
||||
numpy implementation of the `Short Time Fourier Transform` which should
|
||||
match pytorch's `torch.stft` equivalent.
|
||||
|
||||
Args:
|
||||
feature_size (`int`, *optional*, defaults to 80):
|
||||
The feature dimension of the extracted features.
|
||||
sampling_rate (`int`, *optional*, defaults to 16000):
|
||||
The sampling rate at which the audio files should be digitalized
|
||||
expressed in hertz (Hz).
|
||||
chunk_length (`int`, *optional*, defaults to 30):
|
||||
The maximum number of chunks of `sampling_rate` samples used to
|
||||
trim and pad longer or shorter audio sequences.
|
||||
padding_value (`float`, *optional*, defaults to 0.0):
|
||||
Padding value used to pad the audio. Should correspond to silences.
|
||||
dither (`float`, *optional*, defaults to 0.0):
|
||||
Adds dithering. In other words, adds a small Gaussian noise to each frame.
|
||||
E.g. use 0.0001 to add dithering with a normal distribution centered
|
||||
around 0.0 with standard deviation 0.0001 (assuming [-1,+1] range
|
||||
of raw_speech). The value 0.0 means no dithering.
|
||||
Dithering has similar effect as `spectrogram(mel_floor=...)`. It reduces
|
||||
the high log_mel_fbank values for signals with hard-zero sections,
|
||||
when VAD cutoff is present in the signal.
|
||||
|
||||
"""
|
||||
|
||||
model_input_names = ["input_features"]
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
feature_size=80,
|
||||
sampling_rate=16000,
|
||||
chunk_length=30,
|
||||
padding_value=0.0,
|
||||
return_attention_mask=False,
|
||||
dim=80,
|
||||
means=None,
|
||||
inverse_std_variences=None,
|
||||
num_mel_bins=80,
|
||||
frame_length=25,
|
||||
frame_shift=10,
|
||||
dither=0.0,
|
||||
max_length=3000,
|
||||
downsample_rate=2,
|
||||
left_context=3,
|
||||
right_context=3,
|
||||
**kwargs,
|
||||
):
|
||||
super().__init__(
|
||||
feature_size=feature_size,
|
||||
sampling_rate=sampling_rate,
|
||||
padding_value=padding_value,
|
||||
return_attention_mask=return_attention_mask,
|
||||
**kwargs,
|
||||
)
|
||||
self.chunk_length = chunk_length
|
||||
self.max_length = max_length
|
||||
self.dim = dim
|
||||
self.means = means
|
||||
self.inverse_std_variences = inverse_std_variences
|
||||
self.num_mel_bins = num_mel_bins
|
||||
self.frame_length = frame_length
|
||||
self.frame_shift = frame_shift
|
||||
self.dither = dither
|
||||
self.sampling_rate = sampling_rate
|
||||
self.downsample_rate = downsample_rate
|
||||
self.context = left_context + 1 + right_context
|
||||
|
||||
def __call__(
|
||||
self,
|
||||
raw_speech: np.ndarray | list[float] | list[np.ndarray] | list[list[float]],
|
||||
truncation: bool = True,
|
||||
pad_to_multiple_of: int | None = None,
|
||||
return_tensors: str | TensorType | None = None,
|
||||
return_attention_mask: bool | None = None,
|
||||
padding: str | None = "max_length",
|
||||
max_length: int | None = None,
|
||||
sampling_rate: int | None = None,
|
||||
do_normalize: bool | None = None,
|
||||
**kwargs,
|
||||
) -> BatchFeature:
|
||||
if sampling_rate != self.sampling_rate:
|
||||
raise ValueError(
|
||||
f"The model corresponding to this feature extractor: "
|
||||
f"{self.__class__.__name__} was trained using a sampling "
|
||||
f"rate of {self.sampling_rate}. Please make sure that the "
|
||||
f"provided `raw_speech` input was sampled with "
|
||||
f"{self.sampling_rate} and not {sampling_rate}."
|
||||
)
|
||||
|
||||
def padding_position_is_0(padded_input, input_lengths):
|
||||
N, T = padded_input.size()[:2]
|
||||
mask = torch.ones((N, T)).to(padded_input.device)
|
||||
for i in range(N):
|
||||
mask[i, input_lengths[i] :] = 0
|
||||
mask = mask.unsqueeze(dim=1)
|
||||
return mask.to(torch.uint8)
|
||||
|
||||
# initialize the CMVN and Fbank objects
|
||||
self.cmvn = CMVN(self.dim, self.means, self.inverse_std_variences)
|
||||
self.fbank = KaldifeatFbank(
|
||||
num_mel_bins=self.num_mel_bins,
|
||||
frame_length=self.frame_length,
|
||||
frame_shift=self.frame_shift,
|
||||
dither=self.dither,
|
||||
)
|
||||
|
||||
feats = []
|
||||
speech_lengths = []
|
||||
fake_token_lengths = []
|
||||
for speech in raw_speech:
|
||||
"""
|
||||
We must multiply by 32768 here because FireRedASR2 loads audio data
|
||||
using kaldiio.load_mat, while vLLM loads audio data using pyav.
|
||||
"""
|
||||
speech = speech * 32768
|
||||
fbank = self.fbank(sampling_rate, speech)
|
||||
fbank = self.cmvn(fbank)
|
||||
fbank = torch.from_numpy(fbank).float()
|
||||
length = fbank.size(0)
|
||||
feats.append(fbank)
|
||||
speech_lengths.append(length)
|
||||
padded_input2 = fbank
|
||||
padded_input2 = F.pad(
|
||||
padded_input2, (0, 0, 0, self.context - 1), "constant", 0.0
|
||||
)
|
||||
src_mask = padding_position_is_0(
|
||||
padded_input2[None, :, :], torch.tensor([length], dtype=torch.int32)
|
||||
)
|
||||
x_mask = src_mask
|
||||
mask = x_mask[:, :, :-2:2][:, :, :-2:2]
|
||||
input_lengths = mask[:, -1, :].sum(dim=-1)
|
||||
input_lengths = input_lengths // self.downsample_rate
|
||||
fake_token_len = torch.clamp(input_lengths, min=1)
|
||||
fake_token_lengths.append(fake_token_len)
|
||||
|
||||
feats = torch.stack(feats, dim=0)
|
||||
batched_speech = self.pad(
|
||||
BatchFeature({"input_features": feats}),
|
||||
padding=padding,
|
||||
max_length=max_length if max_length else self.max_length,
|
||||
truncation=truncation,
|
||||
pad_to_multiple_of=pad_to_multiple_of,
|
||||
return_attention_mask=return_attention_mask or do_normalize,
|
||||
)
|
||||
|
||||
if return_tensors is not None:
|
||||
batched_speech = batched_speech.convert_to_tensors(return_tensors)
|
||||
|
||||
batched_speech["speech_lengths"] = torch.tensor(speech_lengths)
|
||||
batched_speech["fake_token_lengths"] = torch.concat(fake_token_lengths)
|
||||
return batched_speech
|
||||
|
||||
|
||||
class FireRedASR2Processor(ProcessorMixin):
|
||||
r"""Constructs a FireRedASR2 processor which wraps a FireRedASR2 feature extractor
|
||||
and a FireRedASR2 tokenizer into a single processor.
|
||||
|
||||
[`FireRedASR2Processor`] offers all the functionalities of
|
||||
[`FireRedASR2FeatureExtractor`] and [`Qwen2Tokenizer`]. See the
|
||||
[`~FireRedASR2Processor.__call__`] and [`~FireRedASR2Processor.decode`] for more
|
||||
information.
|
||||
|
||||
Args:
|
||||
feature_extractor (`FireRedASR2FeatureExtractor`): An instance of
|
||||
[`FireRedASR2FeatureExtractor`].
|
||||
The feature extractor is a required input.
|
||||
tokenizer (`Qwen2Tokenizer`):
|
||||
An instance of [`Qwen2Tokenizer`]. The tokenizer is a required
|
||||
input.
|
||||
|
||||
"""
|
||||
|
||||
feature_extractor_class = "FireRedASR2FeatureExtractor"
|
||||
tokenizer_class = ("Qwen2Tokenizer", "Qwen2TokenizerFast")
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
feature_extractor,
|
||||
tokenizer,
|
||||
audio_token="<|AUDIO|>",
|
||||
):
|
||||
super().__init__(feature_extractor, tokenizer)
|
||||
self.current_processor = self.feature_extractor
|
||||
self._in_target_context_manager = False
|
||||
self.audio_token = (
|
||||
tokenizer.audio_token if hasattr(tokenizer, "audio_token") else audio_token
|
||||
)
|
||||
self.audio_token_id = tokenizer.convert_tokens_to_ids(self.audio_token)
|
||||
|
||||
def get_decoder_prompt_ids(self, task=None, language=None, no_timestamps=True):
|
||||
return self.tokenizer.get_decoder_prompt_ids(
|
||||
task=task, language=language, no_timestamps=no_timestamps
|
||||
)
|
||||
|
||||
def __call__(self, *args, **kwargs):
|
||||
"""Forwards the `audio` argument to FireRedASR2FeatureExtractor's
|
||||
[`~FireRedASR2FeatureExtractor.__call__`] and the `text` argument to
|
||||
[`~Qwen2Tokenizer.__call__`]. Please refer to the docstring of the
|
||||
above two methods for more information.
|
||||
"""
|
||||
if self._in_target_context_manager:
|
||||
return self.current_processor(*args, **kwargs)
|
||||
|
||||
audio = kwargs.pop("audio", None)
|
||||
sampling_rate = kwargs.pop("sampling_rate", None)
|
||||
text = kwargs.pop("text", None)
|
||||
if len(args) > 0:
|
||||
audio = args[0]
|
||||
args = args[1:]
|
||||
|
||||
if text is None:
|
||||
raise ValueError("You need to specify `text` input to process.")
|
||||
elif isinstance(text, str):
|
||||
text = [text]
|
||||
elif not isinstance(text, list) and not isinstance(text[0], str):
|
||||
raise ValueError(
|
||||
"Invalid input text. Please provide a string, or a list of strings"
|
||||
)
|
||||
|
||||
if audio is not None:
|
||||
# ensure we have as much audios as audio tokens
|
||||
num_audio_tokens = sum(sample.count(self.audio_token) for sample in text)
|
||||
num_audios = 1 if type(audio) is np.ndarray else len(audio)
|
||||
if num_audio_tokens != num_audios:
|
||||
raise ValueError(
|
||||
f"Found {num_audio_tokens} {self.audio_token} token{'s' if num_audio_tokens > 1 else ''} in provided text but received {num_audios} audio{'s' if num_audios > 1 else ''}" # noqa: E501
|
||||
)
|
||||
inputs = self.feature_extractor(
|
||||
audio, *args, sampling_rate=sampling_rate, **kwargs
|
||||
)
|
||||
|
||||
expanded_text = []
|
||||
for sample in text:
|
||||
replace_str = []
|
||||
while self.audio_token in sample:
|
||||
num_audio_tokens = int(inputs["fake_token_lengths"].item())
|
||||
|
||||
expanded_audio_token = self.audio_token * num_audio_tokens
|
||||
|
||||
replace_str.append(expanded_audio_token)
|
||||
sample = sample.replace(self.audio_token, "<placeholder>", 1)
|
||||
|
||||
while "<placeholder>" in sample:
|
||||
sample = sample.replace("<placeholder>", replace_str.pop(0), 1)
|
||||
expanded_text.append(sample)
|
||||
text = expanded_text
|
||||
|
||||
if text is not None:
|
||||
encodings = self.tokenizer(text, **kwargs)
|
||||
|
||||
if text is None:
|
||||
return inputs
|
||||
|
||||
elif audio is None:
|
||||
return encodings
|
||||
else:
|
||||
inputs["labels"] = encodings["input_ids"]
|
||||
|
||||
return inputs
|
||||
|
||||
def get_prompt_ids(self, text: str, return_tensors="np"):
|
||||
return self.tokenizer.get_prompt_ids(text, return_tensors=return_tensors)
|
||||
|
||||
|
||||
AutoFeatureExtractor.register(
|
||||
"FireRedASR2FeatureExtractor", FireRedASR2FeatureExtractor
|
||||
)
|
||||
@@ -1,429 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
import torchaudio.compliance.kaldi as kaldi
|
||||
from torch.nn.utils.rnn import pad_sequence
|
||||
from transformers import (
|
||||
AutoFeatureExtractor,
|
||||
BatchFeature,
|
||||
)
|
||||
from transformers.feature_extraction_sequence_utils import SequenceFeatureExtractor
|
||||
from transformers.processing_utils import ProcessorMixin
|
||||
from transformers.utils import TensorType
|
||||
|
||||
from vllm.logger import init_logger
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
def apply_cmvn(inputs, cmvn): # noqa
|
||||
"""Apply CMVN with mvn data."""
|
||||
device = inputs.device
|
||||
# dtype = inputs.dtype
|
||||
frame, dim = inputs.shape
|
||||
|
||||
means = cmvn[0:1, :dim]
|
||||
vars = cmvn[1:2, :dim]
|
||||
inputs += means.to(device)
|
||||
inputs *= vars.to(device)
|
||||
|
||||
return inputs.type(torch.float32)
|
||||
|
||||
|
||||
def apply_lfr(inputs, lfr_m, lfr_n):
|
||||
# LFR_inputs = []
|
||||
T = inputs.shape[0]
|
||||
T_lfr = int(np.ceil(T / lfr_n))
|
||||
left_padding = inputs[0].repeat((lfr_m - 1) // 2, 1)
|
||||
inputs = torch.vstack((left_padding, inputs))
|
||||
T = T + (lfr_m - 1) // 2
|
||||
feat_dim = inputs.shape[-1]
|
||||
strides = (lfr_n * feat_dim, 1)
|
||||
sizes = (T_lfr, lfr_m * feat_dim)
|
||||
last_idx = (T - lfr_m) // lfr_n + 1
|
||||
num_padding = lfr_m - (T - last_idx * lfr_n)
|
||||
if num_padding > 0:
|
||||
num_padding = (
|
||||
(2 * lfr_m - 2 * T + (T_lfr - 1 + last_idx) * lfr_n)
|
||||
/ 2
|
||||
* (T_lfr - last_idx)
|
||||
)
|
||||
inputs = torch.vstack([inputs] + [inputs[-1:]] * int(num_padding))
|
||||
LFR_outputs = inputs.as_strided(sizes, strides)
|
||||
return LFR_outputs.clone().type(torch.float32)
|
||||
|
||||
|
||||
def load_cmvn(cmvn_file):
|
||||
with open(cmvn_file, encoding="utf-8") as f:
|
||||
lines = f.readlines()
|
||||
means_list = []
|
||||
vars_list = []
|
||||
for i in range(len(lines)):
|
||||
line_item = lines[i].split()
|
||||
if line_item[0] == "<AddShift>":
|
||||
line_item = lines[i + 1].split()
|
||||
if line_item[0] == "<LearnRateCoef>":
|
||||
add_shift_line = line_item[3 : (len(line_item) - 1)]
|
||||
means_list = list(add_shift_line)
|
||||
continue
|
||||
elif line_item[0] == "<Rescale>":
|
||||
line_item = lines[i + 1].split()
|
||||
if line_item[0] == "<LearnRateCoef>":
|
||||
rescale_line = line_item[3 : (len(line_item) - 1)]
|
||||
vars_list = list(rescale_line)
|
||||
continue
|
||||
means = np.array(means_list).astype(np.float32)
|
||||
vars = np.array(vars_list).astype(np.float32)
|
||||
cmvn = np.array([means, vars])
|
||||
cmvn = torch.as_tensor(cmvn, dtype=torch.float32)
|
||||
return cmvn
|
||||
|
||||
|
||||
class WavFrontend(nn.Module):
|
||||
"""Conventional frontend structure for ASR."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
cmvn_file: str = "null",
|
||||
fs: int = 16000,
|
||||
window: str = "hamming",
|
||||
n_mels: int = 80,
|
||||
frame_length: int = 25,
|
||||
frame_shift: int = 10,
|
||||
filter_length_min: int = -1,
|
||||
filter_length_max: int = -1,
|
||||
lfr_m: int = 1,
|
||||
lfr_n: int = 1,
|
||||
dither: float = 1.0,
|
||||
snip_edges: bool = True,
|
||||
upsacle_samples: bool = True,
|
||||
**kwargs,
|
||||
):
|
||||
super().__init__()
|
||||
self.fs = fs
|
||||
self.window = window
|
||||
self.n_mels = n_mels
|
||||
self.frame_length = frame_length
|
||||
self.frame_shift = frame_shift
|
||||
self.filter_length_min = filter_length_min
|
||||
self.filter_length_max = filter_length_max
|
||||
self.lfr_m = lfr_m
|
||||
self.lfr_n = lfr_n
|
||||
self.cmvn_file = cmvn_file
|
||||
self.dither = dither
|
||||
self.snip_edges = snip_edges
|
||||
self.upsacle_samples = upsacle_samples
|
||||
self.cmvn = None if self.cmvn_file is None else load_cmvn(self.cmvn_file)
|
||||
|
||||
def output_size(self) -> int:
|
||||
return self.n_mels * self.lfr_m
|
||||
|
||||
def forward(
|
||||
self,
|
||||
input: torch.Tensor,
|
||||
input_lengths,
|
||||
**kwargs,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
batch_size = input.size(0)
|
||||
feats = []
|
||||
feats_lens = []
|
||||
for i in range(batch_size):
|
||||
waveform_length = input_lengths[i]
|
||||
waveform = input[i][:waveform_length]
|
||||
if self.upsacle_samples:
|
||||
waveform = waveform * (1 << 15)
|
||||
waveform = waveform.unsqueeze(0)
|
||||
mat = kaldi.fbank(
|
||||
waveform,
|
||||
num_mel_bins=self.n_mels,
|
||||
frame_length=min(self.frame_length, waveform_length / self.fs * 1000),
|
||||
frame_shift=self.frame_shift,
|
||||
dither=self.dither,
|
||||
energy_floor=0.0,
|
||||
window_type=self.window,
|
||||
sample_frequency=self.fs,
|
||||
snip_edges=self.snip_edges,
|
||||
)
|
||||
|
||||
if self.lfr_m != 1 or self.lfr_n != 1:
|
||||
mat = apply_lfr(mat, self.lfr_m, self.lfr_n)
|
||||
if self.cmvn is not None:
|
||||
mat = apply_cmvn(mat, self.cmvn)
|
||||
feat_length = mat.size(0)
|
||||
feats.append(mat)
|
||||
feats_lens.append(feat_length)
|
||||
|
||||
feats_lens = torch.as_tensor(feats_lens)
|
||||
if batch_size == 1:
|
||||
feats_pad = feats[0][None, :, :]
|
||||
else:
|
||||
feats_pad = pad_sequence(feats, batch_first=True, padding_value=0.0)
|
||||
return feats_pad, feats_lens
|
||||
|
||||
|
||||
class FunASRFeatureExtractor(SequenceFeatureExtractor):
|
||||
r"""Constructs a FunASR feature extractor.
|
||||
|
||||
This feature extractor inherits from [`~feature_extraction_sequence_
|
||||
utils.SequenceFeatureExtractor`] which contains most of the main
|
||||
methods. Users should refer to this superclass for more information
|
||||
regarding those methods.
|
||||
|
||||
This class extracts mel-filter bank features from raw speech using a custom
|
||||
numpy implementation of the `Short Time Fourier Transform` which should
|
||||
match pytorch's `torch.stft` equivalent.
|
||||
|
||||
Args:
|
||||
feature_size (`int`, *optional*, defaults to 80):
|
||||
The feature dimension of the extracted features.
|
||||
sampling_rate (`int`, *optional*, defaults to 16000):
|
||||
The sampling rate at which the audio files should be digitalized
|
||||
expressed in hertz (Hz).
|
||||
hop_length (`int`, *optional*, defaults to 160):
|
||||
Length of the overlapping windows for the STFT used to obtain the
|
||||
Mel Frequency coefficients.
|
||||
chunk_length (`int`, *optional*, defaults to 30):
|
||||
The maximum number of chunks of `sampling_rate` samples used to
|
||||
trim and pad longer or shorter audio sequences.
|
||||
n_fft (`int`, *optional*, defaults to 400):
|
||||
Size of the Fourier transform.
|
||||
padding_value (`float`, *optional*, defaults to 0.0):
|
||||
Padding value used to pad the audio. Should correspond to silences.
|
||||
dither (`float`, *optional*, defaults to 0.0):
|
||||
Adds dithering. In other words, adds a small Gaussian noise to each frame.
|
||||
E.g. use 0.0001 to add dithering with a normal distribution centered
|
||||
around 0.0 with standard deviation 0.0001 (assuming [-1,+1] range
|
||||
of raw_speech). The value 0.0 means no dithering.
|
||||
Dithering has similar effect as `spectrogram(mel_floor=...)`. It reduces
|
||||
the high log_mel_fbank values for signals with hard-zero sections,
|
||||
when VAD cutoff is present in the signal.
|
||||
|
||||
"""
|
||||
|
||||
model_input_names = ["input_features"]
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
feature_size=80,
|
||||
sampling_rate=16000,
|
||||
hop_length=160,
|
||||
chunk_length=30,
|
||||
n_fft=400,
|
||||
padding_value=0.0,
|
||||
dither=0.0,
|
||||
max_length=1000,
|
||||
return_attention_mask=False,
|
||||
**kwargs,
|
||||
):
|
||||
super().__init__(
|
||||
feature_size=feature_size,
|
||||
sampling_rate=sampling_rate,
|
||||
padding_value=padding_value,
|
||||
return_attention_mask=return_attention_mask,
|
||||
**kwargs,
|
||||
)
|
||||
self.frontend_conf = kwargs.get("frontend_conf", {})
|
||||
self.max_length = max_length
|
||||
self.n_fft = n_fft
|
||||
self.hop_length = hop_length
|
||||
self.chunk_length = chunk_length
|
||||
self.n_samples = chunk_length * sampling_rate
|
||||
self.nb_max_frames = self.n_samples // hop_length
|
||||
self.sampling_rate = sampling_rate
|
||||
self.dither = dither
|
||||
|
||||
def extract_fbank(
|
||||
self, data, data_len=None, data_type: str = "sound", frontend=None, **kwargs
|
||||
):
|
||||
if isinstance(data, np.ndarray):
|
||||
data = torch.from_numpy(data)
|
||||
if len(data.shape) < 2:
|
||||
data = data[None, :] # data: [batch, N]
|
||||
data_len = [data.shape[1]] if data_len is None else data_len
|
||||
elif isinstance(data, torch.Tensor):
|
||||
if len(data.shape) < 2:
|
||||
data = data[None, :] # data: [batch, N]
|
||||
data_len = [data.shape[1]] if data_len is None else data_len
|
||||
elif isinstance(data, (list, tuple)):
|
||||
data_list, data_len = [], []
|
||||
for data_i in data:
|
||||
if isinstance(data_i, np.ndarray):
|
||||
data_i = torch.from_numpy(data_i)
|
||||
data_list.append(data_i)
|
||||
data_len.append(data_i.shape[0])
|
||||
data = pad_sequence(data_list, batch_first=True)
|
||||
|
||||
data, data_len = frontend(data, data_len, **kwargs)
|
||||
|
||||
if isinstance(data_len, (list, tuple)):
|
||||
data_len = torch.tensor([data_len])
|
||||
return data.to(torch.float32), data_len.to(torch.int32)
|
||||
|
||||
def __call__(
|
||||
self,
|
||||
raw_speech: np.ndarray | list[float] | list[np.ndarray] | list[list[float]],
|
||||
truncation: bool = True,
|
||||
pad_to_multiple_of: int | None = None,
|
||||
return_tensors: str | TensorType | None = None,
|
||||
return_attention_mask: bool | None = None,
|
||||
padding: str | None = "max_length",
|
||||
max_length: int | None = None,
|
||||
sampling_rate: int | None = None,
|
||||
do_normalize: bool | None = None,
|
||||
device: str | None = "cpu",
|
||||
return_token_timestamps: bool | None = None,
|
||||
**kwargs,
|
||||
) -> BatchFeature:
|
||||
frontend = WavFrontend(**self.frontend_conf, dither=self.dither)
|
||||
|
||||
feats = []
|
||||
speech_lengths = []
|
||||
fake_token_lengths = []
|
||||
for speech in raw_speech:
|
||||
feature, length = self.extract_fbank(
|
||||
speech,
|
||||
data_type=kwargs.get("data_type", "sound"),
|
||||
frontend=frontend,
|
||||
is_final=True,
|
||||
)
|
||||
feats.append(feature)
|
||||
speech_lengths.append(length)
|
||||
olens = 1 + (length - 3 + 2 * 1) // 2
|
||||
olens = 1 + (olens - 3 + 2 * 1) // 2
|
||||
fake_token_len = (olens - 1) // 2 + 1
|
||||
fake_token_len = torch.clamp(fake_token_len, min=1)
|
||||
fake_token_lengths.append(fake_token_len)
|
||||
|
||||
feats = torch.concat(feats, dim=0)
|
||||
batched_speech = self.pad(
|
||||
BatchFeature({"input_features": feats}),
|
||||
padding=padding,
|
||||
max_length=max_length if max_length else self.max_length,
|
||||
truncation=truncation,
|
||||
pad_to_multiple_of=pad_to_multiple_of,
|
||||
return_attention_mask=return_attention_mask or do_normalize,
|
||||
)
|
||||
if return_tensors is not None:
|
||||
batched_speech = batched_speech.convert_to_tensors(return_tensors)
|
||||
|
||||
batched_speech["speech_lengths"] = torch.tensor(speech_lengths)
|
||||
batched_speech["fake_token_lengths"] = torch.concat(fake_token_lengths)
|
||||
return batched_speech
|
||||
|
||||
|
||||
class FunASRProcessor(ProcessorMixin):
|
||||
r"""Constructs a FunASR processor which wraps a FunASR feature extractor and
|
||||
a FunASR tokenizer into a single processor.
|
||||
|
||||
[`FunASRProcessor`] offers all the functionalities of
|
||||
[`FunASRFeatureExtractor`] and [`Qwen2Tokenizer`]. See the
|
||||
[`~FunASRProcessor.__call__`] and [`~FunASRProcessor.decode`] for more
|
||||
information.
|
||||
|
||||
Args:
|
||||
feature_extractor (`FunASRFeatureExtractor`): An instance of
|
||||
[`FunASRFeatureExtractor`].
|
||||
The feature extractor is a required input.
|
||||
tokenizer (`Qwen2Tokenizer`):
|
||||
An instance of [`Qwen2Tokenizer`]. The tokenizer is a required
|
||||
input.
|
||||
|
||||
"""
|
||||
|
||||
feature_extractor_class = "FunASRFeatureExtractor"
|
||||
tokenizer_class = ("Qwen2Tokenizer", "Qwen2TokenizerFast")
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
feature_extractor,
|
||||
tokenizer,
|
||||
audio_token="<|AUDIO|>",
|
||||
):
|
||||
super().__init__(feature_extractor, tokenizer)
|
||||
self.current_processor = self.feature_extractor
|
||||
self._in_target_context_manager = False
|
||||
self.audio_token = (
|
||||
tokenizer.audio_token if hasattr(tokenizer, "audio_token") else audio_token
|
||||
)
|
||||
self.audio_token_id = tokenizer.convert_tokens_to_ids(self.audio_token)
|
||||
|
||||
def get_decoder_prompt_ids(self, task=None, language=None, no_timestamps=True):
|
||||
return self.tokenizer.get_decoder_prompt_ids(
|
||||
task=task, language=language, no_timestamps=no_timestamps
|
||||
)
|
||||
|
||||
def __call__(self, *args, **kwargs):
|
||||
"""Forwards the `audio` argument to FunASRFeatureExtractor's
|
||||
[`~FunASRFeatureExtractor.__call__`] and the `text` argument to
|
||||
[`~Qwen2Tokenizer.__call__`]. Please refer to the docstring of the
|
||||
above two methods for more information.
|
||||
"""
|
||||
if self._in_target_context_manager:
|
||||
return self.current_processor(*args, **kwargs)
|
||||
|
||||
audio = kwargs.pop("audio", None)
|
||||
sampling_rate = kwargs.pop("sampling_rate", None)
|
||||
text = kwargs.pop("text", None)
|
||||
if len(args) > 0:
|
||||
audio = args[0]
|
||||
args = args[1:]
|
||||
|
||||
if text is None:
|
||||
raise ValueError("You need to specify `text` input to process.")
|
||||
elif isinstance(text, str):
|
||||
text = [text]
|
||||
elif not isinstance(text, list) and not isinstance(text[0], str):
|
||||
raise ValueError(
|
||||
"Invalid input text. Please provide a string, or a list of strings"
|
||||
)
|
||||
|
||||
if audio is not None:
|
||||
# ensure we have as much audios as audio tokens
|
||||
num_audio_tokens = sum(sample.count(self.audio_token) for sample in text)
|
||||
num_audios = 1 if type(audio) is np.ndarray else len(audio)
|
||||
if num_audio_tokens != num_audios:
|
||||
raise ValueError(
|
||||
f"Found {num_audio_tokens} {self.audio_token} token{'s' if num_audio_tokens > 1 else ''} in provided text but received {num_audios} audio{'s' if num_audios > 1 else ''}" # noqa: E501
|
||||
)
|
||||
inputs = self.feature_extractor(
|
||||
audio, *args, sampling_rate=sampling_rate, **kwargs
|
||||
)
|
||||
|
||||
expanded_text = []
|
||||
for sample in text:
|
||||
replace_str = []
|
||||
while self.audio_token in sample:
|
||||
num_audio_tokens = inputs["fake_token_lengths"].item()
|
||||
|
||||
expanded_audio_token = self.audio_token * num_audio_tokens
|
||||
|
||||
replace_str.append(expanded_audio_token)
|
||||
sample = sample.replace(self.audio_token, "<placeholder>", 1)
|
||||
|
||||
while "<placeholder>" in sample:
|
||||
sample = sample.replace("<placeholder>", replace_str.pop(0), 1)
|
||||
expanded_text.append(sample)
|
||||
text = expanded_text
|
||||
|
||||
if text is not None:
|
||||
encodings = self.tokenizer(text, **kwargs)
|
||||
|
||||
if text is None:
|
||||
return inputs
|
||||
|
||||
elif audio is None:
|
||||
return encodings
|
||||
else:
|
||||
inputs["labels"] = encodings["input_ids"]
|
||||
|
||||
return inputs
|
||||
|
||||
def get_prompt_ids(self, text: str, return_tensors="np"):
|
||||
return self.tokenizer.get_prompt_ids(text, return_tensors=return_tensors)
|
||||
|
||||
|
||||
AutoFeatureExtractor.register("FunASRFeatureExtractor", FunASRFeatureExtractor)
|
||||
@@ -1,35 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
# Adapted from
|
||||
# https://github.com/zai-org/CogAgent
|
||||
from transformers import PythonBackend
|
||||
from transformers.image_processing_backends import TorchvisionBackend
|
||||
from transformers.image_utils import PILImageResampling
|
||||
from transformers.processing_utils import ProcessorMixin
|
||||
|
||||
|
||||
class GLM4VImageProcessorFast(TorchvisionBackend):
|
||||
"""Port of https://huggingface.co/zai-org/glm-4v-9b/blob/main/tokenization_chatglm.py#L177
|
||||
to HF Transformers.
|
||||
"""
|
||||
|
||||
resample = PILImageResampling.BICUBIC
|
||||
image_mean = [0.48145466, 0.4578275, 0.40821073]
|
||||
image_std = [0.26862954, 0.26130258, 0.27577711]
|
||||
size = {"height": 1120, "width": 1120}
|
||||
do_resize = True
|
||||
do_rescale = True
|
||||
do_normalize = True
|
||||
|
||||
|
||||
class GLM4VProcessor(ProcessorMixin):
|
||||
attributes = ["image_processor", "tokenizer"]
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
image_processor: GLM4VImageProcessorFast,
|
||||
tokenizer: PythonBackend,
|
||||
) -> None:
|
||||
self.image_processor = image_processor
|
||||
self.tokenizer = tokenizer
|
||||
@@ -1,387 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
# adapted from https://huggingface.co/h2oai/h2ovl-mississippi-2b/blob/main/modeling_h2ovl_chat.py
|
||||
# https://huggingface.co/h2oai/h2ovl-mississippi-2b/blob/main/image_process.py
|
||||
# --------------------------------------------------------
|
||||
# H2OVL-Mississippi
|
||||
# Copyright (c) 2024 H2O.AI
|
||||
# Licensed under Apache 2.0 License [see LICENSE for details]
|
||||
# --------------------------------------------------------
|
||||
import torch
|
||||
from PIL import Image
|
||||
|
||||
from vllm.tokenizers.hf import HfTokenizer
|
||||
|
||||
from .internvl import (
|
||||
InternVLImageProcessor,
|
||||
InternVLProcessor,
|
||||
build_transform,
|
||||
find_closest_aspect_ratio,
|
||||
get_internvl_target_ratios,
|
||||
)
|
||||
|
||||
|
||||
def resolve_h2ovl_min_max_num(
|
||||
*,
|
||||
min_dynamic_patch: int,
|
||||
max_dynamic_patch: int,
|
||||
dynamic_image_size: bool,
|
||||
use_thumbnail: bool,
|
||||
) -> tuple[int, int]:
|
||||
min_dynamic_patch = min_dynamic_patch if dynamic_image_size else 1
|
||||
max_dynamic_patch = max_dynamic_patch if dynamic_image_size else 1
|
||||
|
||||
if use_thumbnail and max_dynamic_patch != 1:
|
||||
max_dynamic_patch += 1
|
||||
|
||||
return min_dynamic_patch, max_dynamic_patch
|
||||
|
||||
|
||||
def get_h2ovl_target_ratios(
|
||||
min_num: int,
|
||||
max_num: int,
|
||||
*,
|
||||
prior_aspect_ratio: tuple[int, int] | None,
|
||||
) -> list[tuple[int, int]]:
|
||||
target_ratios = get_internvl_target_ratios(min_num, max_num)
|
||||
|
||||
# if prior_aspect_ratio is provided, filter the target ratios
|
||||
if prior_aspect_ratio is not None:
|
||||
target_ratios = [
|
||||
ratio
|
||||
for ratio in target_ratios
|
||||
if prior_aspect_ratio[0] % ratio[0] != 0
|
||||
and prior_aspect_ratio[1] % ratio[1] != 0
|
||||
]
|
||||
|
||||
return target_ratios
|
||||
|
||||
|
||||
# modified to include blocks generated in second pass
|
||||
def calculate_h2ovl_targets(
|
||||
*,
|
||||
orig_width: int,
|
||||
orig_height: int,
|
||||
target_ratios: list[tuple[int, int]],
|
||||
image_size: int,
|
||||
use_thumbnail: bool,
|
||||
) -> tuple[int, int, int, tuple[int, int]]:
|
||||
aspect_ratio = orig_width / orig_height
|
||||
|
||||
# find the closest aspect ratio to the target
|
||||
target_aspect_ratio = find_closest_aspect_ratio(
|
||||
aspect_ratio,
|
||||
target_ratios,
|
||||
width=orig_width,
|
||||
height=orig_height,
|
||||
image_size=image_size,
|
||||
)
|
||||
|
||||
# calculate the target width and height
|
||||
target_width = image_size * target_aspect_ratio[0]
|
||||
target_height = image_size * target_aspect_ratio[1]
|
||||
blocks = target_aspect_ratio[0] * target_aspect_ratio[1]
|
||||
|
||||
# add thumbnail image if num_blocks != 1
|
||||
if use_thumbnail and blocks != 1:
|
||||
blocks += 1
|
||||
|
||||
return blocks, target_width, target_height, target_aspect_ratio
|
||||
|
||||
|
||||
# adapted from https://huggingface.co/OpenGVLab/InternVL2-1B
|
||||
# refactored to handle prior_aspect_ratio
|
||||
def dynamic_preprocess_h2ovl(
|
||||
image: Image.Image,
|
||||
*,
|
||||
target_ratios: list[tuple[int, int]],
|
||||
image_size: int,
|
||||
use_thumbnail: bool,
|
||||
) -> tuple[list[Image.Image], tuple[int, int]]:
|
||||
orig_width, orig_height = image.size
|
||||
|
||||
# calculate the number of blocks without thumbnail
|
||||
(
|
||||
blocks,
|
||||
target_width,
|
||||
target_height,
|
||||
target_aspect_ratio,
|
||||
) = calculate_h2ovl_targets(
|
||||
orig_width=orig_width,
|
||||
orig_height=orig_height,
|
||||
target_ratios=target_ratios,
|
||||
image_size=image_size,
|
||||
use_thumbnail=False,
|
||||
)
|
||||
|
||||
# resize the image
|
||||
resized_img = image.resize((target_width, target_height))
|
||||
processed_images = []
|
||||
for i in range(blocks):
|
||||
box = (
|
||||
(i % (target_width // image_size)) * image_size,
|
||||
(i // (target_width // image_size)) * image_size,
|
||||
((i % (target_width // image_size)) + 1) * image_size,
|
||||
((i // (target_width // image_size)) + 1) * image_size,
|
||||
)
|
||||
# split the image
|
||||
split_img = resized_img.crop(box)
|
||||
processed_images.append(split_img)
|
||||
|
||||
assert len(processed_images) == blocks
|
||||
|
||||
if use_thumbnail and len(processed_images) != 1:
|
||||
thumbnail_img = image.resize((image_size, image_size))
|
||||
processed_images.append(thumbnail_img)
|
||||
|
||||
return processed_images, target_aspect_ratio
|
||||
|
||||
|
||||
def _preprocess_image(
|
||||
image: Image.Image,
|
||||
*,
|
||||
input_size: int,
|
||||
min_num: int,
|
||||
max_num: int,
|
||||
use_thumbnail: bool,
|
||||
prior_aspect_ratio: tuple[int, int] | None,
|
||||
) -> tuple[torch.Tensor, tuple[int, int]]:
|
||||
target_ratios = get_h2ovl_target_ratios(
|
||||
min_num,
|
||||
max_num,
|
||||
prior_aspect_ratio=prior_aspect_ratio,
|
||||
)
|
||||
|
||||
transform = build_transform(input_size=input_size)
|
||||
images, target_aspect_ratio = dynamic_preprocess_h2ovl(
|
||||
image,
|
||||
image_size=input_size,
|
||||
use_thumbnail=use_thumbnail,
|
||||
target_ratios=target_ratios,
|
||||
)
|
||||
|
||||
pixel_values = torch.stack([transform(image) for image in images])
|
||||
return pixel_values, target_aspect_ratio
|
||||
|
||||
|
||||
# refactored to use the _preprocess_image function
|
||||
def image_to_pixel_values_h2ovl(
|
||||
image: Image.Image,
|
||||
*,
|
||||
input_size: int,
|
||||
min_num: int,
|
||||
max_num: int,
|
||||
use_thumbnail: bool,
|
||||
use_msac: bool,
|
||||
) -> torch.Tensor:
|
||||
# when MSAC is turned on, we need to process the image twice
|
||||
if use_msac:
|
||||
# first pass
|
||||
pixel_values1, aspect_ratio1 = _preprocess_image(
|
||||
image,
|
||||
input_size=input_size,
|
||||
min_num=1,
|
||||
max_num=max_num,
|
||||
use_thumbnail=True,
|
||||
prior_aspect_ratio=None,
|
||||
)
|
||||
# second pass
|
||||
pixel_values2, _ = _preprocess_image(
|
||||
image,
|
||||
input_size=input_size,
|
||||
min_num=3,
|
||||
max_num=max_num,
|
||||
use_thumbnail=True,
|
||||
prior_aspect_ratio=aspect_ratio1,
|
||||
)
|
||||
# combine pixel values
|
||||
pixel_values = torch.cat(
|
||||
[pixel_values2[:-1], pixel_values1[:-1], pixel_values2[-1:]], 0
|
||||
)
|
||||
|
||||
else:
|
||||
pixel_values, _ = _preprocess_image(
|
||||
image,
|
||||
input_size=input_size,
|
||||
min_num=min_num,
|
||||
max_num=max_num,
|
||||
use_thumbnail=use_thumbnail,
|
||||
prior_aspect_ratio=None,
|
||||
)
|
||||
|
||||
return pixel_values
|
||||
|
||||
|
||||
class H2OVLImageProcessor(InternVLImageProcessor):
|
||||
def __init__(
|
||||
self,
|
||||
image_size: int,
|
||||
min_dynamic_patch: int,
|
||||
max_dynamic_patch: int,
|
||||
dynamic_image_size: bool,
|
||||
use_thumbnail: bool,
|
||||
use_msac: bool,
|
||||
) -> None:
|
||||
super().__init__(
|
||||
image_size=image_size,
|
||||
min_dynamic_patch=min_dynamic_patch,
|
||||
max_dynamic_patch=max_dynamic_patch,
|
||||
dynamic_image_size=dynamic_image_size,
|
||||
use_thumbnail=use_thumbnail,
|
||||
)
|
||||
|
||||
self.use_msac = use_msac
|
||||
|
||||
def resolve_min_max_num(
|
||||
self,
|
||||
*,
|
||||
min_dynamic_patch: int | None = None,
|
||||
max_dynamic_patch: int | None = None,
|
||||
dynamic_image_size: bool | None = None,
|
||||
use_thumbnail: bool | None = None,
|
||||
) -> tuple[int, int]:
|
||||
if min_dynamic_patch is None:
|
||||
min_dynamic_patch = self.min_dynamic_patch
|
||||
if max_dynamic_patch is None:
|
||||
max_dynamic_patch = self.max_dynamic_patch
|
||||
if dynamic_image_size is None:
|
||||
dynamic_image_size = self.dynamic_image_size
|
||||
if use_thumbnail is None:
|
||||
use_thumbnail = self.use_thumbnail
|
||||
|
||||
return resolve_h2ovl_min_max_num(
|
||||
min_dynamic_patch=min_dynamic_patch,
|
||||
max_dynamic_patch=max_dynamic_patch,
|
||||
dynamic_image_size=dynamic_image_size,
|
||||
use_thumbnail=use_thumbnail,
|
||||
)
|
||||
|
||||
def _images_to_pixel_values_lst(
|
||||
self,
|
||||
images: list[Image.Image],
|
||||
min_dynamic_patch: int | None = None,
|
||||
max_dynamic_patch: int | None = None,
|
||||
dynamic_image_size: bool | None = None,
|
||||
) -> list[torch.Tensor]:
|
||||
use_msac = self.use_msac if len(images) == 1 else False
|
||||
|
||||
min_num, max_num = self.resolve_min_max_num(
|
||||
min_dynamic_patch=min_dynamic_patch,
|
||||
max_dynamic_patch=max_dynamic_patch,
|
||||
dynamic_image_size=dynamic_image_size,
|
||||
use_thumbnail=False, # Applied in image_to_pixel_values
|
||||
)
|
||||
|
||||
return [
|
||||
image_to_pixel_values_h2ovl(
|
||||
image,
|
||||
input_size=self.image_size,
|
||||
min_num=min_num,
|
||||
max_num=max_num,
|
||||
use_thumbnail=self.use_thumbnail,
|
||||
use_msac=use_msac,
|
||||
)
|
||||
for image in images
|
||||
]
|
||||
|
||||
|
||||
class H2OVLProcessor(InternVLProcessor):
|
||||
def __init__(
|
||||
self,
|
||||
image_processor: H2OVLImageProcessor,
|
||||
tokenizer: HfTokenizer,
|
||||
*,
|
||||
image_seq_length: int,
|
||||
start_image_token: str = "<img>",
|
||||
end_image_token: str = "</img>",
|
||||
ctx_image_token: str = "<IMG_CONTEXT>",
|
||||
) -> None:
|
||||
super().__init__(
|
||||
image_processor=image_processor,
|
||||
tokenizer=tokenizer,
|
||||
image_seq_length=image_seq_length,
|
||||
start_image_token=start_image_token,
|
||||
end_image_token=end_image_token,
|
||||
ctx_image_token=ctx_image_token,
|
||||
)
|
||||
|
||||
self.image_processor: H2OVLImageProcessor
|
||||
|
||||
def resolve_target_ratios(
|
||||
self,
|
||||
*,
|
||||
min_dynamic_patch: int | None = None,
|
||||
max_dynamic_patch: int | None = None,
|
||||
dynamic_image_size: bool | None = None,
|
||||
use_thumbnail: bool | None = None,
|
||||
prior_aspect_ratio: tuple[int, int] | None = None,
|
||||
override_min_num: int | None = None,
|
||||
) -> list[tuple[int, int]]:
|
||||
min_num, max_num = self.image_processor.resolve_min_max_num(
|
||||
min_dynamic_patch=min_dynamic_patch,
|
||||
max_dynamic_patch=max_dynamic_patch,
|
||||
dynamic_image_size=dynamic_image_size,
|
||||
use_thumbnail=use_thumbnail,
|
||||
)
|
||||
if override_min_num is not None:
|
||||
min_num = override_min_num
|
||||
|
||||
return get_h2ovl_target_ratios(
|
||||
min_num,
|
||||
max_num,
|
||||
prior_aspect_ratio=prior_aspect_ratio,
|
||||
)
|
||||
|
||||
def get_num_image_tokens(
|
||||
self,
|
||||
*,
|
||||
image_width: int,
|
||||
image_height: int,
|
||||
use_msac: bool | None = None,
|
||||
) -> int:
|
||||
image_processor = self.image_processor
|
||||
use_msac = image_processor.use_msac if use_msac is None else use_msac
|
||||
|
||||
use_thumbnail = image_processor.use_thumbnail
|
||||
|
||||
if use_msac:
|
||||
target_ratios_1 = self.resolve_target_ratios(
|
||||
use_thumbnail=False, # Applied in calculate_targets
|
||||
override_min_num=1,
|
||||
)
|
||||
num_patches_1, _, _, aspect_ratio_1 = calculate_h2ovl_targets(
|
||||
orig_width=image_width,
|
||||
orig_height=image_height,
|
||||
image_size=image_processor.image_size,
|
||||
target_ratios=target_ratios_1,
|
||||
use_thumbnail=True,
|
||||
)
|
||||
|
||||
target_ratios_2 = self.resolve_target_ratios(
|
||||
use_thumbnail=False, # Applied in calculate_targets
|
||||
prior_aspect_ratio=aspect_ratio_1,
|
||||
override_min_num=3,
|
||||
)
|
||||
num_patches_2, _, _, _ = calculate_h2ovl_targets(
|
||||
orig_width=image_width,
|
||||
orig_height=image_height,
|
||||
image_size=image_processor.image_size,
|
||||
target_ratios=target_ratios_2,
|
||||
use_thumbnail=True,
|
||||
)
|
||||
|
||||
num_patches = num_patches_1 + num_patches_2 - 1
|
||||
else:
|
||||
target_ratios = self.resolve_target_ratios(
|
||||
use_thumbnail=False, # Applied in calculate_targets
|
||||
)
|
||||
num_patches, _, _, _ = calculate_h2ovl_targets(
|
||||
orig_width=image_width,
|
||||
orig_height=image_height,
|
||||
image_size=image_processor.image_size,
|
||||
target_ratios=target_ratios,
|
||||
use_thumbnail=use_thumbnail,
|
||||
)
|
||||
|
||||
return num_patches * self.image_seq_length
|
||||
@@ -1,509 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import math
|
||||
from typing import Any, TypedDict
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
from PIL import Image
|
||||
from transformers import (
|
||||
BatchFeature,
|
||||
ImageProcessingMixin,
|
||||
ProcessorMixin,
|
||||
TensorType,
|
||||
)
|
||||
from transformers.processing_utils import ProcessingKwargs
|
||||
from typing_extensions import Unpack
|
||||
|
||||
from vllm.tokenizers.hf import HfTokenizer
|
||||
|
||||
MAX_PIXELS = 60_000_000 # 60-megapixel ceiling ≈ 8200 × 7300 px
|
||||
|
||||
# Vision preprocessing constants
|
||||
VISION_MEAN = (0.5, 0.5, 0.5)
|
||||
VISION_STD = (0.5, 0.5, 0.5)
|
||||
VISION_SCALE = 1 / 255
|
||||
|
||||
|
||||
def _make_writeable(arr: np.ndarray) -> np.ndarray:
|
||||
"""Return *arr* itself if it is already writeable, otherwise try to flip the
|
||||
write flag in-place and finally fall back to `arr.copy()`.
|
||||
This guarantees the buffer handed to `torch.from_numpy()` is always
|
||||
writeable, silencing the PyTorch warning about undefined behaviour.
|
||||
"""
|
||||
if arr.flags.writeable:
|
||||
return arr
|
||||
|
||||
# First, try the cheap path — in-place flag toggle (works for mmap'd arrays
|
||||
# and some shared memory buffers):
|
||||
try:
|
||||
arr.setflags(write=True)
|
||||
return arr # success: no data copy
|
||||
except ValueError:
|
||||
# Buffer is inherently read-only (e.g. backed by PyAV / PIL): make copy
|
||||
return arr.copy()
|
||||
|
||||
|
||||
def extract_image_pil(image: Image.Image) -> torch.Tensor:
|
||||
if image.width * image.height > MAX_PIXELS:
|
||||
raise ValueError(
|
||||
f"Image (w={image.width}, h={image.height}) > MAX=`{MAX_PIXELS}`"
|
||||
)
|
||||
img = image if image.mode == "RGB" else image.convert("RGB")
|
||||
arr = np.asarray(img)
|
||||
arr = _make_writeable(arr)
|
||||
return torch.from_numpy(arr)
|
||||
|
||||
|
||||
def get_image_size_for_max_num_patches(
|
||||
image_height: int,
|
||||
image_width: int,
|
||||
patch_size: int,
|
||||
max_num_patches: int,
|
||||
min_num_patches: int | None = None,
|
||||
eps: float = 1e-5,
|
||||
pixel_shuffle_scale: int = 1,
|
||||
) -> tuple[int, int]:
|
||||
r"""Compute a target resolution whose patch grid satisfies patching parametrization.
|
||||
|
||||
Args:
|
||||
image_height (`int`):
|
||||
Height in pixels of the source image prior to any resizing.
|
||||
image_width (`int`):
|
||||
Width in pixels of the source image prior to any resizing.
|
||||
patch_size (`int`):
|
||||
Size of the square patch used by the vision encoder.
|
||||
max_num_patches (`int`):
|
||||
Upper bound on `(height / patch_size) * (width / patch_size)` after
|
||||
resizing.
|
||||
min_num_patches (`int`, *optional*):
|
||||
Lower bound on the number of patches. When provided the image will
|
||||
be scaled up if necessary.
|
||||
eps (`float`, *optional*, defaults to 1e-5):
|
||||
Convergence tolerance for the internal binary search to determine
|
||||
the target dimensions.
|
||||
pixel_shuffle_scale (`int`, *optional*, defaults to 1):
|
||||
Additional stride multiplier applied when pixel shuffle later
|
||||
reduces spatial resolution.
|
||||
|
||||
Returns:
|
||||
`tuple[int, int]`: Height and width (in pixels) that are multiples of
|
||||
`patch_size * pixel_shuffle_scale` and respect both the maximum and
|
||||
optional minimum patch-count constraints.
|
||||
|
||||
"""
|
||||
|
||||
def get_scaled_image_size(scale, original_size, patch_size, pixel_shuffle_scale):
|
||||
scaled_size = scale * original_size
|
||||
divisor = patch_size * pixel_shuffle_scale
|
||||
scaled_size = math.ceil(scaled_size / divisor) * divisor
|
||||
scaled_size = max(divisor, scaled_size)
|
||||
return int(scaled_size)
|
||||
|
||||
# Ensure divisibility
|
||||
divisor = patch_size * pixel_shuffle_scale
|
||||
adjusted_height = math.ceil(image_height / divisor) * divisor
|
||||
adjusted_height = max(divisor, adjusted_height)
|
||||
adjusted_width = math.ceil(image_width / divisor) * divisor
|
||||
adjusted_width = max(divisor, adjusted_width)
|
||||
|
||||
num_patches = (adjusted_height / patch_size) * (adjusted_width / patch_size)
|
||||
|
||||
if min_num_patches is not None and num_patches < min_num_patches:
|
||||
# Scale up
|
||||
scale_min, scale_max = 1.0, 100.0
|
||||
while (scale_max - scale_min) >= eps:
|
||||
scale = (scale_min + scale_max) / 2
|
||||
target_height = get_scaled_image_size(
|
||||
scale, image_height, patch_size, pixel_shuffle_scale
|
||||
)
|
||||
target_width = get_scaled_image_size(
|
||||
scale, image_width, patch_size, pixel_shuffle_scale
|
||||
)
|
||||
num_patches = (target_height / patch_size) * (target_width / patch_size)
|
||||
if num_patches >= min_num_patches:
|
||||
scale_max = scale
|
||||
else:
|
||||
scale_min = scale
|
||||
scale = scale_max
|
||||
target_height = get_scaled_image_size(
|
||||
scale, image_height, patch_size, pixel_shuffle_scale
|
||||
)
|
||||
target_width = get_scaled_image_size(
|
||||
scale, image_width, patch_size, pixel_shuffle_scale
|
||||
)
|
||||
return target_height, target_width
|
||||
elif num_patches <= max_num_patches:
|
||||
return adjusted_height, adjusted_width
|
||||
else:
|
||||
# Scale down
|
||||
scale_min, scale_max = eps / 10, 1.0
|
||||
while (scale_max - scale_min) >= eps:
|
||||
scale = (scale_min + scale_max) / 2
|
||||
target_height = get_scaled_image_size(
|
||||
scale, image_height, patch_size, pixel_shuffle_scale
|
||||
)
|
||||
target_width = get_scaled_image_size(
|
||||
scale, image_width, patch_size, pixel_shuffle_scale
|
||||
)
|
||||
num_patches = (target_height / patch_size) * (target_width / patch_size)
|
||||
if num_patches <= max_num_patches:
|
||||
scale_min = scale
|
||||
else:
|
||||
scale_max = scale
|
||||
scale = scale_min
|
||||
target_height = get_scaled_image_size(
|
||||
scale, image_height, patch_size, pixel_shuffle_scale
|
||||
)
|
||||
target_width = get_scaled_image_size(
|
||||
scale, image_width, patch_size, pixel_shuffle_scale
|
||||
)
|
||||
return target_height, target_width
|
||||
|
||||
|
||||
_MEAN_TENSOR = torch.tensor(VISION_MEAN, dtype=torch.float32).view(1, 1, 1, -1)
|
||||
_STD_TENSOR = torch.tensor(VISION_STD, dtype=torch.float32).view(1, 1, 1, -1)
|
||||
|
||||
|
||||
def prepare_image_tensor(
|
||||
image: torch.Tensor,
|
||||
scale: float = VISION_SCALE,
|
||||
) -> torch.Tensor:
|
||||
r"""Standardize RGB images prior to patch extraction via rescaling and whitening.
|
||||
|
||||
Args:
|
||||
image (`torch.Tensor`):
|
||||
Tensor with shape `(..., height, width, 3)` containing RGB values.
|
||||
The tensor is converted to floating point if needed.
|
||||
scale (`float`, *optional*, defaults to `VISION_SCALE`):
|
||||
Scalar multiplier applied before normalization.
|
||||
|
||||
Returns:
|
||||
`torch.Tensor`: Normalized tensor with the same shape as the input and
|
||||
dtype `torch.float32`.
|
||||
|
||||
"""
|
||||
if not torch.is_floating_point(image):
|
||||
image = image.float()
|
||||
rescaled = image * scale
|
||||
|
||||
# Use precomputed tensors and move to the correct device if needed
|
||||
mean_tensor = _MEAN_TENSOR.to(image.device)
|
||||
std_tensor = _STD_TENSOR.to(image.device)
|
||||
|
||||
normalized = (rescaled - mean_tensor) / std_tensor
|
||||
return normalized
|
||||
|
||||
|
||||
def patchify_vision(image: torch.Tensor, patch_size: int) -> torch.Tensor:
|
||||
r"""Convert normalized images into flattened ViT-style patches.
|
||||
|
||||
Args:
|
||||
image (`torch.Tensor`):
|
||||
Tensor of shape `(num_images, height, width, channels)`.
|
||||
patch_size (`int`):
|
||||
Edge length of the square patches
|
||||
|
||||
Returns:
|
||||
`torch.Tensor`:
|
||||
Patch tensor where each position stores the flattened pixels
|
||||
belonging to that patch.
|
||||
|
||||
Raises:
|
||||
ValueError: If `height` or `width` is not divisible by `patch_size`.
|
||||
|
||||
"""
|
||||
num_images, height, width, channels = image.shape
|
||||
if height % patch_size or width % patch_size:
|
||||
raise ValueError(
|
||||
"Dimensions of images "
|
||||
f"{image.shape} are not divisible by patch_size={patch_size}."
|
||||
)
|
||||
patches = image.reshape(
|
||||
num_images,
|
||||
height // patch_size,
|
||||
patch_size,
|
||||
width // patch_size,
|
||||
patch_size,
|
||||
channels,
|
||||
)
|
||||
patches = patches.permute(0, 1, 3, 2, 4, 5)
|
||||
patches = patches.reshape(
|
||||
num_images,
|
||||
height // patch_size,
|
||||
width // patch_size,
|
||||
channels * patch_size * patch_size,
|
||||
)
|
||||
return patches
|
||||
|
||||
|
||||
def process_vision_for_patches(
|
||||
images: torch.Tensor,
|
||||
patch_size: int,
|
||||
max_num_patches: int,
|
||||
min_num_patches: int | None = None,
|
||||
pixel_shuffle_scale: int = 1,
|
||||
) -> tuple[torch.Tensor, list[int]]:
|
||||
r"""Resize, normalize, and patchify RGB images for the vision encoder.
|
||||
|
||||
Args:
|
||||
images (`torch.Tensor`):
|
||||
Either `(height, width, channels)` for a single image or
|
||||
`(num_images, height, width, channels)` for a batch. Channels are
|
||||
expected to be RGB.
|
||||
patch_size (`int`):
|
||||
Edge length of square patches; implicitly controls resize grid granularity.
|
||||
max_num_patches (`int`):
|
||||
Maximum number of patches allowed after resizing.
|
||||
min_num_patches (`int`, *optional*):
|
||||
Minimum number of patches. If provided, the routine upsamples images
|
||||
as needed to satisfy the lower bound.
|
||||
pixel_shuffle_scale (`int`, *optional*, defaults to 1):
|
||||
Pixel shuffle scale factor; influences the target grid that the
|
||||
function produces.
|
||||
|
||||
Returns:
|
||||
`tuple[torch.Tensor, list[int]]`: A pair `(patches, dims_virtual)`
|
||||
where `patches` has shape `(num_images, target_h / patch_size, target_w
|
||||
/ patch_size, channels * patch_size**2)` and `dims_virtual` encodes
|
||||
effective `(images, height, width)` dimensions after optional pixel
|
||||
shuffling.
|
||||
|
||||
"""
|
||||
# Add batch dim if single image
|
||||
if images.dim() == 3:
|
||||
images = images.unsqueeze(0)
|
||||
|
||||
# Permute to channel first for resize
|
||||
images = images.permute(0, 3, 1, 2)
|
||||
|
||||
# Get target dimensions
|
||||
_, _, orig_height, orig_width = images.shape
|
||||
target_height, target_width = get_image_size_for_max_num_patches(
|
||||
orig_height,
|
||||
orig_width,
|
||||
patch_size,
|
||||
max_num_patches,
|
||||
min_num_patches=min_num_patches,
|
||||
pixel_shuffle_scale=pixel_shuffle_scale,
|
||||
)
|
||||
|
||||
# Resize
|
||||
images = F.interpolate(
|
||||
images,
|
||||
size=(target_height, target_width),
|
||||
mode="bilinear",
|
||||
align_corners=False,
|
||||
)
|
||||
|
||||
# Back to channel last
|
||||
images = images.permute(0, 2, 3, 1)
|
||||
|
||||
# Normalize
|
||||
images = prepare_image_tensor(images)
|
||||
|
||||
# Patchify
|
||||
patches = patchify_vision(images, patch_size=patch_size)
|
||||
|
||||
# Calculate dimensions for the patches
|
||||
n_images, h_patches, w_patches, _ = patches.shape
|
||||
dims_virtual = (
|
||||
[1, h_patches, w_patches]
|
||||
if pixel_shuffle_scale == 1
|
||||
else [1, h_patches // pixel_shuffle_scale, w_patches // pixel_shuffle_scale]
|
||||
)
|
||||
|
||||
return patches, dims_virtual
|
||||
|
||||
|
||||
class IsaacImagesKwargs(TypedDict, total=False):
|
||||
patch_size: int
|
||||
max_num_patches: int
|
||||
min_num_patches: int
|
||||
pixel_shuffle_scale: int
|
||||
|
||||
|
||||
class IsaacProcessorKwargs(ProcessingKwargs, total=False): # type: ignore[call-arg]
|
||||
images_kwargs: IsaacImagesKwargs
|
||||
_defaults = {
|
||||
"text_kwargs": {"padding": False},
|
||||
"images_kwargs": {},
|
||||
}
|
||||
|
||||
|
||||
class IsaacImageProcessor(ImageProcessingMixin):
|
||||
model_input_names = ["pixel_values", "image_grid_thw"]
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
patch_size: int = 16,
|
||||
vision_max_num_patches: int = 6144,
|
||||
vision_min_num_patches: int = 256,
|
||||
pixel_shuffle_scale: int = 2,
|
||||
) -> None:
|
||||
self.patch_size = patch_size
|
||||
self.vision_max_num_patches = vision_max_num_patches
|
||||
self.vision_min_num_patches = vision_min_num_patches
|
||||
self.pixel_shuffle_scale = pixel_shuffle_scale
|
||||
|
||||
def __call__(
|
||||
self,
|
||||
images: Image.Image | list[Image.Image],
|
||||
return_tensors: str | TensorType | None = None,
|
||||
**kwargs: Unpack[IsaacImagesKwargs],
|
||||
) -> BatchFeature:
|
||||
"""Preprocess images into format compatible with vLLM input processing."""
|
||||
if not isinstance(images, list):
|
||||
images = [images]
|
||||
|
||||
all_pixel_values: list[torch.Tensor] = []
|
||||
all_image_grids: list[torch.Tensor] = []
|
||||
|
||||
for image in images:
|
||||
image_tensor = extract_image_pil(image)
|
||||
|
||||
patches, dims_virtual = process_vision_for_patches(
|
||||
image_tensor,
|
||||
patch_size=kwargs.get("patch_size", self.patch_size),
|
||||
max_num_patches=kwargs.get(
|
||||
"max_num_patches", self.vision_max_num_patches
|
||||
),
|
||||
min_num_patches=kwargs.get(
|
||||
"min_num_patches", self.vision_min_num_patches
|
||||
),
|
||||
pixel_shuffle_scale=kwargs.get(
|
||||
"pixel_shuffle_scale", self.pixel_shuffle_scale
|
||||
),
|
||||
)
|
||||
|
||||
# Isaac packs a dummy temporal dim for images
|
||||
patches = patches.unsqueeze(1) # [N, T=1, Hp, Wp, D]
|
||||
|
||||
hp, wp, dim = patches.shape[-3], patches.shape[-2], patches.shape[-1]
|
||||
current_num_patches = hp * wp
|
||||
pixel_values = patches.reshape(current_num_patches, dim) # [N_tokens, D]
|
||||
|
||||
# Use real patch dimensions for image_grid_thw, not virtual dimensions
|
||||
# This ensures the vision model receives correct grid info for pixel shuffle
|
||||
dims_real = [1, hp, wp] # Real patch dimensions
|
||||
image_grid_thw = torch.tensor(dims_real).unsqueeze(0)
|
||||
|
||||
all_pixel_values.append(pixel_values)
|
||||
all_image_grids.append(image_grid_thw)
|
||||
|
||||
if all_pixel_values:
|
||||
final_pixel_values = torch.cat(all_pixel_values, dim=0)
|
||||
final_image_grids = torch.cat(all_image_grids, dim=0)
|
||||
else:
|
||||
final_pixel_values = torch.empty(0, 0)
|
||||
final_image_grids = torch.empty(0, 3)
|
||||
|
||||
return BatchFeature(
|
||||
data={
|
||||
"pixel_values": final_pixel_values,
|
||||
"image_grid_thw": final_image_grids,
|
||||
},
|
||||
tensor_type=return_tensors,
|
||||
)
|
||||
|
||||
|
||||
class IsaacProcessor(ProcessorMixin):
|
||||
attributes = ["image_processor", "tokenizer"]
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
image_processor: IsaacImageProcessor,
|
||||
tokenizer: HfTokenizer,
|
||||
image_token: str = "<image>",
|
||||
):
|
||||
self.image_processor = image_processor
|
||||
self.tokenizer = tokenizer
|
||||
|
||||
self.image_token = image_token
|
||||
|
||||
def __call__(
|
||||
self,
|
||||
text: str | list[str] | None = None,
|
||||
images: Image.Image | list[Image.Image] | None = None,
|
||||
return_tensors: str | TensorType | None = None,
|
||||
**kwargs: Unpack[IsaacProcessorKwargs], # type: ignore[misc]
|
||||
) -> BatchFeature:
|
||||
output_kwargs = self._merge_kwargs(
|
||||
IsaacProcessorKwargs,
|
||||
tokenizer_init_kwargs=self.tokenizer.init_kwargs,
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
if images is not None:
|
||||
image_inputs = self.image_processor(
|
||||
images, **output_kwargs["images_kwargs"]
|
||||
)
|
||||
image_grid_thw = image_inputs["image_grid_thw"]
|
||||
else:
|
||||
image_inputs = {}
|
||||
image_grid_thw = []
|
||||
|
||||
if text is not None:
|
||||
if not isinstance(text, list):
|
||||
text = [text]
|
||||
|
||||
if image_inputs:
|
||||
text = text.copy() # below lines change text in-place
|
||||
merge_length = self.image_processor.pixel_shuffle_scale**2
|
||||
index = 0
|
||||
for i in range(len(text)):
|
||||
while self.image_token in text[i]:
|
||||
num_image_tokens = image_grid_thw[index].prod() // merge_length
|
||||
text[i] = text[i].replace(
|
||||
self.image_token, "<|placeholder|>" * num_image_tokens, 1
|
||||
)
|
||||
index += 1
|
||||
text[i] = text[i].replace("<|placeholder|>", "<|image_pad|>")
|
||||
|
||||
text_inputs = self.tokenizer(text, **output_kwargs["text_kwargs"])
|
||||
else:
|
||||
text_inputs = {}
|
||||
|
||||
return BatchFeature(
|
||||
data={**text_inputs, **image_inputs},
|
||||
tensor_type=return_tensors,
|
||||
)
|
||||
|
||||
def apply_chat_template(
|
||||
self,
|
||||
messages: list[dict[str, Any]],
|
||||
tokenize: bool = False,
|
||||
add_generation_prompt: bool = False,
|
||||
**kwargs,
|
||||
) -> Any:
|
||||
# Convert mixed content messages to simple text format
|
||||
processed_messages = []
|
||||
|
||||
for message in messages:
|
||||
if "content" in message and isinstance(message["content"], list):
|
||||
# Handle mixed content (text + image)
|
||||
text_parts = []
|
||||
for content_item in message["content"]:
|
||||
if content_item.get("type") == "text":
|
||||
text_parts.append(content_item.get("text", ""))
|
||||
elif content_item.get("type") == "image":
|
||||
# Replace image with vision token
|
||||
text_parts.append(self.image_token)
|
||||
|
||||
processed_message = {
|
||||
"role": message.get("role", "user"),
|
||||
"content": "".join(text_parts),
|
||||
}
|
||||
processed_messages.append(processed_message)
|
||||
else:
|
||||
# Regular text message
|
||||
processed_messages.append(message)
|
||||
|
||||
kwargs["return_dict"] = False
|
||||
return self.tokenizer.apply_chat_template(
|
||||
processed_messages,
|
||||
tokenize=tokenize,
|
||||
add_generation_prompt=add_generation_prompt,
|
||||
**kwargs,
|
||||
)
|
||||
@@ -1,105 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
# Copyright 2026 The Moonshot AI team and the HuggingFace Inc. team.
|
||||
# All rights reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
"""Processor for Kimi-Audio ASR model."""
|
||||
|
||||
import numpy as np
|
||||
from transformers import BatchFeature, ProcessorMixin
|
||||
from transformers.audio_utils import AudioInput
|
||||
from transformers.tokenization_utils_base import PreTokenizedInput, TextInput
|
||||
|
||||
|
||||
class KimiAudioProcessor(ProcessorMixin):
|
||||
# Required for ProcessorMixin
|
||||
attributes = ["feature_extractor", "tokenizer"]
|
||||
feature_extractor_class = "AutoFeatureExtractor"
|
||||
tokenizer_class = "AutoTokenizer"
|
||||
|
||||
# Special token IDs
|
||||
KIMIA_MEDIA_BEGIN: int = 151661
|
||||
KIMIA_MEDIA_END: int = 151663
|
||||
KIMIA_TEXT_BLANK: int = 151666
|
||||
|
||||
# Audio processing constants
|
||||
AUDIO_SEQ_LEN: int = 376
|
||||
|
||||
def __init__(self, feature_extractor=None, tokenizer=None, **kwargs):
|
||||
self.feature_extractor = feature_extractor
|
||||
self.tokenizer = tokenizer
|
||||
|
||||
def __call__(
|
||||
self,
|
||||
text: TextInput
|
||||
| PreTokenizedInput
|
||||
| list[TextInput]
|
||||
| list[PreTokenizedInput]
|
||||
| None = None,
|
||||
audio: AudioInput | None = None,
|
||||
return_tensors: str = "pt",
|
||||
**kwargs,
|
||||
) -> BatchFeature:
|
||||
if text is not None:
|
||||
if not isinstance(text, list):
|
||||
text = [text]
|
||||
|
||||
text_inputs = self.tokenizer(
|
||||
text, return_tensors=return_tensors, padding=True
|
||||
)
|
||||
else:
|
||||
text_inputs = {}
|
||||
|
||||
if audio is not None:
|
||||
# Ensure audio is a list
|
||||
if isinstance(audio, np.ndarray):
|
||||
audio = [audio]
|
||||
|
||||
# Pad audio to hop length (required by WhisperFeatureExtractor)
|
||||
hop_length = self.feature_extractor.hop_length
|
||||
padded_audio = []
|
||||
for aud in audio:
|
||||
length = aud.shape[-1]
|
||||
if length % hop_length != 0:
|
||||
pad_length = hop_length - (length % hop_length)
|
||||
aud = np.pad(
|
||||
aud, (0, pad_length), mode="constant", constant_values=0
|
||||
)
|
||||
padded_audio.append(aud)
|
||||
|
||||
# Use feature_extractor directly like Qwen3ASR does
|
||||
audio_inputs = self.feature_extractor(
|
||||
padded_audio,
|
||||
sampling_rate=16000,
|
||||
padding=True,
|
||||
return_attention_mask=True,
|
||||
return_tensors=return_tensors,
|
||||
)
|
||||
# Rename to match Kimi-Audio expectations
|
||||
if "input_features" in audio_inputs:
|
||||
audio_inputs["whisper_input_features"] = audio_inputs.pop(
|
||||
"input_features"
|
||||
)
|
||||
if "attention_mask" in audio_inputs:
|
||||
audio_inputs["feature_attention_mask"] = audio_inputs.pop(
|
||||
"attention_mask"
|
||||
)
|
||||
else:
|
||||
audio_inputs = {}
|
||||
|
||||
return BatchFeature(
|
||||
data={**text_inputs, **audio_inputs},
|
||||
tensor_type=return_tensors,
|
||||
)
|
||||
@@ -1,92 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from transformers import BaseImageProcessor, BatchFeature, TensorType
|
||||
from transformers.processing_utils import ProcessorMixin
|
||||
|
||||
from vllm.multimodal.inputs import VisionChunk
|
||||
from vllm.tokenizers.hf import HfTokenizer
|
||||
|
||||
|
||||
class KimiK25Processor(ProcessorMixin):
|
||||
attributes = ["image_processor", "tokenizer"]
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
image_processor: BaseImageProcessor,
|
||||
tokenizer: HfTokenizer,
|
||||
media_token_id: int,
|
||||
) -> None:
|
||||
self.image_processor = image_processor
|
||||
self.tokenizer = tokenizer
|
||||
|
||||
self.media_token_id = media_token_id
|
||||
|
||||
def __call__(
|
||||
self,
|
||||
text: str | list[str] | None = None,
|
||||
vision_chunks: list[VisionChunk] | None = None,
|
||||
return_tensors: str | TensorType | None = None,
|
||||
do_rescale: bool = True,
|
||||
do_normalize: bool = True,
|
||||
**kwargs,
|
||||
) -> BatchFeature:
|
||||
"""Args:
|
||||
text: The text to be field to the model.
|
||||
vision_chunks: List of `VisionChunk` items to be processed.
|
||||
For image: `VisionChunkImage` with
|
||||
`type='image', image=PIL.Image`
|
||||
For video_chunk: `VisionChunkVideo` with
|
||||
`type='video_chunk', video_chunk=list[PIL.Image]`
|
||||
Returns:
|
||||
[`BatchFeature`]: A [`BatchFeature`] with the following fields:
|
||||
|
||||
- **input_ids** -- list of token ids to be fed to a model.
|
||||
- **pixel_values** -- Pixel values to be fed to a model.
|
||||
Returned when `vision_chunks` is not `None`.
|
||||
- **grid_thws** -- list of image 3D grid in LLM.
|
||||
Returned when `vision_chunks` is not `None`.
|
||||
|
||||
"""
|
||||
if vision_chunks is not None:
|
||||
mm_inputs = self.image_processor.preprocess(
|
||||
vision_chunks,
|
||||
return_tensors=return_tensors,
|
||||
do_rescale=do_rescale,
|
||||
do_normalize=do_normalize,
|
||||
)
|
||||
else:
|
||||
mm_inputs = {}
|
||||
|
||||
if text is not None:
|
||||
if not isinstance(text, list):
|
||||
text = [text]
|
||||
|
||||
text_inputs = self.tokenizer(text)
|
||||
|
||||
# Note: Modify in-place
|
||||
input_ids: list[list[int]] = text_inputs["input_ids"] # type: ignore
|
||||
|
||||
if vision_chunks is not None:
|
||||
num_tokens_per_chunk = [
|
||||
self.image_processor.media_tokens_calculator(chunk)
|
||||
for chunk in vision_chunks
|
||||
]
|
||||
|
||||
for i in range(len(input_ids)):
|
||||
new_input_ids = []
|
||||
for token in input_ids[i]:
|
||||
if token == self.media_token_id:
|
||||
new_input_ids.extend(
|
||||
[self.media_token_id] * num_tokens_per_chunk.pop(0)
|
||||
)
|
||||
else:
|
||||
new_input_ids.append(token)
|
||||
|
||||
input_ids[i] = new_input_ids
|
||||
else:
|
||||
text_inputs = {}
|
||||
|
||||
return BatchFeature(
|
||||
data={**text_inputs, **mm_inputs},
|
||||
tensor_type=return_tensors,
|
||||
)
|
||||
@@ -1,65 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from transformers import BaseImageProcessor, BatchFeature, TensorType
|
||||
from transformers.processing_utils import ProcessorMixin
|
||||
|
||||
from vllm.tokenizers.hf import HfTokenizer
|
||||
|
||||
|
||||
class KimiK3Processor(ProcessorMixin):
|
||||
"""HF-style processor wrapper for the image-only Kimi-K3 model.
|
||||
|
||||
K3 exposes the standard ``image`` modality, so vLLM calls this processor
|
||||
with ``images=[PIL, ...]``. The underlying checkpoint image processor
|
||||
(``KimiK3VisionProcessor``) works on ``{"type": "image", "image": PIL}``
|
||||
media dicts, so this wrapper adapts bare PIL images into that shape before
|
||||
delegating to ``preprocess``.
|
||||
|
||||
Text is only tokenized here; the single ``<|kimi_image_placeholder|>``
|
||||
token per image is expanded into the resolution-aware media block by the
|
||||
model's ``_get_prompt_updates`` on the vLLM side.
|
||||
"""
|
||||
|
||||
attributes = ["image_processor", "tokenizer"]
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
image_processor: BaseImageProcessor,
|
||||
tokenizer: HfTokenizer,
|
||||
) -> None:
|
||||
self.image_processor = image_processor
|
||||
self.tokenizer = tokenizer
|
||||
|
||||
def __call__(
|
||||
self,
|
||||
text: str | list[str] | None = None,
|
||||
images: object | list[object] | None = None,
|
||||
return_tensors: str | TensorType | None = None,
|
||||
do_rescale: bool = True,
|
||||
do_normalize: bool = True,
|
||||
**kwargs,
|
||||
) -> BatchFeature:
|
||||
if images is not None:
|
||||
if not isinstance(images, list):
|
||||
images = [images]
|
||||
medias = [{"type": "image", "image": image} for image in images]
|
||||
mm_inputs = self.image_processor.preprocess(
|
||||
medias,
|
||||
return_tensors=return_tensors,
|
||||
do_rescale=do_rescale,
|
||||
do_normalize=do_normalize,
|
||||
)
|
||||
else:
|
||||
mm_inputs = {}
|
||||
|
||||
if text is not None:
|
||||
if not isinstance(text, list):
|
||||
text = [text]
|
||||
text_inputs = self.tokenizer(text)
|
||||
else:
|
||||
text_inputs = {}
|
||||
|
||||
return BatchFeature(
|
||||
data={**text_inputs, **mm_inputs},
|
||||
tensor_type=return_tensors,
|
||||
)
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,523 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
# coding=utf-8
|
||||
# Copyright 2025 The OpenBMB Team. All rights reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
"""Processor class for MiniCPMO."""
|
||||
|
||||
import math
|
||||
from typing import TypeAlias
|
||||
|
||||
import numpy as np
|
||||
import regex
|
||||
import torch
|
||||
from transformers.image_processing_utils import BatchFeature
|
||||
from transformers.image_utils import ImageInput
|
||||
from transformers.processing_utils import ProcessorMixin
|
||||
from transformers.tokenization_utils_base import PreTokenizedInput, TextInput
|
||||
from transformers.utils import TensorType
|
||||
|
||||
MiniCPMOBatchFeature: TypeAlias = BatchFeature
|
||||
|
||||
|
||||
class MiniCPMOProcessor(ProcessorMixin):
|
||||
r"""Constructs a MiniCPMV processor which wraps a MiniCPMV image
|
||||
processor and a MiniCPMV tokenizer into a single processor.
|
||||
|
||||
[`MiniCPMVProcessor`] offers all the functionalities of
|
||||
[`MiniCPMVImageProcessor`] and [`LlamaTokenizerWrapper`]. See the
|
||||
[`~MiniCPMVProcessor.__call__`] and [`~MiniCPMVProcessor.decode`]
|
||||
for more information.
|
||||
|
||||
Args:
|
||||
image_processor ([`MiniCPMVImageProcessor`], *optional*):
|
||||
The image processor is a required input.
|
||||
tokenizer ([`LlamaTokenizerWrapper`], *optional*):
|
||||
The tokenizer is a required input.
|
||||
|
||||
"""
|
||||
|
||||
attributes = ["image_processor", "feature_extractor", "tokenizer"]
|
||||
feature_extractor_class = "WhisperFeatureExtractor"
|
||||
image_processor_class = "AutoImageProcessor"
|
||||
tokenizer_class = "AutoTokenizer"
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
image_processor=None,
|
||||
feature_extractor=None,
|
||||
tokenizer=None,
|
||||
pool_step=2,
|
||||
):
|
||||
super().__init__(image_processor, feature_extractor, tokenizer)
|
||||
# Mirror the MiniCPMVProcessor guard: newer (transformers v5.7+)
|
||||
# MiniCPM image processors may drop the legacy `version` attribute,
|
||||
# so fall back to None instead of hard-crashing. `version` only
|
||||
# special-cases the 2.5 tokenization path; other values take the
|
||||
# default branch.
|
||||
self.version = getattr(image_processor, "version", None)
|
||||
self.pool_step = pool_step
|
||||
|
||||
def __call__(
|
||||
self,
|
||||
text: TextInput | PreTokenizedInput | list[TextInput] | list[PreTokenizedInput],
|
||||
images: ImageInput = None,
|
||||
audios: np.ndarray | list[np.ndarray] | list[list[np.ndarray]] = None,
|
||||
audio_parts: list | None = None,
|
||||
max_length: int | None = None,
|
||||
do_pad: bool | None = True,
|
||||
max_slice_nums: int | None = None,
|
||||
use_image_id: bool = True,
|
||||
chunk_input: bool = False,
|
||||
return_tensors: str | TensorType | None = TensorType.PYTORCH,
|
||||
sampling_rate: int | None = 16000,
|
||||
**kwargs,
|
||||
) -> MiniCPMOBatchFeature:
|
||||
if images is not None:
|
||||
image_inputs = self.image_processor(
|
||||
images,
|
||||
do_pad=do_pad,
|
||||
max_slice_nums=max_slice_nums,
|
||||
return_tensors=return_tensors,
|
||||
)
|
||||
else:
|
||||
image_inputs = None
|
||||
|
||||
if audios is not None:
|
||||
audio_features, audio_feature_lens, audio_phs = self.audio_feature_extract(
|
||||
audios, audio_parts, chunk_input, sampling_rate
|
||||
)
|
||||
else:
|
||||
audio_features, audio_feature_lens, audio_phs = [], [], []
|
||||
|
||||
model_inputs = self._convert_omni_to_inputs(
|
||||
image_inputs,
|
||||
audio_phs,
|
||||
text,
|
||||
max_slice_nums=max_slice_nums,
|
||||
use_image_id=use_image_id,
|
||||
max_length=max_length,
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
model_inputs["audio_features"] = audio_features
|
||||
model_inputs["audio_feature_lens"] = audio_feature_lens
|
||||
|
||||
return MiniCPMOBatchFeature(data={**model_inputs})
|
||||
|
||||
def get_audio_placeholder(self, audio_lens, chunk_input, chunk_length):
|
||||
pool_step = self.pool_step
|
||||
feature_lens = math.ceil(audio_lens / self.feature_extractor.hop_length)
|
||||
|
||||
feature_lens = (feature_lens - 1) // 2 + 1
|
||||
output_lens = (feature_lens - pool_step) // pool_step + 1
|
||||
|
||||
audio_start = getattr(self.tokenizer, "audio_start", "<audio>")
|
||||
audio_end = getattr(self.tokenizer, "audio_end", "</audio>")
|
||||
|
||||
if chunk_input:
|
||||
fbank_feat_in_chunk = int(chunk_length * 100)
|
||||
cnn_feat_in_chunk = (fbank_feat_in_chunk - 1) // 2 + 1
|
||||
audio_embeds_in_chunk = (cnn_feat_in_chunk - pool_step) // pool_step + 1
|
||||
num_audio_chunks = (
|
||||
output_lens + audio_embeds_in_chunk - 1
|
||||
) // audio_embeds_in_chunk
|
||||
|
||||
place_holders = ""
|
||||
total_unk_len = 0
|
||||
for _ in range(num_audio_chunks):
|
||||
unk_len = min(audio_embeds_in_chunk, output_lens - total_unk_len)
|
||||
place_holders += audio_start + "<unk>" * unk_len + audio_end
|
||||
total_unk_len += unk_len
|
||||
audio_placeholder = place_holders
|
||||
else:
|
||||
audio_placeholder = audio_start + "<unk>" * output_lens + audio_end
|
||||
|
||||
return audio_placeholder
|
||||
|
||||
def audio_feature_extract(
|
||||
self,
|
||||
audios: np.ndarray | list[np.ndarray] | list[list[np.ndarray]],
|
||||
audio_parts: list | None = None,
|
||||
chunk_input: bool | None = False,
|
||||
sampling_rate: int | None = None,
|
||||
chunk_length: int | None = 1,
|
||||
**kwargs,
|
||||
):
|
||||
if isinstance(audios, np.ndarray):
|
||||
audios_list = [[audios]]
|
||||
elif isinstance(audios[0], np.ndarray):
|
||||
audios_list = [audios]
|
||||
else:
|
||||
audios_list = audios
|
||||
|
||||
if audio_parts is not None:
|
||||
assert len(audio_parts) == len(audios_list)
|
||||
for parts, audios in zip(audio_parts, audios_list):
|
||||
assert len(parts) == len(audios)
|
||||
|
||||
audio_feature_lens_list = []
|
||||
audio_ph_list = []
|
||||
|
||||
audio_features_all = []
|
||||
|
||||
# audio placeholder not dependent on audio_parts
|
||||
for audios in audios_list:
|
||||
if audios:
|
||||
audio_ph_list.append(
|
||||
[
|
||||
self.get_audio_placeholder(len(a), chunk_input, chunk_length)
|
||||
for a in audios
|
||||
]
|
||||
)
|
||||
else:
|
||||
audio_ph_list.append([])
|
||||
|
||||
for idx, audios in enumerate(audios_list):
|
||||
if audio_parts is not None:
|
||||
# same audio part merge
|
||||
audio_part = audio_parts[idx]
|
||||
merge_audio = []
|
||||
cur_audio = []
|
||||
for aid, (part, audio) in enumerate(zip(audio_part, audios)):
|
||||
if aid == 0 or audio_part[aid] == audio_part[aid - 1]:
|
||||
cur_audio.append(audio)
|
||||
else:
|
||||
merge_audio.append(np.hstack(cur_audio))
|
||||
cur_audio = [audio]
|
||||
if cur_audio:
|
||||
merge_audio.append(np.hstack(cur_audio))
|
||||
|
||||
else:
|
||||
merge_audio = audios
|
||||
|
||||
audio_feature_lens = []
|
||||
|
||||
# If the audio exceeds 30 seconds, split it into chunks every 30 seconds.
|
||||
final_merge_audio = []
|
||||
max_audio_inp_len = 30 * (sampling_rate or 16000)
|
||||
for audio in merge_audio:
|
||||
if len(audio) <= max_audio_inp_len:
|
||||
final_merge_audio.append(audio)
|
||||
else:
|
||||
for i in range(math.ceil(len(audio) / max_audio_inp_len)):
|
||||
final_merge_audio.append(
|
||||
audio[i * max_audio_inp_len : (i + 1) * max_audio_inp_len]
|
||||
)
|
||||
|
||||
if audios:
|
||||
audio_inputs = self.feature_extractor(
|
||||
final_merge_audio,
|
||||
sampling_rate=sampling_rate,
|
||||
return_attention_mask=True,
|
||||
padding="max_length",
|
||||
return_tensors="pt",
|
||||
**kwargs,
|
||||
)
|
||||
audio_feature = audio_inputs["input_features"]
|
||||
actual_lens = audio_inputs["attention_mask"].sum(dim=1)
|
||||
|
||||
for feat, lens in zip(audio_feature, actual_lens):
|
||||
audio_features_all.append(feat[:, :lens])
|
||||
audio_feature_lens.append(lens)
|
||||
|
||||
audio_feature_lens = torch.hstack(audio_feature_lens)
|
||||
audio_feature_lens_list.append(audio_feature_lens)
|
||||
else:
|
||||
audio_feature_lens_list.append([])
|
||||
|
||||
if audio_features_all:
|
||||
audio_features = [i.permute(1, 0) for i in audio_features_all]
|
||||
audio_features = torch.nn.utils.rnn.pad_sequence(
|
||||
audio_features, batch_first=True, padding_value=0.0
|
||||
).permute(0, 2, 1)
|
||||
else:
|
||||
audio_features = []
|
||||
|
||||
return audio_features, audio_feature_lens_list, audio_ph_list
|
||||
|
||||
# Copied from transformers.models.clip.processing_clip.CLIPProcessor.batch_decode
|
||||
# with CLIP->Llama
|
||||
def batch_decode(self, *args, **kwargs):
|
||||
"""This method forwards all its arguments to LlamaTokenizerFast's
|
||||
[`~PythonBackend.batch_decode`]. Please refer to the
|
||||
docstring of this method for more information.
|
||||
"""
|
||||
output_ids = args[0]
|
||||
result_text = []
|
||||
for result in output_ids:
|
||||
result = result[result != 0]
|
||||
if len(result) > 0 and result[0] == self.tokenizer.bos_id:
|
||||
result = result[1:]
|
||||
if len(result) > 0 and result[-1] == self.tokenizer.eos_id:
|
||||
result = result[:-1]
|
||||
result_text.append(
|
||||
self.tokenizer.decode(result, *args[1:], **kwargs).strip()
|
||||
)
|
||||
return result_text
|
||||
|
||||
# Copied from transformers.models.clip.processing_clip.CLIPProcessor.decode
|
||||
# with CLIP->Llama
|
||||
def decode(self, *args, **kwargs):
|
||||
"""This method forwards all its arguments to LlamaTokenizerFast's
|
||||
[`~PythonBackend.decode`]. Please refer to the docstring
|
||||
of this method for more information.
|
||||
"""
|
||||
result = args[0]
|
||||
result = result[result != 0]
|
||||
if len(result) > 0 and result[0] == self.tokenizer.bos_id:
|
||||
result = result[1:]
|
||||
if len(result) > 0 and (
|
||||
result[-1] == self.tokenizer.eos_id
|
||||
or (
|
||||
hasattr(self.tokenizer, "eot_id")
|
||||
and result[-1] == self.tokenizer.eot_id
|
||||
)
|
||||
):
|
||||
result = result[:-1]
|
||||
return self.tokenizer.decode(result, *args[1:], **kwargs).strip()
|
||||
|
||||
def _convert(self, input_str, max_inp_length: int | None = None, **kwargs):
|
||||
input_ids = self.tokenizer.encode(input_str, **kwargs)
|
||||
if max_inp_length is not None:
|
||||
input_ids = input_ids[:max_inp_length]
|
||||
input_ids = torch.tensor(input_ids, dtype=torch.int32)
|
||||
|
||||
## image bound
|
||||
start_cond = (input_ids == self.tokenizer.im_start_id) | (
|
||||
input_ids == self.tokenizer.slice_start_id
|
||||
)
|
||||
end_cond = (input_ids == self.tokenizer.im_end_id) | (
|
||||
input_ids == self.tokenizer.slice_end_id
|
||||
)
|
||||
|
||||
image_start_idx = torch.where(start_cond)[0]
|
||||
image_start_idx += 1
|
||||
image_end_idx = torch.where(end_cond)[0]
|
||||
|
||||
assert len(image_start_idx) == len(image_end_idx), (
|
||||
f"The number of image start tokens ({len(image_start_idx)}) "
|
||||
f"and end tokens ({len(image_end_idx)}) must match."
|
||||
)
|
||||
|
||||
image_bounds = torch.hstack(
|
||||
[
|
||||
image_start_idx.unsqueeze(-1),
|
||||
image_end_idx.unsqueeze(-1),
|
||||
]
|
||||
)
|
||||
|
||||
## audio bound
|
||||
audio_start_idx = torch.where(input_ids == self.tokenizer.audio_start_id)[0]
|
||||
audio_end_idx = torch.where(input_ids == self.tokenizer.audio_end_id)[0]
|
||||
assert len(audio_start_idx) == len(audio_end_idx)
|
||||
audio_bounds = torch.hstack(
|
||||
[(audio_start_idx + 1).unsqueeze(-1), audio_end_idx.unsqueeze(-1)]
|
||||
)
|
||||
|
||||
spk_start_idx = torch.where(input_ids == self.tokenizer.spk_start_id)[0]
|
||||
spk_end_idx = torch.where(input_ids == self.tokenizer.spk_end_id)[0]
|
||||
assert len(spk_start_idx) == len(spk_end_idx)
|
||||
spk_bounds = torch.hstack(
|
||||
[(spk_start_idx + 1).unsqueeze(-1), spk_end_idx.unsqueeze(-1)]
|
||||
)
|
||||
|
||||
return input_ids, image_bounds, audio_bounds, spk_bounds
|
||||
|
||||
def _convert_omni_to_inputs(
|
||||
self,
|
||||
images,
|
||||
audio_phs,
|
||||
texts: str | list[str],
|
||||
truncation=None,
|
||||
max_length=None,
|
||||
max_slice_nums=None,
|
||||
use_image_id=None,
|
||||
return_tensors=None,
|
||||
**kwargs,
|
||||
):
|
||||
if images is None and audio_phs is None:
|
||||
model_inputs = self.tokenizer(
|
||||
texts,
|
||||
return_tensors=return_tensors,
|
||||
truncation=truncation,
|
||||
max_length=max_length,
|
||||
**kwargs,
|
||||
)
|
||||
return MiniCPMOBatchFeature(data={**model_inputs})
|
||||
|
||||
image_tag = "(<image>./</image>)"
|
||||
image_pattern = r"\(<image>./</image>\)"
|
||||
audio_tag = "(<audio>./</audio>)"
|
||||
audio_pattern = r"\(<audio>./</audio>\)"
|
||||
split_pattern = rf"({image_pattern}|{audio_pattern})"
|
||||
|
||||
if isinstance(texts, str):
|
||||
texts = [texts]
|
||||
|
||||
bs = len(texts)
|
||||
if images is not None:
|
||||
images, image_sizes, tgt_sizes = (
|
||||
images["pixel_values"],
|
||||
images["image_sizes"],
|
||||
images["tgt_sizes"],
|
||||
)
|
||||
else:
|
||||
images, image_sizes, tgt_sizes = [[]] * bs, [[]] * bs, [[]] * bs
|
||||
|
||||
input_ids_list = []
|
||||
image_bounds_list = []
|
||||
audio_bounds_list = []
|
||||
spk_bounds_list = []
|
||||
|
||||
for index, text in enumerate(texts):
|
||||
text_chunks = regex.split(split_pattern, text)
|
||||
|
||||
image_tags = regex.findall(image_pattern, text)
|
||||
audio_tags = regex.findall(audio_pattern, text)
|
||||
|
||||
if image_tags:
|
||||
assert images is not None
|
||||
assert len(image_tags) == len(image_sizes[index])
|
||||
if audio_tags:
|
||||
assert audio_phs is not None
|
||||
assert len(audio_tags) == len(audio_phs[index])
|
||||
|
||||
image_id = 0
|
||||
audio_id = 0
|
||||
for i, chunk in enumerate(text_chunks):
|
||||
if chunk == image_tag:
|
||||
image_placeholder = (
|
||||
self.image_processor.get_slice_image_placeholder(
|
||||
image_sizes[index][image_id],
|
||||
image_id,
|
||||
max_slice_nums,
|
||||
use_image_id,
|
||||
)
|
||||
)
|
||||
image_id += 1
|
||||
text_chunks[i] = image_placeholder
|
||||
elif chunk == audio_tag:
|
||||
audio_placeholder = audio_phs[index][audio_id]
|
||||
audio_id += 1
|
||||
text_chunks[i] = audio_placeholder
|
||||
|
||||
final_text = "".join(text_chunks)
|
||||
input_ids, image_bounds, audio_bounds, spk_bounds = self._convert(
|
||||
final_text, max_length, **kwargs
|
||||
)
|
||||
|
||||
input_ids_list.append(input_ids)
|
||||
image_bounds_list.append(image_bounds)
|
||||
audio_bounds_list.append(audio_bounds)
|
||||
spk_bounds_list.append(spk_bounds)
|
||||
|
||||
padded_input_ids, padding_lengths = self.pad(
|
||||
input_ids_list, padding_side="left"
|
||||
)
|
||||
attention_mask = torch.ones_like(padded_input_ids, dtype=torch.bool)
|
||||
for i, length in enumerate(padding_lengths):
|
||||
image_bounds_list[i] = image_bounds_list[i] + length
|
||||
audio_bounds_list[i] = audio_bounds_list[i] + length
|
||||
spk_bounds_list[i] = spk_bounds_list[i] + length
|
||||
attention_mask[i, :length] = False
|
||||
|
||||
data = {
|
||||
"input_ids": padded_input_ids,
|
||||
"attention_mask": attention_mask,
|
||||
"pixel_values": images,
|
||||
"image_sizes": image_sizes,
|
||||
"image_bound": image_bounds_list,
|
||||
"tgt_sizes": tgt_sizes,
|
||||
"audio_bounds": audio_bounds_list,
|
||||
"spk_bounds": spk_bounds_list,
|
||||
}
|
||||
|
||||
return data
|
||||
|
||||
@property
|
||||
# Copied from
|
||||
# transformers.models.clip.processing_clip.CLIPProcessor.model_input_names
|
||||
def model_input_names(self):
|
||||
tokenizer_input_names = self.tokenizer.model_input_names
|
||||
image_processor_input_names = self.image_processor.model_input_names
|
||||
feature_extractor_input_names = self.feature_extractor.model_input_names
|
||||
return list(
|
||||
dict.fromkeys(
|
||||
tokenizer_input_names
|
||||
+ image_processor_input_names
|
||||
+ feature_extractor_input_names
|
||||
)
|
||||
)
|
||||
|
||||
def pad(
|
||||
self,
|
||||
inputs,
|
||||
max_length=None,
|
||||
padding_value=0,
|
||||
padding_side="left",
|
||||
):
|
||||
if not inputs:
|
||||
return torch.empty(0), []
|
||||
|
||||
items = []
|
||||
if isinstance(inputs[0], list):
|
||||
assert isinstance(inputs[0][0], torch.Tensor)
|
||||
for it in inputs:
|
||||
for tr in it:
|
||||
items.append(tr)
|
||||
else:
|
||||
assert isinstance(inputs[0], torch.Tensor)
|
||||
items = inputs
|
||||
|
||||
batch_size = len(items)
|
||||
shape = items[0].shape
|
||||
dim = len(shape)
|
||||
assert dim <= 2
|
||||
if max_length is None:
|
||||
max_length = 0
|
||||
max_length = max(max_length, max(item.shape[-1] for item in items))
|
||||
min_length = min(item.shape[-1] for item in items)
|
||||
dtype = items[0].dtype
|
||||
|
||||
if dim == 0:
|
||||
return torch.stack([item for item in items], dim=0), [0]
|
||||
elif dim == 1:
|
||||
if max_length == min_length:
|
||||
return (
|
||||
torch.stack([item for item in items], dim=0),
|
||||
[0] * batch_size,
|
||||
)
|
||||
tensor = torch.zeros((batch_size, max_length), dtype=dtype) + padding_value
|
||||
else:
|
||||
tensor = (
|
||||
torch.zeros((batch_size, max_length, shape[-1]), dtype=dtype)
|
||||
+ padding_value
|
||||
)
|
||||
|
||||
padding_length = []
|
||||
for i, item in enumerate(items):
|
||||
if dim == 1:
|
||||
if padding_side == "left":
|
||||
tensor[i, -len(item) :] = item.clone()
|
||||
else:
|
||||
tensor[i, : len(item)] = item.clone()
|
||||
elif dim == 2:
|
||||
if padding_side == "left":
|
||||
tensor[i, -len(item) :, :] = item.clone()
|
||||
else:
|
||||
tensor[i, : len(item), :] = item.clone()
|
||||
padding_length.append(tensor.shape[-1] - len(item))
|
||||
|
||||
return tensor, padding_length
|
||||
@@ -1,317 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
# Copyright 2024 The HuggingFace Inc. team.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
"""Processor class for MiniCPMV."""
|
||||
|
||||
from typing import TypeAlias
|
||||
|
||||
import regex
|
||||
import torch
|
||||
from transformers.image_processing_utils import BatchFeature
|
||||
from transformers.image_utils import ImageInput
|
||||
from transformers.processing_utils import ProcessorMixin
|
||||
from transformers.tokenization_utils_base import (
|
||||
PaddingStrategy,
|
||||
PreTokenizedInput,
|
||||
TextInput,
|
||||
TruncationStrategy,
|
||||
)
|
||||
from transformers.utils import TensorType
|
||||
|
||||
MiniCPMVBatchFeature: TypeAlias = BatchFeature
|
||||
|
||||
|
||||
class MiniCPMVProcessor(ProcessorMixin):
|
||||
r"""Constructs a MiniCPMV processor which wraps a MiniCPMV image
|
||||
processor and a MiniCPMV tokenizer into a single processor.
|
||||
|
||||
[`MiniCPMVProcessor`] offers all the functionalities of
|
||||
[`MiniCPMVImageProcessor`] and [`LlamaTokenizerWrapper`]. See the
|
||||
[`~MiniCPMVProcessor.__call__`] and [`~MiniCPMVProcessor.decode`]
|
||||
for more information.
|
||||
|
||||
Args:
|
||||
image_processor ([`MiniCPMVImageProcessor`], *optional*):
|
||||
The image processor is a required input.
|
||||
tokenizer ([`LlamaTokenizerWrapper`], *optional*):
|
||||
The tokenizer is a required input.
|
||||
|
||||
"""
|
||||
|
||||
attributes = ["image_processor", "tokenizer"]
|
||||
image_processor_class = "AutoImageProcessor"
|
||||
tokenizer_class = "AutoTokenizer"
|
||||
|
||||
def __init__(self, image_processor=None, tokenizer=None, version=None):
|
||||
super().__init__(image_processor, tokenizer)
|
||||
self.version = version
|
||||
|
||||
def __call__(
|
||||
self,
|
||||
text: TextInput | PreTokenizedInput | list[TextInput] | list[PreTokenizedInput],
|
||||
images: ImageInput = None,
|
||||
padding: bool | str | PaddingStrategy = False,
|
||||
truncation: bool | str | TruncationStrategy = None,
|
||||
max_length: int | None = None,
|
||||
do_pad: bool | None = True,
|
||||
return_tensors: str | TensorType | None = TensorType.PYTORCH,
|
||||
) -> MiniCPMVBatchFeature:
|
||||
"""Run the vendored MiniCPMV processor on a (text, images) pair.
|
||||
|
||||
Batched inputs are supported following the upstream MiniCPM-V
|
||||
processor flow. ``images`` is forwarded to the underlying image
|
||||
processor and ``text`` is tokenized with image placeholders
|
||||
replaced by the appropriate slice tokens. Returns a
|
||||
``MiniCPMVBatchFeature`` with at minimum ``input_ids`` and (when
|
||||
images are provided) ``pixel_values``, ``image_sizes``,
|
||||
``image_bound`` and ``tgt_sizes``.
|
||||
"""
|
||||
if images is not None:
|
||||
image_inputs = self.image_processor(
|
||||
images, do_pad=do_pad, return_tensors=return_tensors
|
||||
)
|
||||
else:
|
||||
image_inputs = {}
|
||||
return self._convert_images_texts_to_inputs(
|
||||
image_inputs, text, max_length=max_length
|
||||
)
|
||||
|
||||
# Copied from transformers.models.clip.processing_clip.CLIPProcessor
|
||||
# .batch_decode with CLIP->Llama
|
||||
def batch_decode(self, *args, **kwargs):
|
||||
"""This method forwards all its arguments to LlamaTokenizerFast's
|
||||
[`~PythonBackend.batch_decode`]. Please refer to the
|
||||
docstring of this method for more information.
|
||||
"""
|
||||
output_ids = args[0]
|
||||
result_text = []
|
||||
|
||||
bos_id = getattr(
|
||||
self.tokenizer,
|
||||
"bos_token_id",
|
||||
getattr(self.tokenizer, "bos_id", 1),
|
||||
)
|
||||
eos_id = getattr(
|
||||
self.tokenizer,
|
||||
"eos_token_id",
|
||||
getattr(self.tokenizer, "eos_id", 2),
|
||||
)
|
||||
|
||||
for result in output_ids:
|
||||
result = result[result != 0]
|
||||
if len(result) > 0 and result[0] == bos_id:
|
||||
result = result[1:]
|
||||
if len(result) > 0 and result[-1] == eos_id:
|
||||
result = result[:-1]
|
||||
result_text.append(
|
||||
self.tokenizer.decode(result, *args[1:], **kwargs).strip()
|
||||
)
|
||||
return result_text
|
||||
|
||||
# Copied from transformers.models.clip.processing_clip.CLIPProcessor
|
||||
# .decode with CLIP->Llama
|
||||
def decode(self, *args, **kwargs):
|
||||
"""This method forwards all its arguments to LlamaTokenizerFast's
|
||||
[`~PythonBackend.decode`]. Please refer to the docstring
|
||||
of this method for more information.
|
||||
"""
|
||||
result = args[0]
|
||||
result = result[result != 0]
|
||||
|
||||
bos_id = getattr(
|
||||
self.tokenizer,
|
||||
"bos_token_id",
|
||||
getattr(self.tokenizer, "bos_id", 1),
|
||||
)
|
||||
eos_id = getattr(
|
||||
self.tokenizer,
|
||||
"eos_token_id",
|
||||
getattr(self.tokenizer, "eos_id", 2),
|
||||
)
|
||||
eot_id = getattr(self.tokenizer, "eot_id", None)
|
||||
|
||||
if len(result) > 0 and result[0] == bos_id:
|
||||
result = result[1:]
|
||||
if len(result) > 0 and (
|
||||
result[-1] == eos_id or (eot_id is not None and result[-1] == eot_id)
|
||||
):
|
||||
result = result[:-1]
|
||||
return self.tokenizer.decode(result, *args[1:], **kwargs).strip()
|
||||
|
||||
def _convert(self, input_str, max_inp_length: int | None = None):
|
||||
add_bos = getattr(self.tokenizer, "add_bos_token", False)
|
||||
if self.version == (2, 5) or add_bos:
|
||||
input_ids = self.tokenizer.encode(input_str)
|
||||
else:
|
||||
bos_id = getattr(
|
||||
self.tokenizer,
|
||||
"bos_token_id",
|
||||
getattr(self.tokenizer, "bos_id", 1),
|
||||
)
|
||||
input_ids = [bos_id] + self.tokenizer.encode(input_str)
|
||||
|
||||
if max_inp_length is not None:
|
||||
input_ids = input_ids[:max_inp_length]
|
||||
input_ids = torch.tensor(input_ids, dtype=torch.int32)
|
||||
|
||||
im_start_id = getattr(
|
||||
self.tokenizer,
|
||||
"im_start_id",
|
||||
self.tokenizer.convert_tokens_to_ids("<im_start>"),
|
||||
)
|
||||
im_end_id = getattr(
|
||||
self.tokenizer,
|
||||
"im_end_id",
|
||||
self.tokenizer.convert_tokens_to_ids("<im_end>"),
|
||||
)
|
||||
|
||||
image_start_tokens = torch.where(input_ids == im_start_id)[0]
|
||||
image_start_tokens += 1
|
||||
image_end_tokens = torch.where(input_ids == im_end_id)[0]
|
||||
assert len(image_start_tokens) == len(image_end_tokens), (
|
||||
f"The number of image start tokens ({len(image_start_tokens)}) "
|
||||
f"and end tokens ({len(image_end_tokens)}) must match."
|
||||
)
|
||||
image_bounds = torch.hstack(
|
||||
[
|
||||
image_start_tokens.unsqueeze(-1),
|
||||
image_end_tokens.unsqueeze(-1),
|
||||
]
|
||||
)
|
||||
return input_ids, image_bounds
|
||||
|
||||
def _convert_images_texts_to_inputs(
|
||||
self,
|
||||
images,
|
||||
texts,
|
||||
do_pad=False,
|
||||
truncation=None,
|
||||
max_length=None,
|
||||
return_tensors=None,
|
||||
):
|
||||
if not len(images):
|
||||
model_inputs = self.tokenizer(
|
||||
texts,
|
||||
return_tensors=return_tensors,
|
||||
padding=do_pad,
|
||||
truncation=truncation,
|
||||
max_length=max_length,
|
||||
)
|
||||
return MiniCPMVBatchFeature(data={**model_inputs})
|
||||
|
||||
pattern = "(<image>./</image>)"
|
||||
images_val = images["pixel_values"]
|
||||
image_sizes = images["image_sizes"]
|
||||
tgt_sizes = images["tgt_sizes"]
|
||||
|
||||
if isinstance(texts, str):
|
||||
texts = [texts]
|
||||
|
||||
input_ids_list = []
|
||||
image_bounds_list = []
|
||||
|
||||
for index, text in enumerate(texts):
|
||||
image_tags = regex.findall(pattern, text)
|
||||
assert len(image_tags) == len(image_sizes[index])
|
||||
text_chunks = text.split(pattern)
|
||||
final_text = ""
|
||||
for i in range(len(image_tags)):
|
||||
placeholder = self.image_processor.get_slice_image_placeholder(
|
||||
image_sizes[index][i]
|
||||
)
|
||||
final_text = final_text + text_chunks[i] + placeholder
|
||||
final_text += text_chunks[-1]
|
||||
input_ids, image_bounds = self._convert(final_text, max_length)
|
||||
input_ids_list.append(input_ids)
|
||||
image_bounds_list.append(image_bounds)
|
||||
|
||||
padded_input_ids, padding_lengths = self.pad(
|
||||
input_ids_list,
|
||||
padding_side="left",
|
||||
)
|
||||
for i, length in enumerate(padding_lengths):
|
||||
image_bounds_list[i] = image_bounds_list[i] + length
|
||||
|
||||
return MiniCPMVBatchFeature(
|
||||
data={
|
||||
"input_ids": padded_input_ids,
|
||||
"attention_mask": padded_input_ids.ne(0),
|
||||
"pixel_values": images_val,
|
||||
"image_sizes": image_sizes,
|
||||
"image_bound": image_bounds_list,
|
||||
"tgt_sizes": tgt_sizes,
|
||||
}
|
||||
)
|
||||
|
||||
@property
|
||||
# Copied from
|
||||
# transformers.models.clip.processing_clip.CLIPProcessor.model_input_names
|
||||
def model_input_names(self):
|
||||
tokenizer_input_names = self.tokenizer.model_input_names
|
||||
image_processor_input_names = self.image_processor.model_input_names
|
||||
return list(dict.fromkeys(tokenizer_input_names + image_processor_input_names))
|
||||
|
||||
# Copied from openbmb/MiniCPM-V-4_5 processing_minicpmv.py.
|
||||
def pad(self, inputs, max_length=None, padding_value=0, padding_side="left"):
|
||||
if not inputs:
|
||||
return torch.empty(0), []
|
||||
|
||||
items = []
|
||||
if isinstance(inputs[0], list):
|
||||
assert isinstance(inputs[0][0], torch.Tensor)
|
||||
for it in inputs:
|
||||
for tr in it:
|
||||
items.append(tr)
|
||||
else:
|
||||
assert isinstance(inputs[0], torch.Tensor)
|
||||
items = inputs
|
||||
|
||||
batch_size = len(items)
|
||||
shape = items[0].shape
|
||||
dim = len(shape)
|
||||
assert dim <= 2
|
||||
if max_length is None:
|
||||
max_length = 0
|
||||
max_length = max(max_length, max(item.shape[-1] for item in items))
|
||||
min_length = min(item.shape[-1] for item in items)
|
||||
dtype = items[0].dtype
|
||||
|
||||
if dim == 0:
|
||||
return torch.stack([item for item in items], dim=0), [0]
|
||||
elif dim == 1:
|
||||
if max_length == min_length:
|
||||
return torch.stack([item for item in items], dim=0), [0] * batch_size
|
||||
tensor = torch.zeros((batch_size, max_length), dtype=dtype) + padding_value
|
||||
else:
|
||||
tensor = (
|
||||
torch.zeros((batch_size, max_length, shape[-1]), dtype=dtype)
|
||||
+ padding_value
|
||||
)
|
||||
|
||||
padding_lengths = []
|
||||
for i, item in enumerate(items):
|
||||
if dim == 1:
|
||||
if padding_side == "left":
|
||||
tensor[i, -len(item) :] = item.clone()
|
||||
else:
|
||||
tensor[i, : len(item)] = item.clone()
|
||||
elif dim == 2:
|
||||
if padding_side == "left":
|
||||
tensor[i, -len(item) :, :] = item.clone()
|
||||
else:
|
||||
tensor[i, : len(item), :] = item.clone()
|
||||
padding_lengths.append(tensor.shape[-1] - len(item))
|
||||
|
||||
return tensor, padding_lengths
|
||||
@@ -1,660 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""MiniMax M3 VL HuggingFace-compatible Processor / ImageProcessor /
|
||||
VideoProcessor, vendored into vLLM so the model loads without
|
||||
``--trust-remote-code`` (the released checkpoint only ships these classes as
|
||||
remote code via ``auto_map``).
|
||||
|
||||
Adapted verbatim from the ``MiniMaxAI/Minimax-M3-preview`` repository files
|
||||
``image_processor.py``, ``video_processor.py`` and ``processing_minimax.py``
|
||||
(revision ``db01c0fe``). Both image and video processors use Qwen-style
|
||||
``smart_resize`` (bound by total pixels). The original async frame-sampling
|
||||
helpers are intentionally omitted: vLLM performs its own frame loading and
|
||||
feeds decoded frames to the processor.
|
||||
"""
|
||||
|
||||
import math
|
||||
|
||||
import torch
|
||||
from torchvision.transforms import InterpolationMode
|
||||
from transformers import AutoTokenizer, BatchFeature
|
||||
from transformers.image_processing_backends import (
|
||||
TorchvisionBackend,
|
||||
group_images_by_shape,
|
||||
reorder_images,
|
||||
)
|
||||
from transformers.image_utils import PILImageResampling, SizeDict
|
||||
from transformers.processing_utils import (
|
||||
ImagesKwargs,
|
||||
ProcessingKwargs,
|
||||
ProcessorMixin,
|
||||
Unpack,
|
||||
VideosKwargs,
|
||||
)
|
||||
from transformers.utils import TensorType
|
||||
from transformers.video_processing_utils import BaseVideoProcessor
|
||||
from transformers.video_utils import group_videos_by_shape, reorder_videos
|
||||
|
||||
# Maximum allowed aspect ratio before smart_resize rejects the input.
|
||||
MAX_RATIO = 200
|
||||
|
||||
# Fixed (non-configurable) bounds for the long-side resize logic, per the
|
||||
# MiniMax-M3 size spec. ``min_short_side_pixel`` is the floor the short edge is
|
||||
# enlarged to; ``*_MAX_TOTAL_PIXELS`` is the hard area cap that, once exceeded,
|
||||
# aborts processing instead of downscaling.
|
||||
MIN_SHORT_SIDE_PIXEL = 112
|
||||
IMAGE_MAX_TOTAL_PIXELS = 12_845_056 # 3584 ** 2 (width * height)
|
||||
VIDEO_MAX_TOTAL_PIXELS = 301_056_000 # width * height * frames
|
||||
|
||||
|
||||
def round_by_factor(number: int | float, factor: int) -> int:
|
||||
return round(number / factor) * factor
|
||||
|
||||
|
||||
def ceil_by_factor(number: int | float, factor: int) -> int:
|
||||
return math.ceil(number / factor) * factor
|
||||
|
||||
|
||||
def floor_by_factor(number: int | float, factor: int) -> int:
|
||||
return math.floor(number / factor) * factor
|
||||
|
||||
|
||||
def _smart_resize_by_long_side(
|
||||
height: int,
|
||||
width: int,
|
||||
factor: int,
|
||||
max_long_side_pixel: int,
|
||||
min_short_side_pixel: int,
|
||||
max_total_pixels: int | None,
|
||||
) -> tuple[int, int]:
|
||||
"""Long-side based resize (MiniMax-M3 size spec).
|
||||
|
||||
(a) if the long side exceeds ``max_long_side_pixel`` → shrink so the long
|
||||
side equals ``max_long_side_pixel``;
|
||||
(b) else if the short side is below ``min_short_side_pixel`` → enlarge so the
|
||||
short side equals ``min_short_side_pixel``;
|
||||
(c) if the resulting area still exceeds ``max_total_pixels`` → raise.
|
||||
|
||||
(a) and (b) are mutually exclusive (they branch on the *original* long side).
|
||||
Both sides are then rounded to a multiple of ``factor``. For videos the
|
||||
``max_total_pixels`` cap is volumetric (width * height * frames) and is
|
||||
enforced by the caller, so pass ``max_total_pixels=None`` here.
|
||||
"""
|
||||
long_side = max(height, width)
|
||||
short_side = min(height, width)
|
||||
|
||||
scaled_height: float = height
|
||||
scaled_width: float = width
|
||||
if long_side > max_long_side_pixel:
|
||||
beta = max_long_side_pixel / long_side
|
||||
scaled_height = height * beta
|
||||
scaled_width = width * beta
|
||||
elif short_side < min_short_side_pixel:
|
||||
beta = min_short_side_pixel / short_side
|
||||
scaled_height = height * beta
|
||||
scaled_width = width * beta
|
||||
|
||||
h_bar = max(factor, round_by_factor(scaled_height, factor))
|
||||
w_bar = max(factor, round_by_factor(scaled_width, factor))
|
||||
|
||||
if max_total_pixels is not None and h_bar * w_bar > max_total_pixels:
|
||||
raise ValueError(
|
||||
f"image area {h_bar * w_bar} exceeds max_total_pixels "
|
||||
f"{max_total_pixels} after resizing"
|
||||
)
|
||||
return h_bar, w_bar
|
||||
|
||||
|
||||
def smart_resize(
|
||||
height: int,
|
||||
width: int,
|
||||
factor: int = 28,
|
||||
min_pixels: int = 4 * 28 * 28,
|
||||
max_pixels: int = 451584,
|
||||
max_long_side_pixel: int | None = None,
|
||||
min_short_side_pixel: int = MIN_SHORT_SIDE_PIXEL,
|
||||
max_total_pixels: int | None = None,
|
||||
) -> tuple[int, int]:
|
||||
"""Rescale (height, width) so each side is a multiple of ``factor``.
|
||||
|
||||
When ``max_long_side_pixel`` is set, use the MiniMax-M3 long-side resize
|
||||
spec (see :func:`_smart_resize_by_long_side`). Otherwise fall back to the
|
||||
Qwen-VL area bound, keeping the total area within ``[min_pixels, max_pixels]``.
|
||||
"""
|
||||
if max(height, width) / min(height, width) > MAX_RATIO:
|
||||
raise ValueError(
|
||||
f"absolute aspect ratio must be smaller than {MAX_RATIO}, "
|
||||
f"got {max(height, width) / min(height, width)}"
|
||||
)
|
||||
if max_long_side_pixel is not None:
|
||||
return _smart_resize_by_long_side(
|
||||
height,
|
||||
width,
|
||||
factor=factor,
|
||||
max_long_side_pixel=max_long_side_pixel,
|
||||
min_short_side_pixel=min_short_side_pixel,
|
||||
max_total_pixels=max_total_pixels,
|
||||
)
|
||||
h_bar = max(factor, round_by_factor(height, factor))
|
||||
w_bar = max(factor, round_by_factor(width, factor))
|
||||
if h_bar * w_bar > max_pixels:
|
||||
beta = math.sqrt((height * width) / max_pixels)
|
||||
h_bar = floor_by_factor(height / beta, factor)
|
||||
w_bar = floor_by_factor(width / beta, factor)
|
||||
elif h_bar * w_bar < min_pixels:
|
||||
beta = math.sqrt(min_pixels / (height * width))
|
||||
h_bar = ceil_by_factor(height * beta, factor)
|
||||
w_bar = ceil_by_factor(width * beta, factor)
|
||||
return h_bar, w_bar
|
||||
|
||||
|
||||
class MiniMaxM3VLImageProcessorKwargs(ImagesKwargs, total=False): # type: ignore[call-arg]
|
||||
patch_size: int
|
||||
temporal_patch_size: int
|
||||
merge_size: int
|
||||
max_pixels: int
|
||||
max_long_side_pixel: int
|
||||
|
||||
|
||||
class MiniMaxM3VLImageProcessor(TorchvisionBackend):
|
||||
do_resize = True
|
||||
resample = PILImageResampling.BICUBIC
|
||||
# required by base-class validation, not used as the resize bound
|
||||
size = {"height": 672, "width": 672}
|
||||
default_to_square = False
|
||||
do_rescale = True
|
||||
rescale_factor = 1 / 255
|
||||
do_normalize = True
|
||||
image_mean = [0.48145466, 0.4578275, 0.40821073]
|
||||
image_std = [0.26862954, 0.26130258, 0.27577711]
|
||||
do_convert_rgb = True
|
||||
patch_size = 14
|
||||
temporal_patch_size = 2
|
||||
merge_size = 2
|
||||
max_pixels = 451584 # 672 * 672
|
||||
# Long-side resize spec (opt-in via ``max_long_side_pixel``). The latter two
|
||||
# are fixed per the spec and are not exposed as configurable kwargs.
|
||||
max_long_side_pixel = None
|
||||
min_short_side_pixel = MIN_SHORT_SIDE_PIXEL
|
||||
max_total_pixels = IMAGE_MAX_TOTAL_PIXELS
|
||||
valid_kwargs = MiniMaxM3VLImageProcessorKwargs
|
||||
model_input_names = ["pixel_values", "image_grid_thw"]
|
||||
|
||||
def __init__(self, **kwargs: Unpack[MiniMaxM3VLImageProcessorKwargs]):
|
||||
super().__init__(**kwargs)
|
||||
|
||||
def preprocess(
|
||||
self, images, **kwargs: Unpack[MiniMaxM3VLImageProcessorKwargs]
|
||||
) -> BatchFeature:
|
||||
return super().preprocess(images, **kwargs)
|
||||
|
||||
def _preprocess(
|
||||
self,
|
||||
images: list[torch.Tensor],
|
||||
do_resize: bool,
|
||||
size: SizeDict,
|
||||
resample: "PILImageResampling | InterpolationMode | int | None",
|
||||
do_rescale: bool,
|
||||
rescale_factor: float,
|
||||
do_normalize: bool,
|
||||
image_mean: "float | list[float] | None",
|
||||
image_std: "float | list[float] | None",
|
||||
patch_size: int,
|
||||
temporal_patch_size: int,
|
||||
merge_size: int,
|
||||
max_pixels: int,
|
||||
max_long_side_pixel: "int | None",
|
||||
disable_grouping: "bool | None",
|
||||
return_tensors: "str | TensorType | None",
|
||||
**kwargs,
|
||||
) -> BatchFeature:
|
||||
grouped_images, grouped_images_index = group_images_by_shape(
|
||||
images, disable_grouping=disable_grouping
|
||||
)
|
||||
resized_images_grouped = {}
|
||||
factor = patch_size * merge_size
|
||||
for shape, stacked_images in grouped_images.items():
|
||||
height, width = stacked_images.shape[-2:]
|
||||
if do_resize:
|
||||
resized_height, resized_width = smart_resize(
|
||||
height,
|
||||
width,
|
||||
factor=factor,
|
||||
max_pixels=max_pixels,
|
||||
max_long_side_pixel=max_long_side_pixel,
|
||||
min_short_side_pixel=self.min_short_side_pixel,
|
||||
max_total_pixels=self.max_total_pixels,
|
||||
)
|
||||
stacked_images = self.resize(
|
||||
stacked_images,
|
||||
size=SizeDict(height=resized_height, width=resized_width),
|
||||
resample=resample,
|
||||
)
|
||||
resized_images_grouped[shape] = stacked_images
|
||||
|
||||
resized_images = reorder_images(resized_images_grouped, grouped_images_index)
|
||||
|
||||
grouped_images, grouped_images_index = group_images_by_shape(
|
||||
resized_images, disable_grouping=disable_grouping
|
||||
)
|
||||
processed_images_grouped = {}
|
||||
processed_grids = {}
|
||||
|
||||
for shape, stacked_images in grouped_images.items():
|
||||
resized_height, resized_width = stacked_images.shape[-2:]
|
||||
|
||||
patches = self.rescale_and_normalize(
|
||||
stacked_images,
|
||||
do_rescale,
|
||||
rescale_factor,
|
||||
do_normalize,
|
||||
image_mean,
|
||||
image_std,
|
||||
)
|
||||
if patches.ndim == 4:
|
||||
patches = patches.unsqueeze(1)
|
||||
|
||||
if patches.shape[1] % temporal_patch_size != 0:
|
||||
repeats = patches[:, -1:].repeat(
|
||||
1,
|
||||
temporal_patch_size - (patches.shape[1] % temporal_patch_size),
|
||||
1,
|
||||
1,
|
||||
1,
|
||||
)
|
||||
patches = torch.cat([patches, repeats], dim=1)
|
||||
|
||||
batch_size, grid_t, channel = patches.shape[:3]
|
||||
grid_t = grid_t // temporal_patch_size
|
||||
grid_h, grid_w = resized_height // patch_size, resized_width // patch_size
|
||||
|
||||
patches = patches.view(
|
||||
batch_size,
|
||||
grid_t,
|
||||
temporal_patch_size,
|
||||
channel,
|
||||
grid_h // merge_size,
|
||||
merge_size,
|
||||
patch_size,
|
||||
grid_w // merge_size,
|
||||
merge_size,
|
||||
patch_size,
|
||||
)
|
||||
patches = patches.permute(0, 1, 4, 7, 5, 8, 3, 2, 6, 9)
|
||||
|
||||
flatten_patches = patches.reshape(
|
||||
batch_size,
|
||||
grid_t * grid_h * grid_w,
|
||||
channel * temporal_patch_size * patch_size * patch_size,
|
||||
)
|
||||
|
||||
processed_images_grouped[shape] = flatten_patches
|
||||
processed_grids[shape] = [[grid_t, grid_h, grid_w]] * batch_size
|
||||
|
||||
processed_images = reorder_images(
|
||||
processed_images_grouped, grouped_images_index
|
||||
)
|
||||
processed_grids = reorder_images(processed_grids, grouped_images_index)
|
||||
|
||||
pixel_values = torch.cat(processed_images, dim=0)
|
||||
image_grid_thw = torch.tensor(processed_grids, dtype=torch.long)
|
||||
|
||||
return BatchFeature(
|
||||
data={"pixel_values": pixel_values, "image_grid_thw": image_grid_thw},
|
||||
tensor_type=return_tensors,
|
||||
)
|
||||
|
||||
def get_number_of_image_patches(self, height: int, width: int, images_kwargs=None):
|
||||
images_kwargs = images_kwargs or {}
|
||||
patch_size = images_kwargs.get("patch_size", self.patch_size)
|
||||
merge_size = images_kwargs.get("merge_size", self.merge_size)
|
||||
max_pixels = images_kwargs.get("max_pixels", self.max_pixels)
|
||||
max_long_side_pixel = images_kwargs.get(
|
||||
"max_long_side_pixel", self.max_long_side_pixel
|
||||
)
|
||||
|
||||
resized_height, resized_width = smart_resize(
|
||||
height,
|
||||
width,
|
||||
factor=patch_size * merge_size,
|
||||
max_pixels=max_pixels,
|
||||
max_long_side_pixel=max_long_side_pixel,
|
||||
min_short_side_pixel=self.min_short_side_pixel,
|
||||
max_total_pixels=self.max_total_pixels,
|
||||
)
|
||||
grid_h, grid_w = resized_height // patch_size, resized_width // patch_size
|
||||
return grid_h * grid_w
|
||||
|
||||
|
||||
class MiniMaxM3VLVideoProcessorKwargs(VideosKwargs, total=False): # type: ignore[call-arg]
|
||||
patch_size: int
|
||||
temporal_patch_size: int
|
||||
merge_size: int
|
||||
min_pixels: int
|
||||
max_pixels: int
|
||||
max_long_side_pixel: int
|
||||
total_pixels: int
|
||||
min_frames: int
|
||||
max_frames: int
|
||||
fps: "float | int"
|
||||
|
||||
|
||||
class MiniMaxM3VLVideoProcessor(BaseVideoProcessor):
|
||||
do_resize = True
|
||||
resample = PILImageResampling.BICUBIC
|
||||
size = {"height": 672, "width": 672}
|
||||
default_to_square = False
|
||||
do_rescale = True
|
||||
rescale_factor = 1 / 255
|
||||
do_normalize = True
|
||||
image_mean = [0.48145466, 0.4578275, 0.40821073]
|
||||
image_std = [0.26862954, 0.26130258, 0.27577711]
|
||||
do_convert_rgb = True
|
||||
do_sample_frames = False
|
||||
patch_size = 14
|
||||
temporal_patch_size = 2
|
||||
merge_size = 2
|
||||
min_pixels = 4 * 28 * 28
|
||||
max_pixels = 768 * 28 * 28 # 602,112
|
||||
total_pixels = int(64000 * 28 * 28 * 0.9) # ~45M, ~64k tokens budget
|
||||
# Long-side resize spec (opt-in via ``max_long_side_pixel``). The video
|
||||
# ``max_total_pixels`` cap is volumetric (width * height * frames) and is
|
||||
# enforced in ``_preprocess`` once the frame count is known.
|
||||
max_long_side_pixel = None
|
||||
min_short_side_pixel = MIN_SHORT_SIDE_PIXEL
|
||||
max_total_pixels = VIDEO_MAX_TOTAL_PIXELS
|
||||
fps = 1.0
|
||||
min_frames = 4
|
||||
max_frames = 768
|
||||
valid_kwargs = MiniMaxM3VLVideoProcessorKwargs
|
||||
model_input_names = ["pixel_values_videos", "video_grid_thw"]
|
||||
|
||||
def __init__(self, **kwargs: Unpack[MiniMaxM3VLVideoProcessorKwargs]):
|
||||
super().__init__(**kwargs)
|
||||
|
||||
def _preprocess(
|
||||
self,
|
||||
videos: list[torch.Tensor],
|
||||
do_resize: bool,
|
||||
size: SizeDict,
|
||||
resample: "PILImageResampling | InterpolationMode | int | None",
|
||||
do_rescale: bool,
|
||||
rescale_factor: float,
|
||||
do_normalize: bool,
|
||||
image_mean: "float | list[float] | None",
|
||||
image_std: "float | list[float] | None",
|
||||
patch_size: int,
|
||||
temporal_patch_size: int,
|
||||
merge_size: int,
|
||||
min_pixels: int,
|
||||
max_pixels: int,
|
||||
max_long_side_pixel: "int | None" = None,
|
||||
return_tensors: "str | TensorType | None" = None,
|
||||
**kwargs,
|
||||
) -> BatchFeature:
|
||||
grouped_videos, grouped_videos_index = group_videos_by_shape(videos)
|
||||
resized_videos_grouped = {}
|
||||
factor = patch_size * merge_size
|
||||
for shape, stacked_videos in grouped_videos.items():
|
||||
batch_size, num_frames, channels, height, width = stacked_videos.shape
|
||||
resized_height, resized_width = height, width
|
||||
if do_resize:
|
||||
resized_height, resized_width = smart_resize(
|
||||
height,
|
||||
width,
|
||||
factor=factor,
|
||||
min_pixels=min_pixels,
|
||||
max_pixels=max_pixels,
|
||||
max_long_side_pixel=max_long_side_pixel,
|
||||
min_short_side_pixel=self.min_short_side_pixel,
|
||||
# Per-frame raise disabled; the video cap is volumetric and
|
||||
# is enforced below once num_frames is known.
|
||||
max_total_pixels=None,
|
||||
)
|
||||
if (
|
||||
max_long_side_pixel is not None
|
||||
and resized_height * resized_width * num_frames
|
||||
> self.max_total_pixels
|
||||
):
|
||||
raise ValueError(
|
||||
f"video area {resized_height * resized_width * num_frames} "
|
||||
f"(width * height * frames) exceeds max_total_pixels "
|
||||
f"{self.max_total_pixels} after resizing"
|
||||
)
|
||||
stacked_videos = stacked_videos.view(
|
||||
batch_size * num_frames, channels, height, width
|
||||
)
|
||||
stacked_videos = self.resize(
|
||||
stacked_videos,
|
||||
size=SizeDict(height=resized_height, width=resized_width),
|
||||
resample=resample,
|
||||
)
|
||||
stacked_videos = stacked_videos.view(
|
||||
batch_size,
|
||||
num_frames,
|
||||
channels,
|
||||
resized_height,
|
||||
resized_width,
|
||||
)
|
||||
resized_videos_grouped[shape] = stacked_videos
|
||||
resized_videos = reorder_videos(resized_videos_grouped, grouped_videos_index)
|
||||
|
||||
grouped_videos, grouped_videos_index = group_videos_by_shape(resized_videos)
|
||||
processed_videos_grouped = {}
|
||||
processed_grids = {}
|
||||
for shape, stacked_videos in grouped_videos.items():
|
||||
resized_height, resized_width = stacked_videos.shape[-2:]
|
||||
patches = self.rescale_and_normalize(
|
||||
stacked_videos,
|
||||
do_rescale,
|
||||
rescale_factor,
|
||||
do_normalize,
|
||||
image_mean,
|
||||
image_std,
|
||||
)
|
||||
|
||||
if pad := -patches.shape[1] % temporal_patch_size:
|
||||
repeats = patches[:, -1:].expand(-1, pad, -1, -1, -1)
|
||||
patches = torch.cat([patches, repeats], dim=1)
|
||||
|
||||
batch_size, grid_t, channels = patches.shape[:3]
|
||||
grid_t = grid_t // temporal_patch_size
|
||||
grid_h, grid_w = resized_height // patch_size, resized_width // patch_size
|
||||
|
||||
patches = patches.view(
|
||||
batch_size,
|
||||
grid_t,
|
||||
temporal_patch_size,
|
||||
channels,
|
||||
grid_h // merge_size,
|
||||
merge_size,
|
||||
patch_size,
|
||||
grid_w // merge_size,
|
||||
merge_size,
|
||||
patch_size,
|
||||
)
|
||||
patches = patches.permute(0, 1, 4, 7, 5, 8, 3, 2, 6, 9)
|
||||
flatten_patches = patches.reshape(
|
||||
batch_size,
|
||||
grid_t * grid_h * grid_w,
|
||||
channels * temporal_patch_size * patch_size * patch_size,
|
||||
)
|
||||
|
||||
processed_videos_grouped[shape] = flatten_patches
|
||||
processed_grids[shape] = [[grid_t, grid_h, grid_w]] * batch_size
|
||||
|
||||
processed_videos = reorder_videos(
|
||||
processed_videos_grouped, grouped_videos_index
|
||||
)
|
||||
processed_grids = reorder_videos(processed_grids, grouped_videos_index)
|
||||
pixel_values_videos = torch.cat(processed_videos, dim=0)
|
||||
video_grid_thw = torch.tensor(processed_grids, dtype=torch.long)
|
||||
|
||||
return BatchFeature(
|
||||
data={
|
||||
"pixel_values_videos": pixel_values_videos,
|
||||
"video_grid_thw": video_grid_thw,
|
||||
},
|
||||
tensor_type=return_tensors,
|
||||
)
|
||||
|
||||
|
||||
class MiniMaxVLProcessorKwargs(ProcessingKwargs, total=False): # type: ignore[call-arg]
|
||||
_defaults = {
|
||||
"videos_kwargs": {
|
||||
"do_resize": False,
|
||||
"return_metadata": True,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
class MiniMaxVLProcessor(ProcessorMixin):
|
||||
IMAGE_TOKEN = "]<]image[>["
|
||||
VIDEO_TOKEN = "]<]video[>["
|
||||
VISION_START_TOKEN = "]<]start of image[>["
|
||||
VISION_END_TOKEN = "]<]end of image[>["
|
||||
|
||||
@classmethod
|
||||
def from_pretrained(cls, pretrained_model_name_or_path, **kwargs):
|
||||
# Bypass ProcessorMixin's dynamic module lookup, which breaks in
|
||||
# transformers >= 5.9 when image_processor_class is a string: the
|
||||
# register() API now stores classes as {"pil": cls} dicts in
|
||||
# _extra_content, but get_possibly_dynamic_module() still calls
|
||||
# .__name__ on the raw value, crashing with AttributeError on dicts.
|
||||
#
|
||||
# Reuse the tokenizer passed by vLLM instead of loading another one,
|
||||
# and keep it out of the kwargs forwarded to the sub-processor loaders.
|
||||
tokenizer = kwargs.pop("tokenizer", None)
|
||||
if tokenizer is None:
|
||||
tokenizer = AutoTokenizer.from_pretrained(
|
||||
pretrained_model_name_or_path, **kwargs
|
||||
)
|
||||
image_processor = MiniMaxM3VLImageProcessor.from_pretrained(
|
||||
pretrained_model_name_or_path, **kwargs
|
||||
)
|
||||
video_processor = MiniMaxM3VLVideoProcessor.from_pretrained(
|
||||
pretrained_model_name_or_path, **kwargs
|
||||
)
|
||||
return cls(
|
||||
image_processor=image_processor,
|
||||
tokenizer=tokenizer,
|
||||
video_processor=video_processor,
|
||||
)
|
||||
|
||||
def __init__(
|
||||
self, image_processor=None, tokenizer=None, video_processor=None, **kwargs
|
||||
):
|
||||
self.image_token_id = tokenizer.convert_tokens_to_ids(self.IMAGE_TOKEN)
|
||||
self.video_token_id = tokenizer.convert_tokens_to_ids(self.VIDEO_TOKEN)
|
||||
super().__init__(image_processor, tokenizer, video_processor)
|
||||
# Video expansion also uses image start/end tokens. Separate video
|
||||
# start/end tokens exist in the tokenizer, but the original MiniMax
|
||||
# serving path did not use them; keep that behavior for compatibility.
|
||||
self.vision_start_token_id = tokenizer.convert_tokens_to_ids(
|
||||
self.VISION_START_TOKEN
|
||||
)
|
||||
self.vision_end_token_id = tokenizer.convert_tokens_to_ids(
|
||||
self.VISION_END_TOKEN
|
||||
)
|
||||
|
||||
def __call__(
|
||||
self,
|
||||
images=None,
|
||||
text=None,
|
||||
videos=None,
|
||||
**kwargs: Unpack[MiniMaxVLProcessorKwargs],
|
||||
) -> BatchFeature:
|
||||
output_kwargs = self._merge_kwargs(
|
||||
MiniMaxVLProcessorKwargs,
|
||||
tokenizer_init_kwargs=self.tokenizer.init_kwargs,
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
if images is not None:
|
||||
images_kwargs = output_kwargs["images_kwargs"]
|
||||
image_inputs = self.image_processor(images=images, **images_kwargs)
|
||||
image_grid_thw = image_inputs["image_grid_thw"]
|
||||
else:
|
||||
image_inputs = {}
|
||||
image_grid_thw = None
|
||||
|
||||
if videos is not None:
|
||||
videos_kwargs = output_kwargs["videos_kwargs"]
|
||||
video_inputs = self.video_processor(videos=videos, **videos_kwargs)
|
||||
video_grid_thw = video_inputs["video_grid_thw"]
|
||||
if not kwargs.get("return_metadata"):
|
||||
video_metadata = video_inputs.pop("video_metadata")
|
||||
else:
|
||||
video_metadata = video_inputs["video_metadata"]
|
||||
else:
|
||||
video_inputs = {}
|
||||
video_grid_thw = None
|
||||
|
||||
if not isinstance(text, list):
|
||||
text = [text]
|
||||
text = text.copy()
|
||||
|
||||
# Expand image tokens
|
||||
if image_grid_thw is not None:
|
||||
merge_length = self.image_processor.merge_size**2
|
||||
placeholder = "]<]placeholder[>["
|
||||
index = 0
|
||||
for i in range(len(text)):
|
||||
while self.IMAGE_TOKEN in text[i]:
|
||||
num_tokens = image_grid_thw[index].prod() // merge_length
|
||||
text[i] = text[i].replace(
|
||||
self.IMAGE_TOKEN,
|
||||
self.VISION_START_TOKEN
|
||||
+ placeholder * num_tokens
|
||||
+ self.VISION_END_TOKEN,
|
||||
1,
|
||||
)
|
||||
index += 1
|
||||
text[i] = text[i].replace(placeholder, self.IMAGE_TOKEN)
|
||||
|
||||
# Expand video tokens
|
||||
if video_grid_thw is not None:
|
||||
merge_length = self.image_processor.merge_size**2
|
||||
placeholder = "]<]placeholder[>["
|
||||
index = 0
|
||||
for i in range(len(text)):
|
||||
while self.VIDEO_TOKEN in text[i]:
|
||||
metadata = video_metadata[index]
|
||||
grid_t = video_grid_thw[index][0]
|
||||
frame_seqlen = video_grid_thw[index][1:].prod() // merge_length
|
||||
|
||||
video_placeholder = ""
|
||||
for frame_idx in range(grid_t):
|
||||
if (
|
||||
metadata.fps is not None
|
||||
and metadata.frames_indices is not None
|
||||
):
|
||||
ts = (
|
||||
metadata.frames_indices[
|
||||
min(
|
||||
frame_idx
|
||||
* self.video_processor.temporal_patch_size,
|
||||
len(metadata.frames_indices) - 1,
|
||||
)
|
||||
]
|
||||
/ metadata.fps
|
||||
)
|
||||
video_placeholder += f"]<]{ts:.1f} seconds[>["
|
||||
video_placeholder += (
|
||||
self.VISION_START_TOKEN
|
||||
+ placeholder * frame_seqlen
|
||||
+ self.VISION_END_TOKEN
|
||||
)
|
||||
|
||||
text[i] = text[i].replace(self.VIDEO_TOKEN, video_placeholder, 1)
|
||||
index += 1
|
||||
text[i] = text[i].replace(placeholder, self.VIDEO_TOKEN)
|
||||
|
||||
# Tokenize
|
||||
return_tensors = output_kwargs["text_kwargs"].pop("return_tensors", None)
|
||||
text_inputs = self.tokenizer(text, **output_kwargs["text_kwargs"])
|
||||
|
||||
return BatchFeature(
|
||||
data={**text_inputs, **image_inputs, **video_inputs},
|
||||
tensor_type=return_tensors,
|
||||
)
|
||||
@@ -1,539 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Custom processor for Moondream3 model."""
|
||||
|
||||
import math
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
from PIL import Image
|
||||
from transformers import AutoProcessor, BatchFeature
|
||||
from transformers.image_utils import ImageInput
|
||||
from transformers.processing_utils import ProcessingKwargs, ProcessorMixin, Unpack
|
||||
from transformers.tokenization_utils_base import (
|
||||
PreTokenizedInput,
|
||||
PreTrainedTokenizerBase,
|
||||
TextInput,
|
||||
)
|
||||
|
||||
from vllm.multimodal.image import convert_image_mode
|
||||
|
||||
__all__ = ["Moondream3Processor"]
|
||||
|
||||
|
||||
class Moondream3ProcessorKwargs(ProcessingKwargs, total=False): # type: ignore[call-arg]
|
||||
_defaults = {
|
||||
"text_kwargs": {
|
||||
"padding": False,
|
||||
},
|
||||
"images_kwargs": {
|
||||
"max_crops": 12,
|
||||
"overlap_margin": 4,
|
||||
"crop_size": 378,
|
||||
"patch_size": 14,
|
||||
"convert_to_rgb": True,
|
||||
"return_tensors": "pt",
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def select_tiling(
|
||||
height: int, width: int, crop_size: int, max_crops: int
|
||||
) -> tuple[int, int]:
|
||||
"""Determine the optimal number of tiles to cover an image."""
|
||||
if height <= crop_size or width <= crop_size:
|
||||
return (1, 1)
|
||||
|
||||
min_h = math.ceil(height / crop_size)
|
||||
min_w = math.ceil(width / crop_size)
|
||||
|
||||
if min_h * min_w > max_crops:
|
||||
ratio = math.sqrt(max_crops / (min_h * min_w))
|
||||
return (max(1, math.floor(min_h * ratio)), max(1, math.floor(min_w * ratio)))
|
||||
|
||||
h_tiles = math.floor(math.sqrt(max_crops * height / width))
|
||||
w_tiles = math.floor(math.sqrt(max_crops * width / height))
|
||||
|
||||
h_tiles = max(h_tiles, min_h)
|
||||
w_tiles = max(w_tiles, min_w)
|
||||
|
||||
if h_tiles * w_tiles > max_crops:
|
||||
if w_tiles > h_tiles:
|
||||
w_tiles = math.floor(max_crops / h_tiles)
|
||||
else:
|
||||
h_tiles = math.floor(max_crops / w_tiles)
|
||||
|
||||
return (max(1, h_tiles), max(1, w_tiles))
|
||||
|
||||
|
||||
class Moondream3Processor(ProcessorMixin):
|
||||
"""Constructs a Moondream3 processor which handles image preprocessing
|
||||
and tokenization for the Moondream3 multimodal model.
|
||||
|
||||
Args:
|
||||
tokenizer: The tokenizer to use for text processing.
|
||||
chat_template: Optional chat template string.
|
||||
crop_size: Size of each image crop.
|
||||
max_crops: Maximum number of crops per image.
|
||||
overlap_margin: Margin for overlapping crops in patches.
|
||||
patch_size: Size of each patch.
|
||||
|
||||
"""
|
||||
|
||||
attributes = ["tokenizer"]
|
||||
valid_kwargs = [
|
||||
"chat_template",
|
||||
"crop_size",
|
||||
"max_crops",
|
||||
"overlap_margin",
|
||||
"patch_size",
|
||||
]
|
||||
|
||||
tokenizer_class = "AutoTokenizer"
|
||||
# Use separate tokenizer repo
|
||||
_tokenizer_repo = "moondream/starmie-v1"
|
||||
|
||||
# Default chat template for Moondream3
|
||||
# Moondream uses special tokens for prompting:
|
||||
# - Token 0 (<|endoftext|>): BOS token (ALWAYS present at position 0)
|
||||
# - Token 1 (<|md_reserved_0|>): Start of instruction
|
||||
# - Token 2 (<|md_reserved_1|>): Separator before question
|
||||
# - Token 3 (<|md_reserved_2|>): End of question / start of answer
|
||||
#
|
||||
# Task routing based on text prefix:
|
||||
# "caption [short|normal|long]" → describe<|md_reserved_1|>{length}
|
||||
# "describe [short|normal|long]" → describe<|md_reserved_1|>{length}
|
||||
# otherwise → query<|md_reserved_1|><text>
|
||||
#
|
||||
# Format with image:
|
||||
# <|endoftext|><image><|md_reserved_0|>{task}<|md_reserved_1|>{q}<|md_reserved_2|>
|
||||
# Format without image:
|
||||
# <|endoftext|><|md_reserved_0|>{task}<|md_reserved_1|>{q}<|md_reserved_2|>
|
||||
_default_chat_template = (
|
||||
"{% for message in messages %}"
|
||||
"{% if message['role'] == 'user' %}"
|
||||
"{% if message['content'] is string %}"
|
||||
# Simple string content (with image assumed) - route by prefix
|
||||
"<|endoftext|><image><|md_reserved_0|>"
|
||||
"{% if message['content'] == 'caption' %}"
|
||||
"describe<|md_reserved_1|>normal<|md_reserved_2|>"
|
||||
"{% elif message['content'].startswith('caption ') %}"
|
||||
"describe<|md_reserved_1|>{{ message['content'][8:] }}<|md_reserved_2|>"
|
||||
"{% elif message['content'] == 'describe' %}"
|
||||
"describe<|md_reserved_1|>normal<|md_reserved_2|>"
|
||||
"{% elif message['content'].startswith('describe ') %}"
|
||||
"describe<|md_reserved_1|>{{ message['content'][9:] }}<|md_reserved_2|>"
|
||||
"{% else %}"
|
||||
"query<|md_reserved_1|>{{ message['content'] }}<|md_reserved_2|>"
|
||||
"{% endif %}"
|
||||
"{% else %}"
|
||||
# List content - build Moondream's image prefix independently of
|
||||
# OpenAI-style content part order, then render the text task.
|
||||
"<|endoftext|>"
|
||||
"{% for content in message['content'] %}"
|
||||
"{% if content['type'] in ['image', 'image_url', 'input_image', 'image_pil'] %}" # noqa: E501
|
||||
"<image>"
|
||||
"{% endif %}"
|
||||
"{% endfor %}"
|
||||
"{% for content in message['content'] %}"
|
||||
"{% if content['type'] == 'text' %}"
|
||||
"<|md_reserved_0|>"
|
||||
"{% if content['text'] == 'caption' %}"
|
||||
"describe<|md_reserved_1|>normal<|md_reserved_2|>"
|
||||
"{% elif content['text'].startswith('caption ') %}"
|
||||
"describe<|md_reserved_1|>{{ content['text'][8:] }}<|md_reserved_2|>"
|
||||
"{% elif content['text'] == 'describe' %}"
|
||||
"describe<|md_reserved_1|>normal<|md_reserved_2|>"
|
||||
"{% elif content['text'].startswith('describe ') %}"
|
||||
"describe<|md_reserved_1|>{{ content['text'][9:] }}<|md_reserved_2|>"
|
||||
"{% else %}"
|
||||
"query<|md_reserved_1|>{{ content['text'] }}<|md_reserved_2|>"
|
||||
"{% endif %}"
|
||||
"{% endif %}"
|
||||
"{% endfor %}"
|
||||
"{% endif %}"
|
||||
"{% elif message['role'] == 'assistant' %}"
|
||||
"{{ message['content'] }}"
|
||||
"{% endif %}"
|
||||
"{% endfor %}"
|
||||
)
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
tokenizer: PreTrainedTokenizerBase | None = None,
|
||||
chat_template: str | None = None,
|
||||
crop_size: int = 378,
|
||||
max_crops: int = 12,
|
||||
overlap_margin: int = 4,
|
||||
patch_size: int = 14,
|
||||
**kwargs,
|
||||
):
|
||||
self.image_token = "<image>"
|
||||
self.crop_size = crop_size
|
||||
self.max_crops = max_crops
|
||||
self.overlap_margin = overlap_margin
|
||||
self.patch_size = patch_size
|
||||
|
||||
# Number of patches per crop (27x27 = 729 for 378/14)
|
||||
self.patches_per_crop = (crop_size // patch_size) ** 2
|
||||
|
||||
# Use default chat template if none provided
|
||||
if chat_template is None:
|
||||
chat_template = self._default_chat_template
|
||||
|
||||
super().__init__(tokenizer, chat_template=chat_template)
|
||||
|
||||
@classmethod
|
||||
def from_pretrained(
|
||||
cls,
|
||||
pretrained_model_name_or_path,
|
||||
**kwargs,
|
||||
):
|
||||
"""Load the processor, using a separate tokenizer repo.
|
||||
|
||||
The moondream3 model uses a custom tokenizer from 'moondream/starmie-v1'
|
||||
instead of having tokenizer files in the model repo.
|
||||
"""
|
||||
from transformers import AutoTokenizer, TokenizersBackend
|
||||
from transformers.utils import cached_file
|
||||
|
||||
tokenizer = kwargs.pop("tokenizer", None)
|
||||
|
||||
tokenizer_kwargs = {
|
||||
"trust_remote_code": kwargs.get("trust_remote_code", False),
|
||||
}
|
||||
for key in (
|
||||
"cache_dir",
|
||||
"force_download",
|
||||
"local_files_only",
|
||||
"revision",
|
||||
"subfolder",
|
||||
"token",
|
||||
):
|
||||
if key in kwargs:
|
||||
tokenizer_kwargs[key] = kwargs[key]
|
||||
|
||||
cached_file_kwargs = {
|
||||
key: tokenizer_kwargs[key]
|
||||
for key in (
|
||||
"cache_dir",
|
||||
"force_download",
|
||||
"local_files_only",
|
||||
"revision",
|
||||
"subfolder",
|
||||
"token",
|
||||
)
|
||||
if key in tokenizer_kwargs
|
||||
}
|
||||
|
||||
def load_tokenizer(repo_or_path):
|
||||
try:
|
||||
return AutoTokenizer.from_pretrained(repo_or_path, **tokenizer_kwargs)
|
||||
except Exception:
|
||||
tokenizer_file = cached_file(
|
||||
repo_or_path,
|
||||
"tokenizer.json",
|
||||
**cached_file_kwargs,
|
||||
)
|
||||
return TokenizersBackend(
|
||||
tokenizer_file=tokenizer_file,
|
||||
clean_up_tokenization_spaces=False,
|
||||
)
|
||||
|
||||
if isinstance(tokenizer, str):
|
||||
tokenizer = load_tokenizer(tokenizer)
|
||||
|
||||
if tokenizer is None:
|
||||
# Prefer model-local tokenizer files first. If unavailable, fall
|
||||
# back to moondream's dedicated tokenizer repository.
|
||||
try:
|
||||
tokenizer = load_tokenizer(pretrained_model_name_or_path)
|
||||
except Exception:
|
||||
tokenizer = load_tokenizer(cls._tokenizer_repo)
|
||||
|
||||
# Configure special tokens for Moondream3
|
||||
# BOS and EOS are both token 0 (<|endoftext|>), matching the native
|
||||
# config (TokenizerConfig.bos_id=0, eos_id=0). This is standard for
|
||||
# GPT-2 style models where <|endoftext|> signals both start and end.
|
||||
# Token 1 (<|md_reserved_0|>) is a template delimiter, NOT the EOS.
|
||||
tokenizer.bos_token = "<|endoftext|>"
|
||||
tokenizer.bos_token_id = 0
|
||||
tokenizer.eos_token = "<|endoftext|>"
|
||||
tokenizer.eos_token_id = 0
|
||||
|
||||
# Extract processor-specific kwargs
|
||||
crop_size = kwargs.pop("crop_size", 378)
|
||||
max_crops = kwargs.pop("max_crops", 12)
|
||||
overlap_margin = kwargs.pop("overlap_margin", 4)
|
||||
patch_size = kwargs.pop("patch_size", 14)
|
||||
chat_template = kwargs.pop("chat_template", None)
|
||||
|
||||
# Set default chat template on tokenizer if not already set
|
||||
if chat_template is None:
|
||||
chat_template = cls._default_chat_template
|
||||
if tokenizer.chat_template is None:
|
||||
tokenizer.chat_template = chat_template
|
||||
|
||||
return cls(
|
||||
tokenizer=tokenizer,
|
||||
chat_template=chat_template,
|
||||
crop_size=crop_size,
|
||||
max_crops=max_crops,
|
||||
overlap_margin=overlap_margin,
|
||||
patch_size=patch_size,
|
||||
)
|
||||
|
||||
def __call__(
|
||||
self,
|
||||
images: ImageInput = None,
|
||||
text: TextInput
|
||||
| PreTokenizedInput
|
||||
| list[TextInput]
|
||||
| list[PreTokenizedInput] = None,
|
||||
**kwargs: Unpack[Moondream3ProcessorKwargs],
|
||||
) -> BatchFeature:
|
||||
"""Process images and text for Moondream3 model.
|
||||
|
||||
Args:
|
||||
images: Input images (PIL Image, numpy array, or list thereof).
|
||||
text: Input text or list of texts.
|
||||
**kwargs: Additional processing arguments.
|
||||
|
||||
Returns:
|
||||
BatchFeature with processed inputs.
|
||||
|
||||
"""
|
||||
output_kwargs = self._merge_kwargs(
|
||||
Moondream3ProcessorKwargs,
|
||||
tokenizer_init_kwargs=self.tokenizer.init_kwargs,
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
# Process images
|
||||
image_features = {}
|
||||
if images is not None:
|
||||
processed_images = []
|
||||
tilings = []
|
||||
|
||||
images_list = images if isinstance(images, list) else [images]
|
||||
for image in images_list:
|
||||
pixel_values, tiling = self.preprocess_image(
|
||||
image, **output_kwargs["images_kwargs"]
|
||||
)
|
||||
processed_images.append(pixel_values)
|
||||
tilings.append(tiling)
|
||||
|
||||
if processed_images:
|
||||
image_features["pixel_values"] = processed_images
|
||||
image_features["tilings"] = tilings
|
||||
|
||||
# Process text
|
||||
if text is not None:
|
||||
if not isinstance(text, list):
|
||||
text = [text]
|
||||
|
||||
# Get text kwargs, remove keys we set ourselves
|
||||
text_kwargs = output_kwargs.get("text_kwargs", {}).copy()
|
||||
text_kwargs.pop("return_tensors", None)
|
||||
text_kwargs.pop("add_special_tokens", None)
|
||||
|
||||
# Tokenize text
|
||||
tokenized = self.tokenizer(
|
||||
text,
|
||||
add_special_tokens=True,
|
||||
return_tensors="pt",
|
||||
**text_kwargs,
|
||||
)
|
||||
|
||||
output = BatchFeature(data=dict(tokenized))
|
||||
|
||||
# Add image features
|
||||
if image_features:
|
||||
output["pixel_values"] = image_features["pixel_values"]
|
||||
output["tilings"] = image_features["tilings"]
|
||||
|
||||
return output
|
||||
|
||||
# If only images were provided
|
||||
return BatchFeature(data=image_features)
|
||||
|
||||
@staticmethod
|
||||
def _image_array_to_uint8(array: np.ndarray) -> np.ndarray:
|
||||
if array.dtype == np.uint8:
|
||||
return np.ascontiguousarray(array)
|
||||
|
||||
if array.dtype == np.bool_:
|
||||
return np.ascontiguousarray(array.astype(np.uint8) * 255)
|
||||
|
||||
if np.issubdtype(array.dtype, np.floating):
|
||||
array = np.nan_to_num(array, nan=0.0, posinf=255.0, neginf=0.0)
|
||||
if array.size > 0 and array.max() <= 1.0:
|
||||
array = array * 255.0
|
||||
array = np.rint(array)
|
||||
|
||||
return np.ascontiguousarray(np.clip(array, 0, 255).astype(np.uint8))
|
||||
|
||||
@staticmethod
|
||||
def _to_pil_image(image: ImageInput) -> Image.Image:
|
||||
if isinstance(image, Image.Image):
|
||||
return image
|
||||
|
||||
if isinstance(image, torch.Tensor):
|
||||
tensor = image.detach().cpu()
|
||||
if tensor.dtype == torch.bfloat16:
|
||||
tensor = tensor.to(torch.float32)
|
||||
image_array = tensor.numpy()
|
||||
elif isinstance(image, np.ndarray):
|
||||
image_array = image
|
||||
else:
|
||||
raise TypeError(
|
||||
"Moondream3 images must be PIL images, numpy arrays, "
|
||||
f"or torch tensors, got {type(image)!r}."
|
||||
)
|
||||
|
||||
if image_array.ndim == 2:
|
||||
image_array = Moondream3Processor._image_array_to_uint8(image_array)
|
||||
return Image.fromarray(image_array)
|
||||
|
||||
if image_array.ndim != 3:
|
||||
raise ValueError(
|
||||
"Moondream3 image arrays must have 2 or 3 dimensions, "
|
||||
f"got shape {image_array.shape}."
|
||||
)
|
||||
|
||||
channel_dims = (1, 3, 4)
|
||||
if image_array.shape[-1] not in channel_dims:
|
||||
if image_array.shape[0] not in channel_dims:
|
||||
raise ValueError(
|
||||
"Moondream3 image arrays must be HWC or CHW with 1, 3, "
|
||||
f"or 4 channels, got shape {image_array.shape}."
|
||||
)
|
||||
image_array = np.transpose(image_array, (1, 2, 0))
|
||||
|
||||
image_array = Moondream3Processor._image_array_to_uint8(image_array)
|
||||
if image_array.shape[-1] == 1:
|
||||
image_array = image_array[..., 0]
|
||||
|
||||
return Image.fromarray(image_array)
|
||||
|
||||
def preprocess_image(
|
||||
self,
|
||||
image: ImageInput,
|
||||
max_crops: int = 12,
|
||||
overlap_margin: int = 4,
|
||||
crop_size: int = 378,
|
||||
patch_size: int = 14,
|
||||
convert_to_rgb: bool = True,
|
||||
return_tensors: str = "pt",
|
||||
) -> tuple[torch.Tensor, tuple[int, int]]:
|
||||
"""Preprocess an image using overlap-and-resize cropping strategy.
|
||||
|
||||
Args:
|
||||
image: Input PIL image, numpy array, or torch tensor.
|
||||
max_crops: Maximum number of crops.
|
||||
overlap_margin: Margin for overlapping in patches.
|
||||
crop_size: Size of each crop.
|
||||
patch_size: Size of each patch.
|
||||
convert_to_rgb: Whether to convert to RGB.
|
||||
return_tensors: Return type ("pt" for PyTorch).
|
||||
|
||||
Returns:
|
||||
Tuple of (pixel_values tensor, tiling tuple).
|
||||
|
||||
"""
|
||||
image = self._to_pil_image(image)
|
||||
if convert_to_rgb:
|
||||
image = convert_image_mode(image, "RGB")
|
||||
|
||||
# Convert to numpy array
|
||||
image_array = np.array(image)
|
||||
original_h, original_w = image_array.shape[:2]
|
||||
|
||||
margin_pixels = patch_size * overlap_margin
|
||||
total_margin_pixels = margin_pixels * 2
|
||||
|
||||
crop_patches = crop_size // patch_size
|
||||
crop_window_patches = crop_patches - (2 * overlap_margin)
|
||||
crop_window_size = crop_window_patches * patch_size
|
||||
|
||||
tiling = select_tiling(
|
||||
original_h - total_margin_pixels,
|
||||
original_w - total_margin_pixels,
|
||||
crop_window_size,
|
||||
max_crops,
|
||||
)
|
||||
|
||||
n_crops = tiling[0] * tiling[1] + 1
|
||||
crops = np.zeros((n_crops, crop_size, crop_size, 3), dtype=np.uint8)
|
||||
|
||||
target_size = (
|
||||
tiling[0] * crop_window_size + total_margin_pixels,
|
||||
tiling[1] * crop_window_size + total_margin_pixels,
|
||||
)
|
||||
|
||||
# Resize image
|
||||
pil_img = Image.fromarray(image_array)
|
||||
resized = pil_img.resize(
|
||||
(int(target_size[1]), int(target_size[0])),
|
||||
resample=Image.Resampling.LANCZOS,
|
||||
)
|
||||
resized_array = np.asarray(resized)
|
||||
|
||||
# Create global crop
|
||||
global_pil = pil_img.resize(
|
||||
(crop_size, crop_size), resample=Image.Resampling.LANCZOS
|
||||
)
|
||||
crops[0] = np.asarray(global_pil)
|
||||
|
||||
# Create local crops
|
||||
for i in range(tiling[0]):
|
||||
for j in range(tiling[1]):
|
||||
y0 = i * crop_window_size
|
||||
x0 = j * crop_window_size
|
||||
y_end = min(y0 + crop_size, resized_array.shape[0])
|
||||
x_end = min(x0 + crop_size, resized_array.shape[1])
|
||||
|
||||
crop_region = resized_array[y0:y_end, x0:x_end]
|
||||
crop_idx = 1 + i * tiling[1] + j
|
||||
h_slice = slice(None, crop_region.shape[0])
|
||||
w_slice = slice(None, crop_region.shape[1])
|
||||
crops[crop_idx, h_slice, w_slice] = crop_region
|
||||
|
||||
# Convert to tensor: (n_crops, H, W, C) -> (n_crops, C, H, W)
|
||||
pixel_values = np.transpose(crops, (0, 3, 1, 2))
|
||||
|
||||
if return_tensors == "pt":
|
||||
# Match HF reference preprocessing exactly: convert uint8 crops to
|
||||
# bfloat16 before in-place normalization.
|
||||
pixel_values = (
|
||||
torch.from_numpy(pixel_values)
|
||||
.to(dtype=torch.bfloat16)
|
||||
.div_(255.0)
|
||||
.sub_(0.5)
|
||||
.div_(0.5)
|
||||
)
|
||||
else:
|
||||
pixel_values = pixel_values.astype(np.float32) / 255.0
|
||||
pixel_values = (pixel_values - 0.5) / 0.5
|
||||
|
||||
return pixel_values, tiling
|
||||
|
||||
def get_num_image_tokens(self) -> int:
|
||||
"""Return the number of image tokens (729 = 27x27 patches)."""
|
||||
return self.patches_per_crop
|
||||
|
||||
def batch_decode(self, *args, **kwargs):
|
||||
"""Forward to tokenizer's batch_decode."""
|
||||
return self.tokenizer.batch_decode(*args, **kwargs)
|
||||
|
||||
def decode(self, *args, **kwargs):
|
||||
"""Forward to tokenizer's decode."""
|
||||
return self.tokenizer.decode(*args, **kwargs)
|
||||
|
||||
@property
|
||||
def model_input_names(self):
|
||||
tokenizer_input_names = self.tokenizer.model_input_names
|
||||
return tokenizer_input_names + ["pixel_values", "tilings"]
|
||||
|
||||
|
||||
AutoProcessor.register("Moondream3Processor", Moondream3Processor)
|
||||
@@ -1,372 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import torch
|
||||
import torchvision.transforms as T
|
||||
from PIL import Image
|
||||
|
||||
from vllm.multimodal.image import convert_image_mode
|
||||
from vllm.tokenizers.hf import HfTokenizer
|
||||
|
||||
from .internvl import InternVLImageProcessor, InternVLProcessor
|
||||
|
||||
|
||||
def build_transform(input_size: int):
|
||||
return T.Compose(
|
||||
[
|
||||
T.Lambda(lambda img: convert_image_mode(img, "RGB")),
|
||||
T.Resize(
|
||||
(input_size, input_size), interpolation=T.InterpolationMode.BICUBIC
|
||||
),
|
||||
T.ToTensor(),
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
# adapted from https://huggingface.co/nvidia/Llama-3.1-Nemotron-Nano-VL-8B-V1
|
||||
def find_closest_aspect_ratio(
|
||||
aspect_ratio: float,
|
||||
target_ratios: list[tuple[int, int]],
|
||||
*,
|
||||
width: int,
|
||||
height: int,
|
||||
image_size: int,
|
||||
) -> tuple[int, int]:
|
||||
best_factor = float("-inf")
|
||||
best_ratio = (1, 1)
|
||||
area = width * height
|
||||
|
||||
for rw, rh in target_ratios:
|
||||
target_aspect_ratio = rw / rh
|
||||
size_factor = min((rw * rh * image_size * image_size) / area, 0.6)
|
||||
ratio_closeness = min(
|
||||
target_aspect_ratio / aspect_ratio, aspect_ratio / target_aspect_ratio
|
||||
)
|
||||
factor = size_factor * ratio_closeness
|
||||
|
||||
if factor > best_factor:
|
||||
best_factor = factor
|
||||
best_ratio = (rw, rh)
|
||||
|
||||
return best_ratio
|
||||
|
||||
|
||||
def calculate_nemotron_vl_targets(
|
||||
*,
|
||||
orig_width: int,
|
||||
orig_height: int,
|
||||
target_ratios: list[tuple[int, int]],
|
||||
image_size: int,
|
||||
use_thumbnail: bool,
|
||||
) -> tuple[int, int, int]:
|
||||
aspect_ratio = orig_width / orig_height
|
||||
|
||||
# find the closest aspect ratio to the target
|
||||
target_aspect_ratio = find_closest_aspect_ratio(
|
||||
aspect_ratio,
|
||||
target_ratios,
|
||||
width=orig_width,
|
||||
height=orig_height,
|
||||
image_size=image_size,
|
||||
)
|
||||
|
||||
# calculate the target width and height
|
||||
target_width = image_size * target_aspect_ratio[0]
|
||||
target_height = image_size * target_aspect_ratio[1]
|
||||
blocks = target_aspect_ratio[0] * target_aspect_ratio[1]
|
||||
|
||||
# add thumbnail image if num_blocks != 1
|
||||
if use_thumbnail and blocks != 1:
|
||||
blocks += 1
|
||||
|
||||
return blocks, target_width, target_height
|
||||
|
||||
|
||||
def dynamic_preprocess_nemotron_vl(
|
||||
image: Image.Image,
|
||||
*,
|
||||
target_ratios: list[tuple[int, int]],
|
||||
image_size: int,
|
||||
use_thumbnail: bool,
|
||||
) -> list[Image.Image]:
|
||||
orig_width, orig_height = image.size
|
||||
|
||||
# calculate the number of blocks without thumbnail
|
||||
blocks, target_width, target_height = calculate_nemotron_vl_targets(
|
||||
orig_width=orig_width,
|
||||
orig_height=orig_height,
|
||||
target_ratios=target_ratios,
|
||||
image_size=image_size,
|
||||
use_thumbnail=False,
|
||||
)
|
||||
|
||||
# resize the image
|
||||
resized_img = image.resize((target_width, target_height))
|
||||
processed_images = []
|
||||
for i in range(blocks):
|
||||
box = (
|
||||
(i % (target_width // image_size)) * image_size,
|
||||
(i // (target_width // image_size)) * image_size,
|
||||
((i % (target_width // image_size)) + 1) * image_size,
|
||||
((i // (target_width // image_size)) + 1) * image_size,
|
||||
)
|
||||
# split the image
|
||||
split_img = resized_img.crop(box)
|
||||
processed_images.append(split_img)
|
||||
|
||||
assert len(processed_images) == blocks
|
||||
|
||||
if use_thumbnail and len(processed_images) != 1:
|
||||
thumbnail_img = image.resize((image_size, image_size))
|
||||
processed_images.append(thumbnail_img)
|
||||
|
||||
return processed_images
|
||||
|
||||
|
||||
def get_nemotron_vl_target_ratios(
|
||||
min_num: int,
|
||||
max_num: int,
|
||||
) -> list[tuple[int, int]]:
|
||||
target_ratios = {
|
||||
(i, j)
|
||||
for n in range(min_num, max_num + 1)
|
||||
for i in range(1, n + 1)
|
||||
for j in range(1, n + 1)
|
||||
if min_num <= i * j <= max_num
|
||||
}
|
||||
return sorted(target_ratios, key=lambda x: x[0] * x[1])
|
||||
|
||||
|
||||
def image_to_pixel_values_nemotron_vl(
|
||||
image: Image.Image,
|
||||
*,
|
||||
input_size: int,
|
||||
min_num: int,
|
||||
max_num: int,
|
||||
use_thumbnail: bool,
|
||||
transform: T.Compose | None = None,
|
||||
) -> torch.Tensor:
|
||||
target_ratios = get_nemotron_vl_target_ratios(min_num, max_num)
|
||||
|
||||
if transform is None:
|
||||
transform = build_transform(input_size=input_size)
|
||||
|
||||
images = dynamic_preprocess_nemotron_vl(
|
||||
image,
|
||||
target_ratios=target_ratios,
|
||||
image_size=input_size,
|
||||
use_thumbnail=use_thumbnail,
|
||||
)
|
||||
|
||||
pixel_values = torch.stack([transform(image) for image in images])
|
||||
return pixel_values
|
||||
|
||||
|
||||
class LlamaNemotronNanoVLImageProcessor(InternVLImageProcessor):
|
||||
def _images_to_pixel_values_lst(
|
||||
self,
|
||||
images: list[Image.Image],
|
||||
min_dynamic_patch: int | None = None,
|
||||
max_dynamic_patch: int | None = None,
|
||||
dynamic_image_size: bool | None = None,
|
||||
) -> list[torch.Tensor]:
|
||||
min_num, max_num = self.resolve_min_max_num(
|
||||
min_dynamic_patch=min_dynamic_patch,
|
||||
max_dynamic_patch=max_dynamic_patch,
|
||||
dynamic_image_size=dynamic_image_size,
|
||||
use_thumbnail=False, # Applied in image_to_pixel_values
|
||||
)
|
||||
|
||||
return [
|
||||
image_to_pixel_values_nemotron_vl(
|
||||
image,
|
||||
input_size=self.image_size,
|
||||
min_num=min_num,
|
||||
max_num=max_num,
|
||||
use_thumbnail=self.use_thumbnail,
|
||||
transform=build_transform(self.image_size),
|
||||
)
|
||||
for image in images
|
||||
]
|
||||
|
||||
|
||||
class LlamaNemotronNanoVLProcessor(InternVLProcessor):
|
||||
"""This model doesn't define its own HF processor,
|
||||
so we implement our own one here.
|
||||
|
||||
The image processor is given by:
|
||||
https://huggingface.co/nvidia/Llama-3.1-Nemotron-Nano-VL-8B-V1/blob/main/image_processing.py
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
image_processor: LlamaNemotronNanoVLImageProcessor,
|
||||
tokenizer: HfTokenizer,
|
||||
*,
|
||||
image_seq_length: int,
|
||||
start_image_token: str = "<img>",
|
||||
end_image_token: str = "</img>",
|
||||
ctx_image_token: str = "<image>",
|
||||
) -> None:
|
||||
super().__init__(
|
||||
image_processor=image_processor,
|
||||
tokenizer=tokenizer,
|
||||
image_seq_length=image_seq_length,
|
||||
start_image_token=start_image_token,
|
||||
end_image_token=end_image_token,
|
||||
ctx_image_token=ctx_image_token,
|
||||
)
|
||||
|
||||
def get_num_image_tokens(
|
||||
self,
|
||||
*,
|
||||
image_width: int,
|
||||
image_height: int,
|
||||
) -> int:
|
||||
image_processor = self.image_processor
|
||||
target_ratios = self.resolve_target_ratios(
|
||||
use_thumbnail=False, # Applied in calculate_targets
|
||||
)
|
||||
|
||||
num_patches, _, _ = calculate_nemotron_vl_targets(
|
||||
orig_width=image_width,
|
||||
orig_height=image_height,
|
||||
image_size=image_processor.image_size,
|
||||
target_ratios=target_ratios,
|
||||
use_thumbnail=image_processor.use_thumbnail,
|
||||
)
|
||||
|
||||
return num_patches * self.image_seq_length
|
||||
|
||||
|
||||
# SigLIP normalization constants
|
||||
SIGLIP_MEAN = (0.5, 0.5, 0.5)
|
||||
SIGLIP_STD = (0.5, 0.5, 0.5)
|
||||
|
||||
|
||||
def build_siglip_transform(
|
||||
input_size: int,
|
||||
*,
|
||||
normalize_on_cpu: bool = True,
|
||||
):
|
||||
"""Build transform for SigLIP vision encoder with normalization.
|
||||
|
||||
Extends the base transform from nemotron_vl with SigLIP-specific normalization.
|
||||
"""
|
||||
transforms = [
|
||||
T.Lambda(lambda img: convert_image_mode(img, "RGB")),
|
||||
T.Resize((input_size, input_size), interpolation=T.InterpolationMode.BICUBIC),
|
||||
T.ToTensor() if normalize_on_cpu else T.PILToTensor(),
|
||||
]
|
||||
if normalize_on_cpu:
|
||||
transforms.append(T.Normalize(mean=SIGLIP_MEAN, std=SIGLIP_STD))
|
||||
|
||||
return T.Compose(transforms)
|
||||
|
||||
|
||||
class LlamaNemotronVLEmbedImageProcessor(InternVLImageProcessor):
|
||||
def __init__(
|
||||
self,
|
||||
image_size: int,
|
||||
min_dynamic_patch: int,
|
||||
max_dynamic_patch: int,
|
||||
dynamic_image_size: bool,
|
||||
use_thumbnail: bool,
|
||||
*,
|
||||
do_rescale: bool = True,
|
||||
do_normalize: bool = True,
|
||||
) -> None:
|
||||
if do_rescale != do_normalize:
|
||||
raise ValueError(
|
||||
"LlamaNemotronVLEmbedImageProcessor requires do_rescale and "
|
||||
"do_normalize to be both enabled or both disabled."
|
||||
)
|
||||
|
||||
super().__init__(
|
||||
image_size=image_size,
|
||||
min_dynamic_patch=min_dynamic_patch,
|
||||
max_dynamic_patch=max_dynamic_patch,
|
||||
dynamic_image_size=dynamic_image_size,
|
||||
use_thumbnail=use_thumbnail,
|
||||
)
|
||||
self.normalize_on_cpu = do_normalize
|
||||
|
||||
def _images_to_pixel_values_lst(
|
||||
self,
|
||||
images: list[Image.Image],
|
||||
min_dynamic_patch: int | None = None,
|
||||
max_dynamic_patch: int | None = None,
|
||||
dynamic_image_size: bool | None = None,
|
||||
) -> list[torch.Tensor]:
|
||||
min_num, max_num = self.resolve_min_max_num(
|
||||
min_dynamic_patch=min_dynamic_patch,
|
||||
max_dynamic_patch=max_dynamic_patch,
|
||||
dynamic_image_size=dynamic_image_size,
|
||||
use_thumbnail=False, # Applied in image_to_pixel_values
|
||||
)
|
||||
|
||||
return [
|
||||
image_to_pixel_values_nemotron_vl(
|
||||
image,
|
||||
input_size=self.image_size,
|
||||
min_num=min_num,
|
||||
max_num=max_num,
|
||||
use_thumbnail=self.use_thumbnail,
|
||||
transform=build_siglip_transform(
|
||||
self.image_size,
|
||||
normalize_on_cpu=self.normalize_on_cpu,
|
||||
),
|
||||
)
|
||||
for image in images
|
||||
]
|
||||
|
||||
|
||||
class LlamaNemotronVLEmbedProcessor(InternVLProcessor):
|
||||
"""Processor for LlamaNemotronVL embedding model.
|
||||
|
||||
Inherits from NemotronVLProcessor and specializes it for embedding tasks:
|
||||
- Uses SigLIP transform with normalization instead of base transform
|
||||
- Uses different image context token (<IMG_CONTEXT> vs <image>)
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
image_processor: LlamaNemotronVLEmbedImageProcessor,
|
||||
tokenizer: HfTokenizer,
|
||||
*,
|
||||
image_seq_length: int,
|
||||
start_image_token: str = "<img>",
|
||||
end_image_token: str = "</img>",
|
||||
ctx_image_token: str = "<IMG_CONTEXT>",
|
||||
) -> None:
|
||||
super().__init__(
|
||||
image_processor=image_processor,
|
||||
tokenizer=tokenizer,
|
||||
image_seq_length=image_seq_length,
|
||||
start_image_token=start_image_token,
|
||||
end_image_token=end_image_token,
|
||||
ctx_image_token=ctx_image_token,
|
||||
)
|
||||
|
||||
self.image_processor: LlamaNemotronVLEmbedImageProcessor
|
||||
|
||||
def get_num_image_tokens(
|
||||
self,
|
||||
*,
|
||||
image_width: int,
|
||||
image_height: int,
|
||||
) -> int:
|
||||
image_processor = self.image_processor
|
||||
target_ratios = self.resolve_target_ratios(
|
||||
use_thumbnail=False, # Applied in calculate_targets
|
||||
)
|
||||
|
||||
num_patches, _, _ = calculate_nemotron_vl_targets(
|
||||
orig_width=image_width,
|
||||
orig_height=image_height,
|
||||
image_size=image_processor.image_size,
|
||||
target_ratios=target_ratios,
|
||||
use_thumbnail=image_processor.use_thumbnail,
|
||||
)
|
||||
|
||||
return num_patches * self.image_seq_length
|
||||
@@ -1,63 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
# adapted from https://huggingface.co/nvidia/NVLM-D-72B/blob/main/modeling_nvlm_d.py
|
||||
# --------------------------------------------------------
|
||||
# NVLM-D
|
||||
# Copyright (c) 2024 NVIDIA
|
||||
# Licensed under Apache 2.0 License [see LICENSE for details]
|
||||
# --------------------------------------------------------
|
||||
from vllm.multimodal.processing import PromptUpdateDetails, cached_encode
|
||||
from vllm.tokenizers.hf import HfTokenizer
|
||||
|
||||
from .internvl import InternVLImageProcessor, InternVLProcessor
|
||||
|
||||
|
||||
class NVLMProcessor(InternVLProcessor):
|
||||
def __init__(
|
||||
self,
|
||||
image_processor: InternVLImageProcessor,
|
||||
tokenizer: HfTokenizer,
|
||||
*,
|
||||
image_seq_length: int,
|
||||
start_image_token: str = "<Image>",
|
||||
end_image_token: str = "</Image>",
|
||||
ctx_image_token: str = "<|vision_pad|>",
|
||||
) -> None:
|
||||
super().__init__(
|
||||
image_processor=image_processor,
|
||||
tokenizer=tokenizer,
|
||||
image_seq_length=image_seq_length,
|
||||
start_image_token=start_image_token,
|
||||
end_image_token=end_image_token,
|
||||
ctx_image_token=ctx_image_token,
|
||||
)
|
||||
|
||||
def get_image_repl(
|
||||
self,
|
||||
num_patches: int | None,
|
||||
num_features: int | None = None,
|
||||
) -> PromptUpdateDetails:
|
||||
if num_patches is None:
|
||||
raise NotImplementedError("Embedding inputs are not supported")
|
||||
|
||||
num_features = num_patches * self.image_seq_length
|
||||
|
||||
tile_pos_identifiers = [f"<tile_{i}>" for i in range(1, num_patches)]
|
||||
if self.image_processor.use_thumbnail:
|
||||
tile_pos_identifiers += ["<tile_global_thumbnail>"]
|
||||
|
||||
context_size = num_features // num_patches
|
||||
features = "".join(
|
||||
(identifier + self.ctx_image_token * context_size)
|
||||
for identifier in tile_pos_identifiers
|
||||
)
|
||||
|
||||
# We include the start and end as well because "<Image><tile" is
|
||||
# tokenized as ["<Image", "><", "tile"], resulting in assertion error
|
||||
# when trying to find "<tile" as a subsequence of "<Image><tile"
|
||||
repl = self.start_image_token + features + self.end_image_token
|
||||
|
||||
full_ids = cached_encode(self.tokenizer, repl, add_special_tokens=False)
|
||||
|
||||
return PromptUpdateDetails.select_token_id(full_ids, self.ctx_image_token_id)
|
||||
@@ -1,102 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from collections.abc import Sequence
|
||||
from typing import Any
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
from PIL import Image
|
||||
from transformers import ImageProcessingMixin, ProcessorMixin
|
||||
|
||||
IMAGENET_MEAN = np.array([0.484375, 0.455078125, 0.40625], dtype=np.float32)
|
||||
IMAGENET_STD = np.array([0.228515625, 0.2236328125, 0.224609375], dtype=np.float32)
|
||||
SIGLIP_MEAN = np.array([0.5, 0.5, 0.5], dtype=np.float32)
|
||||
SIGLIP_STD = np.array([0.5, 0.5, 0.5], dtype=np.float32)
|
||||
|
||||
|
||||
def to_rgb_image(image: Any) -> Image.Image:
|
||||
if isinstance(image, Image.Image):
|
||||
return image.convert("RGB")
|
||||
|
||||
if isinstance(image, torch.Tensor):
|
||||
image = image.detach().cpu().numpy()
|
||||
if not isinstance(image, np.ndarray):
|
||||
raise TypeError(
|
||||
"OpenVLA image input must be a PIL image, numpy array, or torch tensor; "
|
||||
f"got {type(image)}"
|
||||
)
|
||||
|
||||
if image.ndim != 3:
|
||||
raise ValueError(
|
||||
f"OpenVLA image input must have 3 dimensions, got shape {image.shape}"
|
||||
)
|
||||
|
||||
if image.shape[0] in (1, 3):
|
||||
image = np.moveaxis(image, 0, -1)
|
||||
|
||||
if image.shape[-1] == 1:
|
||||
image = np.repeat(image, 3, axis=-1)
|
||||
elif image.shape[-1] != 3:
|
||||
raise ValueError(
|
||||
f"OpenVLA image input must have 1 or 3 channels, got shape {image.shape}"
|
||||
)
|
||||
|
||||
if image.dtype != np.uint8:
|
||||
image = image.astype(np.float32)
|
||||
if image.max(initial=0.0) <= 1.0:
|
||||
image = image * 255.0
|
||||
image = np.clip(image, 0, 255).astype(np.uint8)
|
||||
|
||||
return Image.fromarray(image).convert("RGB")
|
||||
|
||||
|
||||
def preprocess_openvla_image(image: Any, image_size: int) -> torch.Tensor:
|
||||
rgb_image = to_rgb_image(image)
|
||||
rgb_image = rgb_image.resize(
|
||||
(image_size, image_size),
|
||||
Image.Resampling.BICUBIC,
|
||||
)
|
||||
|
||||
raw = np.asarray(rgb_image, dtype=np.float32) / 255.0
|
||||
dinov2_pixels = ((raw - IMAGENET_MEAN) / IMAGENET_STD).transpose(2, 0, 1)
|
||||
siglip_pixels = ((raw - SIGLIP_MEAN) / SIGLIP_STD).transpose(2, 0, 1)
|
||||
pixel_values = np.concatenate([dinov2_pixels, siglip_pixels], axis=0)
|
||||
return torch.from_numpy(pixel_values)
|
||||
|
||||
|
||||
class OpenVLAImageProcessor(ImageProcessingMixin):
|
||||
def __init__(self, *, image_size: int) -> None:
|
||||
self.image_size = image_size
|
||||
|
||||
def __call__(
|
||||
self,
|
||||
images: Any | None = None,
|
||||
**kwargs: object,
|
||||
) -> dict[str, object]:
|
||||
if images is None:
|
||||
return {}
|
||||
if not isinstance(images, Sequence) or isinstance(images, (str, bytes)):
|
||||
images = [images]
|
||||
if len(images) == 0:
|
||||
return {}
|
||||
|
||||
pixel_values = torch.stack(
|
||||
[
|
||||
preprocess_openvla_image(image, image_size=self.image_size)
|
||||
for image in images
|
||||
],
|
||||
dim=0,
|
||||
)
|
||||
return {"pixel_values": pixel_values}
|
||||
|
||||
|
||||
class OpenVLAProcessor(ProcessorMixin):
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
image_processor: OpenVLAImageProcessor,
|
||||
tokenizer: Any,
|
||||
) -> None:
|
||||
self.image_processor = image_processor
|
||||
self.tokenizer = tokenizer
|
||||
@@ -1,457 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
# ruff: noqa: E501
|
||||
# coding=utf-8
|
||||
# adapted from https://github.com/AIDC-AI/Ovis/blob/35ab51a1a1e3542fa6db260a1084cefbc8f164bb/ovis/vllm/processing_ovis.py
|
||||
# Copyright 2025 The Qwen Team and The HuggingFace Inc. team. All rights reserved.
|
||||
#
|
||||
# This code is based on EleutherAI's GPT-NeoX library and the GPT-NeoX
|
||||
# and OPT implementations in this library. It has been modified from its
|
||||
# original forms to accommodate minor architectural differences compared
|
||||
# to GPT-NeoX and OPT used by the Meta AI team that trained the model.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
from functools import cached_property
|
||||
|
||||
import PIL
|
||||
import torch
|
||||
from transformers import BatchFeature
|
||||
from transformers.image_utils import ImageInput
|
||||
from transformers.processing_utils import ProcessingKwargs, ProcessorMixin, Unpack
|
||||
from transformers.tokenization_utils_base import PreTokenizedInput, TextInput
|
||||
|
||||
from vllm.multimodal.image import convert_image_mode
|
||||
|
||||
__all__ = ["OvisProcessor"]
|
||||
IGNORE_ID = -100
|
||||
|
||||
|
||||
class OvisProcessorKwargs(ProcessingKwargs, total=False): # type: ignore[call-arg]
|
||||
_defaults = {
|
||||
"text_kwargs": {
|
||||
"padding": False,
|
||||
},
|
||||
"images_kwargs": {
|
||||
"do_convert_rgb": True,
|
||||
"return_tensors": "pt",
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
class OvisProcessor(ProcessorMixin):
|
||||
r"""Constructs an Ovis processor which wraps an Ovis image processor and a Qwen2 tokenizer into a single processor.
|
||||
[`OvisProcessor`] offers all the functionalities of [`Qwen2VLImageProcessor`] and [`Qwen2TokenizerFast`]. See the
|
||||
[`~OvisProcessor.__call__`] and [`~OvisProcessor.decode`] for more information.
|
||||
|
||||
Args:
|
||||
image_processor ([`Qwen2VLImageProcessor`], *optional*):
|
||||
The image processor is a required input.
|
||||
tokenizer ([`Qwen2TokenizerFast`], *optional*):
|
||||
The tokenizer is a required input.
|
||||
chat_template (`str`, *optional*): A Jinja template which will be used to convert lists of messages
|
||||
in a chat into a tokenizable string.
|
||||
|
||||
"""
|
||||
|
||||
attributes = ["image_processor", "tokenizer"]
|
||||
valid_kwargs = ["chat_template", "image_pad_token", "image_segment_len"]
|
||||
|
||||
image_processor_class = "AutoImageProcessor"
|
||||
tokenizer_class = "AutoTokenizer"
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
image_processor=None,
|
||||
tokenizer=None,
|
||||
chat_template=None,
|
||||
image_pad_token=None,
|
||||
image_segment_len=255,
|
||||
**kwargs,
|
||||
):
|
||||
self.image_token = "<image>"
|
||||
self.image_pad_token = image_pad_token
|
||||
self.image_segment_len = image_segment_len
|
||||
super().__init__(image_processor, tokenizer, chat_template=chat_template)
|
||||
|
||||
@cached_property
|
||||
def extra_special_tokens(self):
|
||||
image_pad_token_id = self.tokenizer.get_vocab()[self.image_pad_token]
|
||||
extra_special_tokens = {
|
||||
"image_token": -200,
|
||||
"image_atom": -300,
|
||||
"image_start": -301,
|
||||
"image_prefix": -302,
|
||||
"image_col_sep": -303,
|
||||
"image_row_sep": -304,
|
||||
"image_end": -305,
|
||||
"image_pad": image_pad_token_id,
|
||||
}
|
||||
return extra_special_tokens
|
||||
|
||||
def __call__(
|
||||
self,
|
||||
images: ImageInput = None,
|
||||
text: TextInput
|
||||
| PreTokenizedInput
|
||||
| list[TextInput]
|
||||
| list[PreTokenizedInput] = None,
|
||||
**kwargs: Unpack[OvisProcessorKwargs],
|
||||
) -> BatchFeature:
|
||||
"""Main method to prepare for the model one or several sequences(s) and image(s). This method forwards the `text`
|
||||
and `kwargs` arguments to Qwen2TokenizerFast's [`~Qwen2TokenizerFast.__call__`] if `text` is not `None` to encode
|
||||
the text. To prepare the vision inputs, this method forwards the `vision_infos` and `kwargs` arguments to
|
||||
Qwen2VLImageProcessor's [`~Qwen2VLImageProcessor.__call__`] if `vision_infos` is not `None`.
|
||||
|
||||
Args:
|
||||
images (`PIL.Image.Image`, `np.ndarray`, `torch.Tensor`, `list[PIL.Image.Image]`, `list[np.ndarray]`, `list[torch.Tensor]`):
|
||||
The image or batch of images to be prepared. Each image can be a PIL image, NumPy array or PyTorch
|
||||
tensor. Both channels-first and channels-last formats are supported.
|
||||
text (`str`, `list[str]`, `list[list[str]]`):
|
||||
The sequence or batch of sequences to be encoded. Each sequence can be a string or a list of strings
|
||||
(pretokenized string). If the sequences are provided as list of strings (pretokenized), you must set
|
||||
`is_split_into_words=True` (to lift the ambiguity with a batch of sequences).
|
||||
videos (`np.ndarray`, `torch.Tensor`, `list[np.ndarray]`, `list[torch.Tensor]`):
|
||||
The image or batch of videos to be prepared. Each video can be a 4D NumPy array or PyTorch
|
||||
tensor, or a nested list of 3D frames. Both channels-first and channels-last formats are supported.
|
||||
return_tensors (`str` or [`~utils.TensorType`], *optional*):
|
||||
If set, will return tensors of a particular framework. Acceptable values are:
|
||||
- `'tf'`: Return TensorFlow `tf.constant` objects.
|
||||
- `'pt'`: Return PyTorch `torch.Tensor` objects.
|
||||
- `'np'`: Return NumPy `np.ndarray` objects.
|
||||
- `'jax'`: Return JAX `jnp.ndarray` objects.
|
||||
|
||||
Returns:
|
||||
[`BatchFeature`]: A [`BatchFeature`] with the following fields:
|
||||
- **input_ids** -- List of token ids to be fed to a model. Returned when `text` is not `None`.
|
||||
- **attention_mask** -- List of indices specifying which tokens should be attended to by the model (when
|
||||
`return_attention_mask=True` or if *"attention_mask"* is in `self.model_input_names` and if `text` is not
|
||||
`None`).
|
||||
- **pixel_values** -- Pixel values to be fed to a model. Returned when `images` is not `None`.
|
||||
- **pixel_values_videos** -- Pixel values of videos to be fed to a model. Returned when `videos` is not `None`.
|
||||
- **image_grid_thw** -- List of image 3D grid in LLM. Returned when `images` is not `None`.
|
||||
- **video_grid_thw** -- List of video 3D grid in LLM. Returned when `videos` is not `None`.
|
||||
- **second_per_grid_ts** -- List of video seconds per time grid. Returned when `videos` is not `None`.
|
||||
|
||||
"""
|
||||
max_partition = kwargs.pop("max_partition", 9)
|
||||
covering_threshold = kwargs.pop("covering_threshold", 0.9)
|
||||
|
||||
output_kwargs = self._merge_kwargs(
|
||||
OvisProcessorKwargs,
|
||||
tokenizer_init_kwargs=self.tokenizer.init_kwargs,
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
# Process all images first
|
||||
image_features = {}
|
||||
if images is not None:
|
||||
processed_images = []
|
||||
image_placeholders_list = []
|
||||
grids = []
|
||||
|
||||
# Process each image
|
||||
for image in images if isinstance(images, list) else [images]:
|
||||
pixel_values, image_placeholders, grid = self.preprocess_image(
|
||||
image=image,
|
||||
max_partition=max_partition,
|
||||
covering_threshold=covering_threshold,
|
||||
**output_kwargs["images_kwargs"],
|
||||
)
|
||||
processed_images.append(pixel_values)
|
||||
image_placeholders_list.append(image_placeholders)
|
||||
grids.append(grid)
|
||||
|
||||
# assign all processed images
|
||||
if processed_images:
|
||||
image_features["image_placeholders"] = image_placeholders_list
|
||||
|
||||
# Process text input
|
||||
if text is not None:
|
||||
if not isinstance(text, list):
|
||||
text = [text]
|
||||
|
||||
tokenized_batched_text = self._tokenize_with_image_symbol(text)
|
||||
image_token_id = self.get_token_value("image_token")
|
||||
replaced_ids_list = []
|
||||
idx = 0
|
||||
for ids_tensor in tokenized_batched_text:
|
||||
if (
|
||||
image_token_id in ids_tensor
|
||||
and "image_placeholders" in image_features
|
||||
):
|
||||
if idx < len(image_features["image_placeholders"]):
|
||||
# Converts in list for ease of use
|
||||
ids_list = ids_tensor.tolist()
|
||||
|
||||
new_ids = []
|
||||
|
||||
# replace placeholders
|
||||
for i, token_id in enumerate(ids_list):
|
||||
if token_id == image_token_id:
|
||||
placeholder_ids = image_features["image_placeholders"][
|
||||
idx
|
||||
]
|
||||
new_ids.extend(placeholder_ids)
|
||||
idx += 1
|
||||
else:
|
||||
new_ids.append(token_id)
|
||||
|
||||
# Converts back to tensors
|
||||
ids_tensor = torch.tensor(new_ids, dtype=torch.long)
|
||||
else:
|
||||
raise RuntimeError(
|
||||
"Mismatch between the images you provided and the number of placeholder present in the text"
|
||||
)
|
||||
|
||||
replaced_ids_list.append(ids_tensor)
|
||||
|
||||
if replaced_ids_list:
|
||||
replaced_and_tokenized_ids = torch.stack(replaced_ids_list)
|
||||
else:
|
||||
replaced_and_tokenized_ids = torch.tensor([], dtype=torch.long)
|
||||
|
||||
# Create the output with text features
|
||||
output = BatchFeature(
|
||||
data={
|
||||
"input_ids": replaced_and_tokenized_ids,
|
||||
}
|
||||
)
|
||||
|
||||
# Add image features if present
|
||||
if image_features:
|
||||
output["pixel_values"] = processed_images
|
||||
output["grids"] = grids
|
||||
|
||||
return output
|
||||
|
||||
# If only images were provided
|
||||
return BatchFeature(data=image_features)
|
||||
|
||||
def _tokenize_with_image_symbol(self, text_list: list[str]) -> torch.LongTensor:
|
||||
batch_token_ids = []
|
||||
for text in text_list:
|
||||
text_chunks = [
|
||||
self.tokenizer(chunk, add_special_tokens=False).input_ids
|
||||
for chunk in text.split(self.image_token)
|
||||
]
|
||||
token_ids = []
|
||||
num_chuck = len(text_chunks)
|
||||
for i, chunk in enumerate(text_chunks):
|
||||
token_ids.extend(chunk)
|
||||
if i < num_chuck - 1:
|
||||
token_ids.append(self.get_token_value("image_token"))
|
||||
batch_token_ids.append(token_ids)
|
||||
return torch.tensor(batch_token_ids, dtype=torch.long)
|
||||
|
||||
def get_image_size(self):
|
||||
size = self.image_processor.size
|
||||
if "shortest_edge" in size:
|
||||
width = height = size["shortest_edge"]
|
||||
elif "height" in size and "width" in size:
|
||||
width = size["width"]
|
||||
height = size["height"]
|
||||
else:
|
||||
raise ValueError("Can't parse image size from image_processor config.")
|
||||
return height, width
|
||||
|
||||
def get_token_value(self, tok):
|
||||
return self.extra_special_tokens[tok]
|
||||
|
||||
def construct_image_indicators(self, grid):
|
||||
image_placeholders = [
|
||||
self.get_token_value("image_start"),
|
||||
self.get_token_value("image_atom"),
|
||||
self.get_token_value("image_prefix"),
|
||||
]
|
||||
if grid[0] * grid[1] > 1:
|
||||
for r in range(grid[0]):
|
||||
for c in range(grid[1]):
|
||||
image_placeholders.append(self.get_token_value("image_atom"))
|
||||
if c < grid[1] - 1:
|
||||
image_placeholders.append(self.get_token_value("image_col_sep"))
|
||||
if r < grid[0] - 1:
|
||||
image_placeholders.append(self.get_token_value("image_row_sep"))
|
||||
image_placeholders.append(self.get_token_value("image_end"))
|
||||
return image_placeholders
|
||||
|
||||
def construct_image_placeholders(self, grid):
|
||||
image_placeholders = self.construct_image_indicators(grid)
|
||||
|
||||
image_atom_token_id = self.get_token_value("image_atom")
|
||||
# Extract the padding token ID from tokenizer
|
||||
image_padding_token_id = self.get_token_value("image_pad")
|
||||
|
||||
# Create a new list with padding tokens inserted
|
||||
padded_placeholder_tokens = []
|
||||
for token in image_placeholders:
|
||||
padded_placeholder_tokens.append(image_padding_token_id)
|
||||
if token == image_atom_token_id:
|
||||
padded_placeholder_tokens.extend(
|
||||
[image_padding_token_id] * self.image_segment_len
|
||||
)
|
||||
return padded_placeholder_tokens
|
||||
|
||||
def preprocess_image(
|
||||
self,
|
||||
image: PIL.Image.Image,
|
||||
max_partition,
|
||||
covering_threshold,
|
||||
do_convert_rgb,
|
||||
return_tensors,
|
||||
):
|
||||
def _preprocess(img: PIL.Image.Image, side):
|
||||
# first resize and preprocess
|
||||
w, h = img.size
|
||||
if w == h:
|
||||
new_width = new_height = side
|
||||
elif w > h:
|
||||
new_width = side
|
||||
new_height = int(h / w * new_width)
|
||||
else:
|
||||
new_height = side
|
||||
new_width = int(w / h * new_height)
|
||||
new_size = dict(height=new_height, width=new_width)
|
||||
pixel_values = self.image_processor.preprocess(
|
||||
img, size=new_size, return_tensors=return_tensors
|
||||
)["pixel_values"]
|
||||
|
||||
# then pad to square
|
||||
square_values = torch.zeros(
|
||||
[1, 3, side, side], dtype=pixel_values.dtype, device=pixel_values.device
|
||||
)
|
||||
new_height, new_width = pixel_values.shape[2:]
|
||||
if new_height == new_width:
|
||||
square_values[:, :, :, :] = pixel_values
|
||||
elif new_height > new_width:
|
||||
from_index = (side - new_width) // 2
|
||||
square_values[:, :, :, from_index : from_index + new_width] = (
|
||||
pixel_values
|
||||
)
|
||||
else:
|
||||
from_index = (side - new_height) // 2
|
||||
square_values[:, :, from_index : from_index + new_height, :] = (
|
||||
pixel_values
|
||||
)
|
||||
|
||||
return square_values
|
||||
|
||||
def _partition(img, grid) -> list[tuple[int, int, int, int]]:
|
||||
w, h = img.size
|
||||
row_height = h // grid[0]
|
||||
col_width = w // grid[1]
|
||||
|
||||
partition = []
|
||||
for row in range(grid[0]):
|
||||
for col in range(grid[1]):
|
||||
left = col * col_width
|
||||
upper = row * row_height
|
||||
right = w if col == grid[1] - 1 else (col + 1) * col_width
|
||||
lower = h if row == grid[0] - 1 else (row + 1) * row_height
|
||||
partition.append((left, upper, right, lower))
|
||||
|
||||
return partition
|
||||
|
||||
def _covering_area(left, upper, right, lower, side):
|
||||
w = right - left
|
||||
h = lower - upper
|
||||
w, h = max(w, h), min(w, h)
|
||||
if w > side:
|
||||
h = h / w * side
|
||||
w = side
|
||||
return w * h
|
||||
|
||||
def _get_best_grid(img, side):
|
||||
img_area = img.size[0] * img.size[1]
|
||||
|
||||
candidate_grids = []
|
||||
for i in range(1, max_partition + 1):
|
||||
for j in range(1, max_partition + 1):
|
||||
if i * j <= max_partition:
|
||||
candidate_grids.append((i, j))
|
||||
|
||||
all_grids = []
|
||||
good_grids = []
|
||||
for grid in candidate_grids:
|
||||
partition = _partition(img, grid)
|
||||
covering_ratio = (
|
||||
sum([_covering_area(*p, side) for p in partition]) / img_area
|
||||
)
|
||||
assert covering_ratio <= 1.0
|
||||
all_grids.append((grid, covering_ratio))
|
||||
if covering_ratio > covering_threshold:
|
||||
good_grids.append((grid, covering_ratio))
|
||||
|
||||
if len(good_grids) > 0:
|
||||
# pick the good partition with minimum #sub_images and break the tie using covering_ratio
|
||||
return sorted(good_grids, key=lambda x: (x[0][0] * x[0][1], -x[1]))[0][
|
||||
0
|
||||
]
|
||||
else:
|
||||
# pick the partition with maximum covering_ratio and break the tie using #sub_images
|
||||
return sorted(all_grids, key=lambda x: (-x[1], x[0][0] * x[0][1]))[0][0]
|
||||
|
||||
if do_convert_rgb:
|
||||
image = convert_image_mode(image, "RGB")
|
||||
|
||||
sides = self.get_image_size()
|
||||
if sides[0] != sides[1]:
|
||||
raise ValueError("get_image_size() returns non-square size")
|
||||
side = sides[0]
|
||||
grid = _get_best_grid(image, side)
|
||||
partition = _partition(image, grid)
|
||||
crops = [image.crop(p) for p in partition]
|
||||
if len(crops) > 1:
|
||||
crops.insert(0, image)
|
||||
pixel_values = torch.cat([_preprocess(crop, side) for crop in crops], dim=0)
|
||||
image_placeholders = self.construct_image_placeholders(grid)
|
||||
return torch.tensor(pixel_values), image_placeholders, torch.tensor(grid)
|
||||
|
||||
def batch_decode(self, *args, **kwargs):
|
||||
"""This method forwards all its arguments to Qwen2TokenizerFast's [`~PythonBackend.batch_decode`]. Please
|
||||
refer to the docstring of this method for more information.
|
||||
"""
|
||||
return self.tokenizer.batch_decode(*args, **kwargs)
|
||||
|
||||
def decode(self, *args, **kwargs):
|
||||
"""This method forwards all its arguments to Qwen2TokenizerFast's [`~PythonBackend.decode`]. Please refer to
|
||||
the docstring of this method for more information.
|
||||
"""
|
||||
return self.tokenizer.decode(*args, **kwargs)
|
||||
|
||||
def post_process_image_text_to_text(self, generated_outputs):
|
||||
"""Post-process the output of the model to decode the text.
|
||||
|
||||
Args:
|
||||
generated_outputs (`torch.Tensor` or `np.ndarray`):
|
||||
The output of the model `generate` function. The output is expected to be a tensor of shape `(batch_size, sequence_length)`
|
||||
or `(sequence_length,)`.
|
||||
|
||||
Returns:
|
||||
`list[str]`: The decoded text.
|
||||
|
||||
"""
|
||||
return self.tokenizer.batch_decode(
|
||||
generated_outputs,
|
||||
skip_special_tokens=True,
|
||||
clean_up_tokenization_spaces=False,
|
||||
)
|
||||
|
||||
@property
|
||||
def model_input_names(self):
|
||||
tokenizer_input_names = self.tokenizer.model_input_names
|
||||
image_processor_input_names = self.image_processor.model_input_names
|
||||
names_from_processor = list(
|
||||
dict.fromkeys(tokenizer_input_names + image_processor_input_names)
|
||||
)
|
||||
return names_from_processor + ["second_per_grid_ts"]
|
||||
@@ -1,476 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import math
|
||||
from functools import cached_property
|
||||
|
||||
import numpy as np
|
||||
import PIL
|
||||
import torch
|
||||
from transformers import BatchFeature
|
||||
from transformers.image_utils import ImageInput
|
||||
from transformers.processing_utils import ProcessingKwargs, ProcessorMixin, Unpack
|
||||
from transformers.tokenization_utils_base import PreTokenizedInput, TextInput
|
||||
|
||||
__all__ = ["Ovis2_5Processor"]
|
||||
IMAGE_TOKEN = "<image>"
|
||||
VIDEO_TOKEN = "<video>"
|
||||
MIN_PIXELS = 448 * 448
|
||||
MAX_PIXELS = 1792 * 1792
|
||||
|
||||
|
||||
class Ovis2_5ProcessorKwargs(ProcessingKwargs, total=False): # type: ignore[call-arg]
|
||||
_defaults = {
|
||||
"text_kwargs": {
|
||||
"padding": False,
|
||||
},
|
||||
"images_kwargs": {
|
||||
"do_convert_rgb": True,
|
||||
},
|
||||
"videos_kwargs": {
|
||||
"do_convert_rgb": True,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
class Ovis2_5Processor(ProcessorMixin):
|
||||
r"""Constructs an Ovis processor which wraps an Ovis image processor
|
||||
and a Qwen2 tokenizer into a single processor.
|
||||
[`OvisProcessor`] offers all the functionalities of
|
||||
[`Qwen2VLImageProcessor`] and [`Qwen2TokenizerFast`].
|
||||
See the [`~OvisProcessor.__call__`] and [`~OvisProcessor.decode`]
|
||||
for more information.
|
||||
|
||||
Args:
|
||||
image_processor ([`Qwen2VLImageProcessor`], *optional*):
|
||||
The image processor is a required input.
|
||||
tokenizer ([`Qwen2TokenizerFast`], *optional*):
|
||||
The tokenizer is a required input.
|
||||
chat_template (`str`, *optional*): A Jinja template which will
|
||||
be used to convert lists of messages in a chat into
|
||||
a tokenizable string.
|
||||
|
||||
"""
|
||||
|
||||
attributes = ["image_processor", "tokenizer"]
|
||||
valid_kwargs = ["chat_template", "image_pad_token"]
|
||||
|
||||
image_processor_class = "AutoImageProcessor"
|
||||
tokenizer_class = "AutoTokenizer"
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
image_processor=None,
|
||||
tokenizer=None,
|
||||
chat_template=None,
|
||||
image_pad_token=None,
|
||||
patch_size=16,
|
||||
hidden_stride=2,
|
||||
temporal_patch_size=1,
|
||||
**kwargs,
|
||||
):
|
||||
self.image_token = IMAGE_TOKEN
|
||||
self.video_token = VIDEO_TOKEN
|
||||
self.image_pad_token = "<|image_pad|>"
|
||||
|
||||
self.patch_size = patch_size
|
||||
self.hidden_stride = hidden_stride
|
||||
self.temporal_patch_size = temporal_patch_size
|
||||
super().__init__(image_processor, tokenizer, chat_template=chat_template)
|
||||
|
||||
@cached_property
|
||||
def extra_special_tokens(self):
|
||||
required_tokens = {
|
||||
"image_token": "<image>",
|
||||
"video_token": "<video>",
|
||||
"visual_atom": "<ovis_visual_atom>",
|
||||
"image_start": "<ovis_image_start>",
|
||||
"image_end": "<ovis_image_end>",
|
||||
"video_start": "<ovis_video_start>",
|
||||
"video_end": "<ovis_video_end>",
|
||||
"image_pad": "<|image_pad|>",
|
||||
}
|
||||
|
||||
# The checkpoint defines both `additional_special_tokens` and
|
||||
# `extra_special_tokens`, with the latter empty. Transformers ignores
|
||||
# the former because the latter is explicitly empty, so the tokens are
|
||||
# missing from the vocab. Re-add them to restore the expected ids.
|
||||
self.tokenizer.add_tokens(list(required_tokens.values()), special_tokens=True)
|
||||
|
||||
# in TokenizerPool tokenizer.get_added_vocab() gets values from an instance
|
||||
# that was updated with tokenizer.add_tokens(), so it is favourable over
|
||||
# self.tokenizer.convert_tokens_to_ids(), which refers to a copy before
|
||||
# mutation by tokenizer.add_tokens()
|
||||
added_vocab = self.tokenizer.get_added_vocab()
|
||||
|
||||
return {
|
||||
key: added_vocab[token_name] for key, token_name in required_tokens.items()
|
||||
}
|
||||
|
||||
def __call__(
|
||||
self,
|
||||
images: ImageInput = None,
|
||||
videos: np.ndarray | list[ImageInput] = None,
|
||||
text: TextInput
|
||||
| PreTokenizedInput
|
||||
| list[TextInput]
|
||||
| list[PreTokenizedInput] = None,
|
||||
**kwargs: Unpack[Ovis2_5ProcessorKwargs],
|
||||
) -> BatchFeature:
|
||||
"""Main method to prepare for the model one or several sequences(s)
|
||||
and image(s). This method forwards the `text`and `kwargs` arguments
|
||||
to Qwen2TokenizerFast's [`~Qwen2TokenizerFast.__call__`] if `text`
|
||||
is not `None` to encode the text. To prepare the vision inputs,
|
||||
this method forwards the `vision_infos` and `kwargs` arguments to
|
||||
Qwen2VLImageProcessor's [`~Qwen2VLImageProcessor.__call__`]
|
||||
if `vision_infos` is not `None`.
|
||||
|
||||
Args:
|
||||
images (`PIL.Image.Image`, `np.ndarray`, `torch.Tensor`, or a list):
|
||||
The image or batch of images to be prepared. Each image can
|
||||
be a PIL image, NumPy array or PyTorch tensor. Both
|
||||
channels-first and channels-last formats are supported.
|
||||
videos (`np.ndarray`, `torch.Tensor`, or a list of either):
|
||||
The video or batch of videos to be prepared. Each video can
|
||||
be a 4D NumPy array or PyTorch tensor, or a nested list of
|
||||
3D frames. Both channels-first and channels-last formats are
|
||||
supported.
|
||||
text (`str`, `list[str]`, `list[list[str]]`):
|
||||
The sequence or batch of sequences to be encoded. Each
|
||||
sequence can be a string or a list of strings (pretokenized
|
||||
string). If the sequences are provided as list of strings
|
||||
(pretokenized), you must set `is_split_into_words=True` (to
|
||||
lift the ambiguity with a batch of sequences).
|
||||
return_tensors (`str` or [`~utils.TensorType`], *optional*):
|
||||
If set, will return tensors of a particular framework.
|
||||
Acceptable values are:
|
||||
- `'tf'`: Return TensorFlow `tf.constant` objects.
|
||||
- `'pt'`: Return PyTorch `torch.Tensor` objects.
|
||||
- `'np'`: Return NumPy `np.ndarray` objects.
|
||||
- `'jax'`: Return JAX `jnp.ndarray` objects.
|
||||
|
||||
Returns:
|
||||
[`BatchFeature`]: A [`BatchFeature`] with the following fields:
|
||||
- **input_ids** -- list of token ids to be fed to a model.
|
||||
Returned when `text` is not `None`.
|
||||
- **attention_mask** -- list of indices specifying which tokens
|
||||
should be attended to by the model (when
|
||||
`return_attention_mask=True` or if *"attention_mask"*
|
||||
is in `self.model_input_names` and if `text` is not `None`).
|
||||
- **pixel_values** -- Pixel values to be fed to a model.
|
||||
Returned when `images` is not `None`.
|
||||
- **pixel_values_videos** -- Pixel values of videos to be fed to
|
||||
a model. Returned when `videos` is not `None`.
|
||||
- **image_grid_thw** -- list of image 3D grid in LLM. Returned
|
||||
when `images` is not `None`.
|
||||
- **video_grid_thw** -- list of video 3D grid in LLM. Returned
|
||||
when `videos` is not `None`.
|
||||
- **second_per_grid_ts** -- list of video seconds per time grid.
|
||||
Returned when `videos` is not `None`.
|
||||
|
||||
"""
|
||||
output_kwargs = self._merge_kwargs(
|
||||
Ovis2_5ProcessorKwargs,
|
||||
tokenizer_init_kwargs=self.tokenizer.init_kwargs,
|
||||
**kwargs,
|
||||
)
|
||||
# Process all images first
|
||||
visual_features = {}
|
||||
output = BatchFeature()
|
||||
if images is not None:
|
||||
processed_images = []
|
||||
image_placeholders_list = []
|
||||
grids = []
|
||||
# Process each image
|
||||
for image in images if isinstance(images, list) else [images]:
|
||||
pixel_values, image_placeholders, grid = self.preprocess_multidata(
|
||||
images=image,
|
||||
**output_kwargs["images_kwargs"],
|
||||
)
|
||||
processed_images.append(pixel_values)
|
||||
image_placeholders_list.append(image_placeholders)
|
||||
grids.append(grid)
|
||||
|
||||
# assign all processed images
|
||||
if processed_images:
|
||||
visual_features["image_placeholders"] = image_placeholders_list
|
||||
output["pixel_values"] = processed_images
|
||||
output["grids"] = grids
|
||||
|
||||
if videos is not None:
|
||||
processed_videos = []
|
||||
videos_placeholders_list = []
|
||||
grids = []
|
||||
# Process each video
|
||||
for video in videos if isinstance(videos, list) else [videos]:
|
||||
pixel_values, video_placeholders, grid = self.preprocess_multidata(
|
||||
video=video,
|
||||
**output_kwargs["videos_kwargs"],
|
||||
)
|
||||
processed_videos.append(pixel_values)
|
||||
videos_placeholders_list.append(video_placeholders)
|
||||
grids.append(grid)
|
||||
# assign all processed videos
|
||||
if processed_videos:
|
||||
visual_features["video_placeholders"] = videos_placeholders_list
|
||||
output["video_pixel_values"] = processed_videos
|
||||
output["video_grids"] = grids
|
||||
|
||||
# Process text input
|
||||
if text is not None:
|
||||
if not isinstance(text, list):
|
||||
text = [text]
|
||||
tokenized_batched_text = self._tokenize_with_visual_symbol(text)
|
||||
image_token_id = self.get_token_value("image_token")
|
||||
video_token_id = self.get_token_value("video_token")
|
||||
replaced_ids_list = []
|
||||
image_idx = 0
|
||||
video_idx = 0
|
||||
for ids_tensor in tokenized_batched_text:
|
||||
has_image_tokens = (
|
||||
image_token_id in ids_tensor
|
||||
and "image_placeholders" in visual_features
|
||||
and image_idx < len(visual_features["image_placeholders"])
|
||||
)
|
||||
has_video_tokens = (
|
||||
video_token_id in ids_tensor
|
||||
and "video_placeholders" in visual_features
|
||||
and video_idx < len(visual_features["video_placeholders"])
|
||||
)
|
||||
if has_image_tokens or has_video_tokens:
|
||||
# Convert to list for easier manipulation
|
||||
ids_list = ids_tensor.tolist()
|
||||
new_ids = []
|
||||
|
||||
# Replace placeholders
|
||||
for token_id in ids_list:
|
||||
if token_id == image_token_id:
|
||||
new_ids.extend(
|
||||
visual_features["image_placeholders"][image_idx]
|
||||
)
|
||||
image_idx += 1
|
||||
elif token_id == video_token_id:
|
||||
new_ids.extend(
|
||||
visual_features["video_placeholders"][video_idx]
|
||||
)
|
||||
video_idx += 1
|
||||
else:
|
||||
new_ids.append(token_id)
|
||||
# Convert back to tensor
|
||||
ids_tensor = torch.tensor(new_ids, dtype=torch.long)
|
||||
replaced_ids_list.append(ids_tensor)
|
||||
if replaced_ids_list:
|
||||
replaced_and_tokenized_ids = torch.stack(replaced_ids_list)
|
||||
else:
|
||||
replaced_and_tokenized_ids = torch.tensor([], dtype=torch.long)
|
||||
output["input_ids"] = replaced_and_tokenized_ids
|
||||
|
||||
return output
|
||||
# If only images were provided
|
||||
return BatchFeature(data=visual_features)
|
||||
|
||||
def _tokenize_with_visual_symbol(self, text_list: list[str]) -> torch.LongTensor:
|
||||
batch_token_ids = []
|
||||
for text in text_list:
|
||||
token_ids = []
|
||||
video_token_id = self.get_token_value("video_token")
|
||||
image_token_id = self.get_token_value("image_token")
|
||||
video_split_texts = text.split(self.video_token)
|
||||
|
||||
for j, video_segment in enumerate(video_split_texts):
|
||||
image_split_texts = video_segment.split(self.image_token)
|
||||
text_chunks = [
|
||||
self.tokenizer(chunk, add_special_tokens=False).input_ids
|
||||
for chunk in image_split_texts
|
||||
]
|
||||
segment_tokens = []
|
||||
for i, chunk in enumerate(text_chunks):
|
||||
segment_tokens.extend(chunk)
|
||||
if i < len(text_chunks) - 1:
|
||||
segment_tokens.append(image_token_id)
|
||||
token_ids.extend(segment_tokens)
|
||||
if j < len(video_split_texts) - 1:
|
||||
token_ids.append(video_token_id)
|
||||
|
||||
batch_token_ids.append(token_ids)
|
||||
return torch.tensor(batch_token_ids, dtype=torch.long)
|
||||
|
||||
# Copied from qwen2_vl
|
||||
def smart_resize(
|
||||
self,
|
||||
height: int,
|
||||
width: int,
|
||||
factor: int = 28,
|
||||
min_pixels: int = MIN_PIXELS,
|
||||
max_pixels: int = MAX_PIXELS,
|
||||
):
|
||||
"""Rescales the image so that the following conditions are met:
|
||||
1. Both dimensions (height and width) are divisible by 'factor'.
|
||||
2. The total number of pixels is within the range
|
||||
['min_pixels', 'max_pixels'].
|
||||
3. The aspect ratio of the image is maintained as closely as possible.
|
||||
"""
|
||||
if height < factor or width < factor:
|
||||
print(
|
||||
f"height:{height} or width:{width} must be larger than factor:{factor}"
|
||||
)
|
||||
if height < width:
|
||||
width = round(factor / height * width)
|
||||
height = factor
|
||||
else:
|
||||
height = round(factor / width * height)
|
||||
width = factor
|
||||
|
||||
elif max(height, width) / min(height, width) > 200:
|
||||
print(
|
||||
f"absolute aspect ratio must be smaller than 200, "
|
||||
f"got {max(height, width) / min(height, width)}"
|
||||
)
|
||||
if height > width:
|
||||
height = 200 * width
|
||||
else:
|
||||
width = 200 * height
|
||||
|
||||
h_bar = round(height / factor) * factor
|
||||
w_bar = round(width / factor) * factor
|
||||
if h_bar * w_bar > max_pixels:
|
||||
beta = math.sqrt((height * width) / max_pixels)
|
||||
h_bar = math.floor(height / beta / factor) * factor
|
||||
w_bar = math.floor(width / beta / factor) * factor
|
||||
elif h_bar * w_bar < min_pixels:
|
||||
beta = math.sqrt(min_pixels / (height * width))
|
||||
h_bar = math.ceil(height * beta / factor) * factor
|
||||
w_bar = math.ceil(width * beta / factor) * factor
|
||||
return h_bar, w_bar
|
||||
|
||||
def get_token_value(self, tok):
|
||||
return self.extra_special_tokens[tok]
|
||||
|
||||
def construct_visual_indicators(self, grid, is_video: bool = False):
|
||||
if is_video:
|
||||
start_token = self.get_token_value("video_start")
|
||||
end_token = self.get_token_value("video_end")
|
||||
else:
|
||||
start_token = self.get_token_value("image_start")
|
||||
end_token = self.get_token_value("image_end")
|
||||
|
||||
image_placeholders = [start_token, self.get_token_value("visual_atom")]
|
||||
if grid[0] * grid[1] > 1:
|
||||
for r in range(grid[0]):
|
||||
for c in range(grid[1]):
|
||||
image_placeholders.append(self.get_token_value("visual_atom"))
|
||||
|
||||
image_placeholders.append(end_token)
|
||||
return image_placeholders
|
||||
|
||||
def construct_visual_placeholders(self, grid, is_video: bool = False):
|
||||
visual_placeholders = self.construct_visual_indicators((1, 1), is_video)
|
||||
|
||||
image_atom_token_id = self.get_token_value("visual_atom")
|
||||
# Extract the padding token ID from tokenizer
|
||||
image_padding_token_id = self.get_token_value("image_pad")
|
||||
|
||||
num_image_atoms = grid[0] * grid[1] * grid[2]
|
||||
num_image_atoms //= self.hidden_stride**2
|
||||
num_image_atoms //= self.temporal_patch_size
|
||||
|
||||
# Create a new list with padding tokens inserted
|
||||
padded_placeholder_tokens = []
|
||||
for token in visual_placeholders:
|
||||
if token == image_atom_token_id:
|
||||
padded_placeholder_tokens.extend(
|
||||
[image_padding_token_id] * num_image_atoms
|
||||
)
|
||||
else:
|
||||
padded_placeholder_tokens.append(image_padding_token_id)
|
||||
return padded_placeholder_tokens
|
||||
|
||||
def preprocess_multidata(
|
||||
self,
|
||||
images: PIL.Image.Image | list[PIL.Image.Image] | None = None,
|
||||
video: list[PIL.Image.Image] | np.ndarray | None = None,
|
||||
do_convert_rgb: bool | None = True,
|
||||
min_pixels: int = MIN_PIXELS,
|
||||
max_pixels: int = MAX_PIXELS,
|
||||
return_tensors: str | None = "pt",
|
||||
):
|
||||
is_video = False
|
||||
if images is not None:
|
||||
if not isinstance(images, list):
|
||||
images = [images]
|
||||
elif video is not None:
|
||||
is_video = True
|
||||
# type of video in dummy_mm_data is np.ndarray
|
||||
if isinstance(video, np.ndarray):
|
||||
images = []
|
||||
for i in range(video.shape[0]):
|
||||
image = PIL.Image.fromarray(video[i].astype(np.uint8))
|
||||
images.append(image)
|
||||
elif isinstance(video, list):
|
||||
images = video
|
||||
else:
|
||||
raise ValueError("Either images or video should be provided.")
|
||||
assert images is not None
|
||||
min_pixels = min(
|
||||
max_pixels if max_pixels is not None else MAX_PIXELS,
|
||||
min_pixels if min_pixels is not None else MIN_PIXELS,
|
||||
)
|
||||
images = [
|
||||
image.convert("RGB") if do_convert_rgb and image.mode != "RGB" else image
|
||||
for image in images
|
||||
]
|
||||
|
||||
width, height = images[0].size
|
||||
resized_height, resized_width = height, width
|
||||
processed_images = []
|
||||
for image in images:
|
||||
resized_height, resized_width = self.smart_resize(
|
||||
height,
|
||||
width,
|
||||
factor=self.patch_size * self.hidden_stride,
|
||||
min_pixels=min_pixels,
|
||||
max_pixels=max_pixels,
|
||||
)
|
||||
new_size = dict(height=resized_height, width=resized_width)
|
||||
image_pt = self.image_processor.preprocess(image, size=new_size)[
|
||||
"pixel_values"
|
||||
][0]
|
||||
|
||||
processed_images.append(image_pt)
|
||||
|
||||
patches = np.array(processed_images)
|
||||
if patches.shape[0] % self.temporal_patch_size != 0:
|
||||
num_to_pad = self.temporal_patch_size - (
|
||||
patches.shape[0] % self.temporal_patch_size
|
||||
)
|
||||
repeats = np.repeat(patches[-1][np.newaxis], num_to_pad, axis=0)
|
||||
patches = np.concatenate([patches, repeats], axis=0)
|
||||
channel = patches.shape[1]
|
||||
grid_t = patches.shape[0] // self.temporal_patch_size
|
||||
grid_h = resized_height // self.patch_size
|
||||
grid_w = resized_width // self.patch_size
|
||||
|
||||
patches = patches.reshape(
|
||||
grid_t,
|
||||
self.temporal_patch_size,
|
||||
channel,
|
||||
grid_h // self.hidden_stride,
|
||||
self.hidden_stride,
|
||||
self.patch_size,
|
||||
grid_w // self.hidden_stride,
|
||||
self.hidden_stride,
|
||||
self.patch_size,
|
||||
)
|
||||
patches = patches.transpose(0, 3, 6, 4, 7, 2, 1, 5, 8)
|
||||
flatten_patches = patches.reshape(
|
||||
grid_t * grid_h * grid_w,
|
||||
channel * self.temporal_patch_size * self.patch_size * self.patch_size,
|
||||
)
|
||||
|
||||
visual_placeholders = self.construct_visual_placeholders(
|
||||
[grid_t, grid_h, grid_w], is_video
|
||||
)
|
||||
return (
|
||||
torch.tensor(flatten_patches),
|
||||
visual_placeholders,
|
||||
torch.tensor([[grid_t, grid_h, grid_w]]),
|
||||
)
|
||||
@@ -1,79 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import torch
|
||||
from mistral_common.protocol.instruct.chunk import ImageChunk
|
||||
from mistral_common.tokens.tokenizers.multimodal import ImageEncoder
|
||||
from PIL import Image
|
||||
from transformers import BatchFeature, ImageProcessingMixin, ProcessorMixin, TensorType
|
||||
from transformers.image_utils import ImageInput
|
||||
|
||||
from vllm.tokenizers.mistral import MistralTokenizer
|
||||
|
||||
|
||||
class MistralCommonImageProcessor(ImageProcessingMixin):
|
||||
"""Provide a HF-compatible interface for
|
||||
`mistral_common.tokens.tokenizers.multimodal.ImageEncoder`.
|
||||
"""
|
||||
|
||||
def __init__(self, mm_encoder: ImageEncoder) -> None:
|
||||
self.mm_encoder = mm_encoder
|
||||
|
||||
def __call__(
|
||||
self,
|
||||
images: ImageInput,
|
||||
return_tensors: str | TensorType | None = None,
|
||||
**kwargs,
|
||||
) -> BatchFeature:
|
||||
images_lst = [images] if not isinstance(images, list) else images
|
||||
|
||||
images_processed = list[torch.Tensor]()
|
||||
|
||||
for image in images_lst:
|
||||
image_inputs = self.mm_encoder(ImageChunk(image=image))
|
||||
image_processed = torch.tensor(image_inputs.image)
|
||||
|
||||
images_processed.append(image_processed)
|
||||
|
||||
return BatchFeature({"images": images_processed}, tensor_type=return_tensors)
|
||||
|
||||
def get_number_of_image_patches(
|
||||
self,
|
||||
height: int,
|
||||
width: int,
|
||||
) -> tuple[int, int, int]:
|
||||
image = Image.new("RGB", (width, height))
|
||||
ncols, nrows = self.mm_encoder._image_to_num_tokens(image)
|
||||
return ncols * nrows, nrows, ncols
|
||||
|
||||
# Copied from Transformers (Apache-2.0):
|
||||
# https://github.com/huggingface/transformers/blob/d20946079fd422335fbae3eeb98b7cd88334612f/src/transformers/image_processing_base.py#L473
|
||||
def fetch_images(self, image_url_or_urls):
|
||||
from transformers.image_utils import is_valid_image, load_image
|
||||
|
||||
if isinstance(image_url_or_urls, (list, tuple)):
|
||||
return [self.fetch_images(x) for x in image_url_or_urls]
|
||||
if isinstance(image_url_or_urls, str):
|
||||
return load_image(image_url_or_urls)
|
||||
if is_valid_image(image_url_or_urls):
|
||||
return image_url_or_urls
|
||||
raise TypeError(
|
||||
"only a single or a list of entries is supported but got "
|
||||
f"type={type(image_url_or_urls)}"
|
||||
)
|
||||
|
||||
|
||||
class MistralCommonPixtralProcessor(ProcessorMixin):
|
||||
attributes = ["image_processor", "tokenizer"]
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
tokenizer: MistralTokenizer,
|
||||
image_processor: MistralCommonImageProcessor,
|
||||
) -> None:
|
||||
self.tokenizer = tokenizer.transformers_tokenizer
|
||||
self.image_processor = image_processor
|
||||
|
||||
image_special_ids = self.image_processor.mm_encoder.special_ids
|
||||
self.image_break_id = image_special_ids.img_break
|
||||
self.image_token_id = image_special_ids.img
|
||||
self.image_end_id = image_special_ids.img_end
|
||||
@@ -1,514 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from itertools import product
|
||||
from math import ceil
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
from PIL import Image
|
||||
from torchvision import transforms
|
||||
from torchvision.transforms.functional import InterpolationMode
|
||||
from transformers import (
|
||||
BatchFeature,
|
||||
ImageProcessingMixin,
|
||||
ProcessorMixin,
|
||||
TensorType,
|
||||
)
|
||||
|
||||
from vllm.tokenizers import TokenizerLike
|
||||
|
||||
MAX_IMAGE_SIZE: int = 3024
|
||||
|
||||
ImageWithPatches = tuple[Image.Image, list[Image.Image], list[bool]]
|
||||
|
||||
|
||||
class Step3VisionProcessor:
|
||||
def __init__(self, size, interpolation_mode="bicubic", patch_size=None):
|
||||
mean = [0.48145466, 0.4578275, 0.40821073]
|
||||
std = [0.26862954, 0.26130258, 0.27577711]
|
||||
patch_size = patch_size if patch_size is not None else size
|
||||
|
||||
self.transform = transforms.Compose(
|
||||
[
|
||||
transforms.ToTensor(),
|
||||
transforms.Normalize(mean, std),
|
||||
transforms.Resize(
|
||||
(size, size),
|
||||
interpolation=InterpolationMode.BICUBIC
|
||||
if interpolation_mode == "bicubic"
|
||||
else InterpolationMode.BILINEAR,
|
||||
antialias=True,
|
||||
),
|
||||
]
|
||||
)
|
||||
|
||||
self.patch_transform = (
|
||||
transforms.Compose(
|
||||
[
|
||||
transforms.ToTensor(),
|
||||
transforms.Normalize(mean, std),
|
||||
transforms.Resize(
|
||||
(patch_size, patch_size),
|
||||
interpolation=InterpolationMode.BICUBIC
|
||||
if interpolation_mode == "bicubic"
|
||||
else InterpolationMode.BILINEAR,
|
||||
antialias=True,
|
||||
),
|
||||
]
|
||||
)
|
||||
if patch_size is not None
|
||||
else None
|
||||
)
|
||||
|
||||
def __call__(self, image, is_patch=False):
|
||||
if is_patch:
|
||||
assert self.patch_transform is not None
|
||||
return {"pixel_values": self.patch_transform(image).unsqueeze(0)}
|
||||
|
||||
return {"pixel_values": self.transform(image).unsqueeze(0)}
|
||||
|
||||
|
||||
class ImagePatcher:
|
||||
def __init__(self, enable_patch: bool = True) -> None:
|
||||
self.enable_patch = enable_patch
|
||||
|
||||
def determine_window_size(self, long: int, short: int) -> int:
|
||||
if long < 728:
|
||||
return short if long / short > 1.5 else 0
|
||||
return min(short, 504) if long / short > 4 else 504
|
||||
|
||||
def slide_window(
|
||||
self,
|
||||
width: int,
|
||||
height: int,
|
||||
sizes: list[tuple[int, int]],
|
||||
steps: list[tuple[int, int]],
|
||||
img_rate_thr: float = 0.6,
|
||||
) -> tuple[list[tuple[int, int, int, int]], tuple[int, int]]:
|
||||
assert 1 >= img_rate_thr >= 0, "The `in_rate_thr` should lie in 0~1"
|
||||
windows = []
|
||||
# Sliding windows.
|
||||
for size, step in zip(sizes, steps):
|
||||
size_w, size_h = size
|
||||
step_w, step_h = step
|
||||
|
||||
x_num = 1 if width <= size_w else ceil((width - size_w) / step_w + 1)
|
||||
x_start = [step_w * i for i in range(x_num)]
|
||||
if len(x_start) > 1 and x_start[-1] + size_w > width:
|
||||
x_start[-1] = width - size_w
|
||||
|
||||
y_num = 1 if height <= size_h else ceil((height - size_h) / step_h + 1)
|
||||
y_start = [step_h * i for i in range(y_num)]
|
||||
if len(y_start) > 1 and y_start[-1] + size_h > height:
|
||||
y_start[-1] = height - size_h
|
||||
|
||||
start = np.array(list(product(y_start, x_start)), dtype=int)
|
||||
start[:, [0, 1]] = start[:, [1, 0]]
|
||||
windows.append(np.concatenate([start, start + size], axis=1))
|
||||
windows = np.concatenate(windows, axis=0)
|
||||
|
||||
return [
|
||||
(int(box[0]), int(box[1]), int(box[2] - box[0]), int(box[3] - box[1]))
|
||||
for box in windows
|
||||
], (x_num, y_num)
|
||||
|
||||
def square_pad(self, img: Image.Image) -> Image.Image:
|
||||
w, h = img.size
|
||||
if w == h:
|
||||
return img
|
||||
size = max(w, h)
|
||||
padded = Image.new(img.mode, (size, size), 0)
|
||||
padded.paste(img, (0, 0))
|
||||
return padded
|
||||
|
||||
def get_image_size_for_padding(
|
||||
self, img_width: int, img_height: int
|
||||
) -> tuple[int, int]:
|
||||
ratio = img_width / img_height
|
||||
if min(img_height, img_width) < 32 and (ratio > 4 or ratio < 1 / 4):
|
||||
new_size = max(img_height, img_width)
|
||||
return new_size, new_size
|
||||
return img_width, img_height
|
||||
|
||||
def get_image_size_for_preprocess(
|
||||
self, img_width: int, img_height: int
|
||||
) -> tuple[int, int]:
|
||||
if max(img_height, img_width) > MAX_IMAGE_SIZE:
|
||||
scale_factor = MAX_IMAGE_SIZE / max(img_height, img_width)
|
||||
img_width = int(img_width * scale_factor)
|
||||
img_height = int(img_height * scale_factor)
|
||||
return img_width, img_height
|
||||
|
||||
def get_image_size_for_crop(
|
||||
self, img_width: int, img_height: int, window_size: int
|
||||
):
|
||||
w_ratio = img_width / window_size
|
||||
h_ratio = img_height / window_size
|
||||
|
||||
if w_ratio < 1:
|
||||
width_new = img_width
|
||||
else:
|
||||
decimal_w = w_ratio - img_width // window_size
|
||||
w_ratio = int(w_ratio) + 1 if decimal_w > 0.2 else int(w_ratio)
|
||||
width_new = window_size * w_ratio
|
||||
if h_ratio < 1:
|
||||
height_new = img_height
|
||||
else:
|
||||
decimal_h = h_ratio - img_height // window_size
|
||||
h_ratio = int(h_ratio) + 1 if decimal_h > 0.2 else int(h_ratio)
|
||||
height_new = window_size * h_ratio
|
||||
return int(width_new), int(height_new)
|
||||
|
||||
def patch_crop(self, img: Image.Image, i: int, j: int, th: int, tw: int):
|
||||
target = img.crop((j, i, j + tw, i + th))
|
||||
return target
|
||||
|
||||
def get_num_patches(self, img_width: int, img_height: int) -> tuple[int, int]:
|
||||
img_width, img_height = self.get_image_size_for_padding(img_width, img_height)
|
||||
img_width, img_height = self.get_image_size_for_preprocess(
|
||||
img_width, img_height
|
||||
)
|
||||
window_size = self.determine_window_size(
|
||||
max(img_height, img_width), min(img_height, img_width)
|
||||
)
|
||||
if window_size == 0 or not self.enable_patch:
|
||||
return 0, 0
|
||||
else:
|
||||
img_width, img_height = self.get_image_size_for_crop(
|
||||
img_width, img_height, window_size
|
||||
)
|
||||
center_list, (x_num, y_num) = self.slide_window(
|
||||
img_width,
|
||||
img_height,
|
||||
[(window_size, window_size)],
|
||||
[(window_size, window_size)],
|
||||
)
|
||||
full_rows = (len(center_list) - 1) // x_num + 1
|
||||
if len(center_list) > 0 and len(center_list) % x_num == 0:
|
||||
full_rows -= 1
|
||||
return len(center_list), full_rows
|
||||
|
||||
def __call__(
|
||||
self, img: Image.Image
|
||||
) -> tuple[Image.Image, list[Image.Image], list[bool]]:
|
||||
img_width, img_height = img.size
|
||||
new_img_width, new_img_height = self.get_image_size_for_padding(
|
||||
img_width, img_height
|
||||
)
|
||||
if new_img_width != img_width or new_img_height != img_height:
|
||||
img = self.square_pad(img)
|
||||
img_width, img_height = img.size
|
||||
|
||||
new_img_width, new_img_height = self.get_image_size_for_preprocess(
|
||||
img_width, img_height
|
||||
)
|
||||
img = img.resize((new_img_width, new_img_height), Image.Resampling.BILINEAR)
|
||||
window_size = self.determine_window_size(
|
||||
max(new_img_height, new_img_width), min(new_img_height, new_img_width)
|
||||
)
|
||||
|
||||
if window_size == 0 or not self.enable_patch:
|
||||
return img, [], []
|
||||
else:
|
||||
new_img_width, new_img_height = self.get_image_size_for_crop(
|
||||
new_img_width, new_img_height, window_size
|
||||
)
|
||||
if (new_img_width, new_img_height) != (img_width, img_height):
|
||||
img_for_crop = img.resize(
|
||||
(new_img_width, new_img_height), Image.Resampling.BILINEAR
|
||||
)
|
||||
else:
|
||||
img_for_crop = img
|
||||
|
||||
patches = []
|
||||
newlines = []
|
||||
center_list, (x_num, y_num) = self.slide_window(
|
||||
new_img_width,
|
||||
new_img_height,
|
||||
[(window_size, window_size)],
|
||||
[(window_size, window_size)],
|
||||
)
|
||||
for patch_id, center_lf_point in enumerate(center_list):
|
||||
x, y, patch_w, patch_h = center_lf_point
|
||||
big_patch = self.patch_crop(img_for_crop, y, x, patch_h, patch_w)
|
||||
patches.append(big_patch)
|
||||
if (patch_id + 1) % x_num == 0:
|
||||
newlines.append(patch_id)
|
||||
|
||||
if newlines and newlines[-1] == len(patches) - 1:
|
||||
newlines.pop()
|
||||
|
||||
return (
|
||||
img,
|
||||
patches,
|
||||
[i in newlines for i in range(len(patches))],
|
||||
)
|
||||
|
||||
|
||||
class Step3VLImageProcessor(ImageProcessingMixin):
|
||||
def __init__(
|
||||
self,
|
||||
image_size: int = 728,
|
||||
patch_size: int = 504,
|
||||
num_image_feature_size: int = 169,
|
||||
num_patch_feature_size: int = 81,
|
||||
enable_patch: bool = True,
|
||||
) -> None:
|
||||
self.image_size = image_size
|
||||
self.patch_size = patch_size
|
||||
self.num_image_feature_size = num_image_feature_size
|
||||
self.num_patch_feature_size = num_patch_feature_size
|
||||
self.image_preprocessor = Step3VisionProcessor(
|
||||
image_size, "bilinear", patch_size
|
||||
)
|
||||
self.patcher = ImagePatcher(enable_patch=enable_patch)
|
||||
|
||||
def get_num_image_tokens(self, img_width: int, img_height: int) -> int:
|
||||
num_patches, num_newlines = self.patcher.get_num_patches(img_width, img_height)
|
||||
|
||||
return (
|
||||
num_patches * (self.num_patch_feature_size + 2)
|
||||
+ self.num_image_feature_size
|
||||
+ 2
|
||||
+ num_newlines
|
||||
)
|
||||
|
||||
def _split_images(self, images: list[Image.Image]) -> list[ImageWithPatches]:
|
||||
result = []
|
||||
for img in images:
|
||||
result.append(self.patcher(img))
|
||||
return result
|
||||
|
||||
def _convert_images_to_pixel_values(
|
||||
self,
|
||||
images: list[Image.Image],
|
||||
is_patch: bool = False,
|
||||
) -> list[torch.Tensor]:
|
||||
return [
|
||||
self.image_preprocessor(img, is_patch=is_patch)["pixel_values"]
|
||||
for img in images
|
||||
]
|
||||
|
||||
def __call__(
|
||||
self,
|
||||
images: Image.Image | list[Image.Image],
|
||||
return_tensors: str | TensorType | None = None,
|
||||
) -> BatchFeature:
|
||||
if not isinstance(images, list):
|
||||
images = [images]
|
||||
|
||||
split_images_data = self._split_images(images)
|
||||
pixel_values_lst = []
|
||||
patch_pixel_values_lst = []
|
||||
patch_newline_mask_lst = []
|
||||
num_patches = []
|
||||
for raw_img, img_patches, patch_newline_mask in split_images_data:
|
||||
pixel_values_lst.extend(self._convert_images_to_pixel_values([raw_img]))
|
||||
num_patches.append(len(img_patches))
|
||||
patch_pixel_values_lst.extend(
|
||||
self._convert_images_to_pixel_values(img_patches, is_patch=True)
|
||||
)
|
||||
patch_newline_mask_lst.extend(patch_newline_mask)
|
||||
|
||||
pixel_values = torch.cat(pixel_values_lst)
|
||||
patch_size = self.patch_size
|
||||
image_inputs = {
|
||||
"pixel_values": pixel_values,
|
||||
"num_patches": num_patches,
|
||||
"patch_pixel_values": (
|
||||
torch.cat(patch_pixel_values_lst)
|
||||
if patch_pixel_values_lst
|
||||
else pixel_values.new_empty((0, 3, patch_size, patch_size))
|
||||
),
|
||||
"patch_newline_mask": torch.tensor(
|
||||
patch_newline_mask_lst, dtype=torch.bool
|
||||
),
|
||||
}
|
||||
return BatchFeature(image_inputs, tensor_type=return_tensors)
|
||||
|
||||
|
||||
class Step3VLProcessor(ProcessorMixin):
|
||||
attributes = ["image_processor", "tokenizer"]
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
image_processor: Step3VLImageProcessor,
|
||||
tokenizer: TokenizerLike,
|
||||
) -> None:
|
||||
self.image_processor = image_processor
|
||||
self.tokenizer = tokenizer
|
||||
|
||||
self.image_start_token = image_start_token = "<im_start>"
|
||||
self.image_end_token = image_end_token = "<im_end>"
|
||||
self.patch_start_token = patch_start_token = "<patch_start>"
|
||||
self.patch_end_token = patch_end_token = "<patch_end>"
|
||||
self.patch_newline_token = patch_newline_token = "<patch_newline>"
|
||||
self.image_start_token_id = tokenizer.convert_tokens_to_ids(image_start_token)
|
||||
self.image_end_token_id = tokenizer.convert_tokens_to_ids(image_end_token)
|
||||
self.patch_start_token_id = tokenizer.convert_tokens_to_ids(patch_start_token)
|
||||
self.patch_end_token_id = tokenizer.convert_tokens_to_ids(patch_end_token)
|
||||
self.patch_newline_token_id = tokenizer.convert_tokens_to_ids(
|
||||
patch_newline_token
|
||||
)
|
||||
|
||||
self.image_token = image_token = "<im_patch>"
|
||||
self.image_feature_tokens = image_token * image_processor.num_image_feature_size
|
||||
self.patch_feature_tokens = image_token * image_processor.num_patch_feature_size
|
||||
|
||||
self.image_token_id = image_token_id = tokenizer.convert_tokens_to_ids(
|
||||
image_token
|
||||
)
|
||||
self.image_feature_token_ids = [
|
||||
image_token_id
|
||||
] * image_processor.num_image_feature_size
|
||||
self.patch_feature_token_ids = [
|
||||
image_token_id
|
||||
] * image_processor.num_patch_feature_size
|
||||
|
||||
def _get_patch_repl_text(
|
||||
self,
|
||||
num_patches: int,
|
||||
patch_newline_mask: list[bool],
|
||||
) -> str:
|
||||
assert len(patch_newline_mask) == num_patches
|
||||
|
||||
parts = []
|
||||
for i in range(num_patches):
|
||||
parts.extend(
|
||||
[
|
||||
self.patch_start_token,
|
||||
self.patch_feature_tokens,
|
||||
self.patch_end_token,
|
||||
]
|
||||
)
|
||||
if patch_newline_mask[i]:
|
||||
parts.append(self.patch_newline_token)
|
||||
|
||||
return "".join(parts)
|
||||
|
||||
def _get_patch_repl_ids(
|
||||
self,
|
||||
num_patches: int,
|
||||
patch_newline_mask: list[bool],
|
||||
) -> list[int]:
|
||||
assert len(patch_newline_mask) == num_patches
|
||||
|
||||
parts = []
|
||||
for i in range(num_patches):
|
||||
parts.extend(
|
||||
[
|
||||
self.patch_start_token_id,
|
||||
*self.patch_feature_token_ids,
|
||||
self.patch_end_token_id,
|
||||
]
|
||||
)
|
||||
if patch_newline_mask[i]:
|
||||
parts.append(self.patch_newline_token_id)
|
||||
|
||||
return parts
|
||||
|
||||
def _get_image_repl_text(
|
||||
self,
|
||||
num_images: int,
|
||||
) -> str:
|
||||
parts = [
|
||||
self.image_start_token,
|
||||
self.image_feature_tokens,
|
||||
self.image_end_token,
|
||||
] * num_images
|
||||
|
||||
return "".join(parts)
|
||||
|
||||
def _get_image_repl_ids(
|
||||
self,
|
||||
num_images: int,
|
||||
) -> list[int]:
|
||||
part = [
|
||||
self.image_start_token_id,
|
||||
*self.image_feature_token_ids,
|
||||
self.image_end_token_id,
|
||||
]
|
||||
return part * num_images
|
||||
|
||||
def get_image_repl_feature_text(
|
||||
self,
|
||||
num_images: int,
|
||||
num_patches: int,
|
||||
patch_new_line_idx: list[bool],
|
||||
) -> str:
|
||||
patch_repl = self._get_patch_repl_text(num_patches, patch_new_line_idx)
|
||||
image_repl = self._get_image_repl_text(num_images)
|
||||
return patch_repl + image_repl
|
||||
|
||||
def get_image_repl_feature_ids(
|
||||
self,
|
||||
num_images: int,
|
||||
num_patches: int,
|
||||
patch_new_line_idx: list[bool],
|
||||
) -> list[int]:
|
||||
patch_repl = self._get_patch_repl_ids(num_patches, patch_new_line_idx)
|
||||
image_repl = self._get_image_repl_ids(num_images)
|
||||
return patch_repl + image_repl
|
||||
|
||||
def replace_placeholder(self, text: str, placeholder: str, repls: list[str]) -> str:
|
||||
parts = text.split(placeholder)
|
||||
|
||||
if len(parts) - 1 != len(repls):
|
||||
raise ValueError(
|
||||
"The number of placeholders does not match the number of replacements."
|
||||
)
|
||||
|
||||
result = [parts[0]]
|
||||
for i, repl in enumerate(repls):
|
||||
result.append(repl)
|
||||
result.append(parts[i + 1])
|
||||
|
||||
return "".join(result)
|
||||
|
||||
def __call__(
|
||||
self,
|
||||
text: str | list[str] | None = None,
|
||||
images: Image.Image | list[Image.Image] | None = None,
|
||||
return_tensors: str | TensorType | None = None,
|
||||
) -> BatchFeature:
|
||||
if images is not None:
|
||||
image_inputs = self.image_processor(
|
||||
images=images,
|
||||
return_tensors=return_tensors,
|
||||
)
|
||||
num_patches = image_inputs["num_patches"]
|
||||
patch_newline_mask = image_inputs["patch_newline_mask"]
|
||||
else:
|
||||
image_inputs = {}
|
||||
num_patches = []
|
||||
patch_newline_mask = []
|
||||
|
||||
if text is not None:
|
||||
if not isinstance(text, list):
|
||||
text = [text]
|
||||
|
||||
if image_inputs:
|
||||
image_token = self.image_token
|
||||
image_repl_str_lst = []
|
||||
start = 0
|
||||
for n_patches in num_patches:
|
||||
image_repl_str = self.get_image_repl_feature_text(
|
||||
1, n_patches, patch_newline_mask[start : start + n_patches]
|
||||
)
|
||||
image_repl_str_lst.append(image_repl_str)
|
||||
|
||||
start += n_patches
|
||||
|
||||
text = [
|
||||
self.replace_placeholder(t, image_token, image_repl_str_lst)
|
||||
for t in text
|
||||
]
|
||||
|
||||
text_inputs = self.tokenizer(text)
|
||||
else:
|
||||
text_inputs = {}
|
||||
|
||||
return BatchFeature(
|
||||
data={**text_inputs, **image_inputs},
|
||||
tensor_type=return_tensors,
|
||||
)
|
||||
@@ -1,112 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from math import ceil
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
from mistral_common.tokens.tokenizers.audio import AudioEncoder
|
||||
from transformers import (
|
||||
BatchFeature,
|
||||
ProcessorMixin,
|
||||
SequenceFeatureExtractor,
|
||||
TensorType,
|
||||
)
|
||||
from transformers.audio_utils import AudioInput
|
||||
|
||||
from vllm.tokenizers.mistral import MistralTokenizer
|
||||
|
||||
|
||||
class MistralCommonFeatureExtractor(SequenceFeatureExtractor):
|
||||
"""Provide a HF-compatible interface for
|
||||
`mistral_common.tokens.tokenizers.multimodal.AudioEncoder`.
|
||||
"""
|
||||
|
||||
def __init__(self, audio_encoder: AudioEncoder) -> None:
|
||||
self.audio_encoder = audio_encoder
|
||||
|
||||
@property
|
||||
def sampling_rate(self):
|
||||
return self.audio_encoder.audio_config.sampling_rate
|
||||
|
||||
@property
|
||||
def frame_rate(self):
|
||||
return self.audio_encoder.audio_config.frame_rate
|
||||
|
||||
def __call__(
|
||||
self,
|
||||
audios: AudioInput,
|
||||
return_tensors: str | TensorType | None = None,
|
||||
**kwargs,
|
||||
) -> BatchFeature:
|
||||
audios_lst = [audios] if not isinstance(audios, list) else audios
|
||||
|
||||
audios_processed = list[torch.Tensor]()
|
||||
|
||||
for audio in audios_lst:
|
||||
audio = np.asarray(audio, dtype=np.float32).ravel()
|
||||
if not self.audio_encoder.audio_config.is_streaming:
|
||||
audio = self.audio_encoder.pad(audio, self.sampling_rate)
|
||||
|
||||
audios_processed.append(torch.from_numpy(audio))
|
||||
|
||||
return BatchFeature(
|
||||
{"audio_arrays": audios_processed}, tensor_type=return_tensors
|
||||
)
|
||||
|
||||
def get_num_audio_tokens(self, audio_length: int) -> int:
|
||||
return ceil(audio_length / (self.sampling_rate // self.frame_rate))
|
||||
|
||||
def fetch_audio(self, audio_url_or_urls, sampling_rate=None):
|
||||
"""HF-compatible duck-typed ``fetch_audio``.
|
||||
|
||||
Mirrors :meth:`transformers.SequenceFeatureExtractor.fetch_audio` so
|
||||
:class:`transformers.ProcessorMixin.prepare_inputs_layout` works on this
|
||||
duck-typed feature extractor.
|
||||
|
||||
Accepts the same shapes as ``SequenceFeatureExtractor.fetch_audio``:
|
||||
|
||||
* ``np.ndarray`` / ``torch.Tensor`` — returned as-is.
|
||||
* ``list[float]`` — returned as-is (a single audio sample).
|
||||
* ``str`` URL or path — delegated to
|
||||
:func:`transformers.audio_utils.load_audio`.
|
||||
* ``list`` of any of the above — recursed element-wise.
|
||||
|
||||
``ProcessorMixin.prepare_inputs_layout`` always passes already-decoded
|
||||
audio (numpy array or torch tensor), so the str / list-of-str branches
|
||||
exist only to keep the contract identical to the upstream method.
|
||||
"""
|
||||
from transformers.audio_utils import is_valid_audio
|
||||
|
||||
sampling_rate = sampling_rate if sampling_rate else self.sampling_rate
|
||||
if is_valid_audio(audio_url_or_urls):
|
||||
return audio_url_or_urls
|
||||
if isinstance(audio_url_or_urls, (list, tuple)):
|
||||
return [
|
||||
self.fetch_audio(x, sampling_rate=sampling_rate)
|
||||
for x in audio_url_or_urls
|
||||
]
|
||||
if isinstance(audio_url_or_urls, str):
|
||||
from transformers.audio_utils import load_audio
|
||||
|
||||
return load_audio(audio_url_or_urls, sampling_rate=sampling_rate)
|
||||
raise TypeError(
|
||||
"only a numpy array, torch tensor, str URL/path, or list of those "
|
||||
f"is supported but got type={type(audio_url_or_urls)}"
|
||||
)
|
||||
|
||||
|
||||
class MistralCommonVoxtralProcessor(ProcessorMixin):
|
||||
attributes = ["feature_extractor", "tokenizer"]
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
tokenizer: MistralTokenizer,
|
||||
feature_extractor: MistralCommonFeatureExtractor,
|
||||
) -> None:
|
||||
self.tokenizer = tokenizer.transformers_tokenizer
|
||||
self.feature_extractor = feature_extractor
|
||||
|
||||
audio_special_ids = self.feature_extractor.audio_encoder.special_ids
|
||||
self.audio_token_id = audio_special_ids.audio
|
||||
self.begin_audio_token_id = audio_special_ids.begin_audio
|
||||
Reference in New Issue
Block a user