diff --git a/docs/models/supported_models.md b/docs/models/supported_models.md index f9c70adec7..85efca1e9b 100644 --- a/docs/models/supported_models.md +++ b/docs/models/supported_models.md @@ -409,8 +409,6 @@ th { | `GraniteSWAForCausalLM` | Granite SWA | `ibm-granite/granite-swash-2b` | ✅︎ | ✅︎ | | `GritLM` | GritLM | `parasail-ai/GritLM-7B-vllm`. | ✅︎ | ✅︎ | | `HrmTextForCausalLM` | HRM-Text | `sapientinc/HRM-Text-1B`, etc. | | | -| `HunYuanDenseV1ForCausalLM` | Hunyuan Dense | `tencent/Hunyuan-7B-Instruct` | ✅︎ | ✅︎ | -| `HunYuanMoEV1ForCausalLM` | Hunyuan-A13B | `tencent/Hunyuan-A13B-Instruct`, `tencent/Hunyuan-A13B-Pretrain`, `tencent/Hunyuan-A13B-Instruct-FP8`, etc. | ✅︎ | ✅︎ | | `HYV3ForCausalLM` | HY3 | `tencent/Hy3-preview-Base`, `tencent/Hy3-preview` | ✅︎ | ✅︎ | | `HyperCLOVAXForCausalLM` | HyperCLOVAX-SEED-Think-14B | `naver-hyperclovax/HyperCLOVAX-SEED-Think-14B` | ✅︎ | ✅︎ | | `InternLM2ForCausalLM` | InternLM2 | `internlm/internlm2-7b`, `internlm/internlm2-chat-7b`, etc. | ✅︎ | ✅︎ | @@ -478,6 +476,8 @@ Some models are supported only via the [Transformers modeling backend](#transfor | Architecture | Models | Example HF Models | [LoRA](../features/lora.md) | [PP](../serving/parallelism_scaling.md) | | ------------ | ------ | ----------------- | -------------------- | ------------------------- | | `GPTBigCodeForCausalLM` | StarCoder, SantaCoder, WizardCoder | `bigcode/starcoder`, `bigcode/gpt_bigcode-santacoder`, `WizardLM/WizardCoder-15B-V1.0`, etc. | ✅︎ | | +| `HunYuanDenseV1ForCausalLM` | Hunyuan Dense | `tencent/Hunyuan-7B-Instruct` | ✅︎ | ✅︎ | +| `HunYuanMoEV1ForCausalLM` | Hunyuan-A13B | `tencent/Hunyuan-A13B-Instruct`, `tencent/Hunyuan-A13B-Pretrain`, `tencent/Hunyuan-A13B-Instruct-FP8`, etc. | ✅︎ | ✅︎ | | `OlmoForCausalLM` | OLMo | `allenai/OLMo-1B-hf`, `allenai/OLMo-7B-hf`, etc. | ✅︎ | ✅︎ | | `Olmo2ForCausalLM` | OLMo2 | `allenai/OLMo-2-0425-1B`, etc. | ✅︎ | ✅︎ | | `SmolLM3ForCausalLM` | SmolLM3 | `HuggingFaceTB/SmolLM3-3B` | ✅︎ | ✅︎ | @@ -552,7 +552,6 @@ These models primarily accept the [`LLM.generate`](./generative_models.md#llmgen | `HCXVisionForCausalLM` | HyperCLOVAX-SEED-Vision-Instruct-3B | T + I+ + V+ | `naver-hyperclovax/HyperCLOVAX-SEED-Vision-Instruct-3B` | | | | `HCXVisionV2ForCausalLM` | HyperCLOVAX-SEED-Think-32B | T + I+ + V+ | `naver-hyperclovax/HyperCLOVAX-SEED-Think-32B` | | | | `H2OVLChatModel` | H2OVL | T + IE+ | `h2oai/h2ovl-mississippi-800m`, `h2oai/h2ovl-mississippi-2b`, etc. | ✅︎ | ✅︎ | -| `HunYuanVLForConditionalGeneration` | HunyuanOCR | T + IE+ | `tencent/HunyuanOCR`, etc. | ✅︎ | ✅︎ | | `Idefics3ForConditionalGeneration` | Idefics3 | T + I | `HuggingFaceM4/Idefics3-8B-Llama3`, etc. | ✅︎ | | | `IsaacForConditionalGeneration` | Isaac | T + I+ | `PerceptronAI/Isaac-0.1` | ✅︎ | ✅︎ | | `InternS1ForConditionalGeneration` | Intern-S1 | T + IE+ + VE+ | `internlm/Intern-S1`, `internlm/Intern-S1-mini`, etc. | ✅︎ | ✅︎ | @@ -629,6 +628,7 @@ Some models are supported only via the [Transformers modeling backend](#transfor | Architecture | Models | Inputs | Example HF Models | [LoRA](../features/lora.md) | [PP](../serving/parallelism_scaling.md) | | ------------ | ------ | ------ | ----------------- | --------------------------- | --------------------------------------- | | `Emu3ForConditionalGeneration` | Emu3 | T + I+ | `BAAI/Emu3-Chat-hf` | ✅︎ | ✅︎ | +| `HunYuanVLForConditionalGeneration` | HunyuanOCR | T + IE+ | `tencent/HunyuanOCR`, etc. | ✅︎ | ✅︎ | | `VibeVoiceAsrForConditionalGeneration` | VibeVoice-ASR | T + A+ | `microsoft/VibeVoice-ASR-HF` | ✅︎ | ✅︎ | ^ You need to set the architecture name via `--hf-overrides` to match the one in vLLM.
diff --git a/vllm/model_executor/models/hunyuan_v1.py b/vllm/model_executor/models/hunyuan_v1.py deleted file mode 100644 index cf1933bd7d..0000000000 --- a/vllm/model_executor/models/hunyuan_v1.py +++ /dev/null @@ -1,1038 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project - -# coding=utf-8 -# Copyright 2024 The HunYuan team. -# Copyright 2023 The vLLM team. -# Copyright 2022 EleutherAI and the HuggingFace Inc. team. All rights reserved. -# -# This code is based on EleutherAI's GPT-NeoX library and the GPT-NeoX -# and OPT implementations in this library. It has been modified from its -# original forms to accommodate minor architectural differences compared -# to GPT-NeoX and OPT used by the Meta AI team that trained the model. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Inference-only HunYuan model compatible with HuggingFace weights.""" - -import typing -from collections.abc import Callable, Iterable -from itertools import islice - -import regex as re -import torch -from torch import nn -from transformers import PretrainedConfig - -from vllm.compilation.decorators import support_torch_compile -from vllm.config import CacheConfig, VllmConfig, get_current_vllm_config -from vllm.distributed import ( - get_ep_group, - get_pp_group, - get_tensor_model_parallel_world_size, -) -from vllm.model_executor.layers.activation import SiluAndMul -from vllm.model_executor.layers.attention import Attention -from vllm.model_executor.layers.fused_moe import ( - FusedMoEFactory, - fused_moe_make_expert_params_mapping, -) -from vllm.model_executor.layers.layernorm import RMSNorm -from vllm.model_executor.layers.linear import ( - ColumnParallelLinear, - MergedColumnParallelLinear, - QKVParallelLinear, - ReplicatedLinear, - RowParallelLinear, -) -from vllm.model_executor.layers.logits_processor import LogitsProcessor -from vllm.model_executor.layers.quantization import QuantizationConfig -from vllm.model_executor.layers.rotary_embedding import get_rope -from vllm.model_executor.layers.vocab_parallel_embedding import ( - ParallelLMHead, - VocabParallelEmbedding, -) -from vllm.model_executor.model_loader.weight_utils import ( - default_weight_loader, - maybe_remap_kv_scale_name, -) -from vllm.sequence import IntermediateTensors -from vllm.v1.attention.backend import AttentionType - -from .interfaces import ( - EagleModelMixin, - MixtureOfExperts, - SupportsEagle, - SupportsEagle3, - SupportsLoRA, - SupportsPP, -) -from .utils import ( - AutoWeightsLoader, - PPMissingLayer, - is_pp_missing_parameter, - make_layers, - maybe_prefix, -) - - -def _is_moe(config: PretrainedConfig) -> bool: - num_experts = getattr(config, "num_experts", None) - if isinstance(num_experts, int): - return num_experts > 1 - if isinstance(num_experts, list) and num_experts: - # Ensure all elements are integers before calling max. - if all(isinstance(e, int) for e in num_experts): - return max(num_experts) > 1 - else: - return False - return False - - -def _get_cla_factor(config: PretrainedConfig) -> int: - if not getattr(config, "use_cla", False): - return 1 - return getattr(config, "cla_share_factor", 1) - - -class HunYuanMLP(nn.Module): - def __init__( - self, - hidden_size: int, - intermediate_size: int, - hidden_act: str, - quant_config: QuantizationConfig | None = None, - bias: bool = False, - prefix: str = "", - reduce_results: bool = True, - ) -> None: - super().__init__() - self.gate_up_proj = MergedColumnParallelLinear( - input_size=hidden_size, - output_sizes=[intermediate_size] * 2, - bias=bias, - quant_config=quant_config, - prefix=f"{prefix}.gate_up_proj", - ) - self.down_proj = RowParallelLinear( - input_size=intermediate_size, - output_size=hidden_size, - bias=bias, - quant_config=quant_config, - prefix=f"{prefix}.down_proj", - reduce_results=reduce_results, - ) - if hidden_act != "silu": - raise ValueError( - f"Unsupported activation: {hidden_act}. Only silu is supported for now." - ) - self.act_fn = SiluAndMul() - - def forward(self, x): - gate_up, _ = self.gate_up_proj(x) - x = self.act_fn(gate_up) - x, _ = self.down_proj(x) - return x - - -class HunYuanAttention(nn.Module): - def __init__( - self, - config: PretrainedConfig, - hidden_size: int, - num_heads: int, - num_kv_heads: int, - max_position_embeddings: int = 8192, - quant_config: QuantizationConfig | None = None, - bias: bool = False, - cache_config: CacheConfig | None = None, - prefix: str = "", - layer_id: int = -1, - ) -> None: - super().__init__() - self.hidden_size = hidden_size - tp_size = get_tensor_model_parallel_world_size() - self.total_num_heads = num_heads - assert self.total_num_heads % tp_size == 0 - self.num_heads = self.total_num_heads // tp_size - self.total_num_kv_heads = num_kv_heads - if self.total_num_kv_heads >= tp_size: - # Number of KV heads is greater than TP size, so we partition - # the KV heads across multiple tensor parallel GPUs. - assert self.total_num_kv_heads % tp_size == 0 - else: - # Number of KV heads is less than TP size, so we replicate - # the KV heads across multiple tensor parallel GPUs. - assert tp_size % self.total_num_kv_heads == 0 - self.num_kv_heads = max(1, self.total_num_kv_heads // tp_size) - - if hasattr(config, "head_dim") and config.head_dim: - self.head_dim = config.head_dim - elif hasattr(config, "attention_head_dim"): - self.head_dim = config.attention_head_dim - else: - self.head_dim = self.hidden_size // self.total_num_heads - self.q_size = self.num_heads * self.head_dim - self.kv_size = self.num_kv_heads * self.head_dim - self.scaling = self.head_dim**-0.5 - self.max_position_embeddings = max_position_embeddings - self.use_qk_norm = getattr(config, "use_qk_norm", False) - self.layer_id = layer_id - - self.qkv_proj = QKVParallelLinear( - hidden_size=hidden_size, - head_size=self.head_dim, - total_num_heads=self.total_num_heads, - total_num_kv_heads=self.total_num_kv_heads, - bias=bias, - quant_config=quant_config, - prefix=f"{prefix}.qkv_proj", - ) - - self.o_proj = RowParallelLinear( - input_size=self.total_num_heads * self.head_dim, - output_size=hidden_size, - bias=bias, - quant_config=quant_config, - prefix=f"{prefix}.o_proj", - ) - - self.rotary_emb = get_rope( - self.head_dim, - max_position=max_position_embeddings, - rope_parameters=config.rope_parameters, - is_neox_style=True, - ) - self.attn = Attention( - self.num_heads, - self.head_dim, - self.scaling, - num_kv_heads=self.num_kv_heads, - cache_config=cache_config, - quant_config=quant_config, - prefix=f"{prefix}.attn", - ) - - if self.use_qk_norm: - self.query_layernorm = RMSNorm(self.head_dim, eps=config.rms_norm_eps) - self.key_layernorm = RMSNorm(self.head_dim, eps=config.rms_norm_eps) - - def forward( - self, - positions: torch.Tensor, - hidden_states: torch.Tensor, - kv_states: tuple[torch.Tensor] | None = None, - ) -> torch.Tensor: - qkv, _ = self.qkv_proj(hidden_states) - q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1) - q, k = self.rotary_emb(positions, q, k) - ori_k = k - if self.use_qk_norm: - q = self.query_layernorm( - q.view(-1, self.num_heads, self.head_dim), - ) - k = self.key_layernorm( - k.view(-1, self.num_kv_heads, self.head_dim), - ) - - attn_output = self.attn(q, k, v) - # For o_proj - attn_output = attn_output.view(q.shape[0], -1) - output, _ = self.o_proj(attn_output) - return output, (ori_k, v) - - -class HunYuanCrossAttention(nn.Module): - def __init__( - self, - config: PretrainedConfig, - hidden_size: int, - num_heads: int, - num_kv_heads: int, - max_position_embeddings: int = 8192, - quant_config: QuantizationConfig | None = None, - bias: bool = False, - cache_config: CacheConfig | None = None, - prefix: str = "", - layer_id: int = -1, - ) -> None: - super().__init__() - self.hidden_size = hidden_size - tp_size = get_tensor_model_parallel_world_size() - self.total_num_heads = num_heads - assert self.total_num_heads % tp_size == 0 - self.num_heads = self.total_num_heads // tp_size - self.total_num_kv_heads = num_kv_heads - if self.total_num_kv_heads >= tp_size: - # Number of KV heads is greater than TP size, so we partition - # the KV heads across multiple tensor parallel GPUs. - assert self.total_num_kv_heads % tp_size == 0 - else: - # Number of KV heads is less than TP size, so we replicate - # the KV heads across multiple tensor parallel GPUs. - assert tp_size % self.total_num_kv_heads == 0 - self.num_kv_heads = max(1, self.total_num_kv_heads // tp_size) - # MistralConfig has an optional head_dim introduced by Mistral-Nemo - if hasattr(config, "head_dim"): - self.head_dim = config.head_dim - elif hasattr(config, "attention_head_dim"): - self.head_dim = config.attention_head_dim - else: - self.head_dim = self.hidden_size // self.total_num_heads - self.q_size = self.num_heads * self.head_dim - self.kv_size = self.num_kv_heads * self.head_dim - self.scaling = self.head_dim**-0.5 - self.max_position_embeddings = max_position_embeddings - self.use_qk_norm = getattr(config, "use_qk_norm", False) - self.layer_id = layer_id - - self.q_proj = ColumnParallelLinear( - hidden_size, - hidden_size, - bias=bias, - quant_config=quant_config, - prefix=f"{prefix}.q_proj", - ) - - self.o_proj = RowParallelLinear( - input_size=self.total_num_heads * self.head_dim, - output_size=hidden_size, - bias=bias, - quant_config=quant_config, - prefix=f"{prefix}.o_proj", - ) - - self.rotary_emb = get_rope( - self.head_dim, - max_position=max_position_embeddings, - rope_parameters=config.rope_parameters, - is_neox_style=True, - ) - self.attn = Attention( - self.num_heads, - self.head_dim, - self.scaling, - num_kv_heads=self.num_kv_heads, - cache_config=cache_config, - quant_config=quant_config, - prefix=f"{prefix}.attn", - attn_type=AttentionType.ENCODER_DECODER, - ) - - if self.use_qk_norm: - self.query_layernorm = RMSNorm(self.head_dim, eps=config.rms_norm_eps) - self.key_layernorm = RMSNorm(self.head_dim, eps=config.rms_norm_eps) - - def forward( - self, - positions: torch.Tensor, - hidden_states: torch.Tensor, - kv_states: tuple[torch.Tensor] | None = None, - ) -> torch.Tensor: - assert kv_states is not None - ori_k, v = kv_states # use last layer kv, - k = ori_k - q, _ = self.q_proj(hidden_states) - k_tmp = torch.empty_like(k) # Todo: reduant rotary embedding - q, _ = self.rotary_emb(positions, q, k_tmp) - if self.use_qk_norm: - q = self.query_layernorm( - q.view(-1, self.num_heads, self.head_dim), - ) - k = self.key_layernorm( - k.view(-1, self.num_kv_heads, self.head_dim), - ) - - attn_output = self.attn(q, k, v) - # For o_proj - attn_output = attn_output.view(q.shape[0], -1) - output, _ = self.o_proj(attn_output) - return output, (ori_k, v) - - -class HunYuanSparseMoeBlock(nn.Module): - def __init__( - self, - config: PretrainedConfig, - quant_config: QuantizationConfig | None = None, - layer_id: int = -1, - prefix: str = "", - enable_eplb: bool = False, - ): - super().__init__() - self.tp_size = get_tensor_model_parallel_world_size() - - self.ep_group = get_ep_group().device_group - self.ep_size = self.ep_group.size() - self.n_routed_experts = config.num_experts - - if self.tp_size > config.num_experts: - raise ValueError( - f"Tensor parallel size {self.tp_size} is greater than " - f"the number of experts {config.num_experts}." - ) - - # Get layer_id topk if config.moe_topk is a list - if isinstance(config.moe_topk, list): - assert layer_id >= 0 - assert len(config.moe_topk) > layer_id - top_k = config.moe_topk[layer_id] - else: - top_k = config.moe_topk - - # If it is moe, moe_intermediate_size is preferred - intermediate_size = config.intermediate_size - if config.moe_intermediate_size is not None: - intermediate_size = ( - config.moe_intermediate_size - if isinstance(config.moe_intermediate_size, int) - else config.moe_intermediate_size[layer_id] - ) - - # Load balancing settings. - vllm_config = get_current_vllm_config() - eplb_config = vllm_config.parallel_config.eplb_config - self.enable_eplb = enable_eplb - - self.n_logical_experts = self.n_routed_experts - self.n_redundant_experts = eplb_config.num_redundant_experts - self.n_physical_experts = self.n_logical_experts + self.n_redundant_experts - self.n_local_physical_experts = self.n_physical_experts // self.ep_size - self.gate = ReplicatedLinear( - config.hidden_size, - config.num_experts, - bias=False, - quant_config=None, - prefix=f"{prefix}.gate", - ) - if config.use_mixed_mlp_moe > 0: - # Get layer_id num_shared_expert if config.num_shared_expert is - # a list. - if isinstance(config.num_shared_expert, list): - assert layer_id >= 0 - assert len(config.num_shared_expert) > layer_id - num_shared_expert = config.num_shared_expert[layer_id] - else: - num_shared_expert = config.num_shared_expert - - self.shared_mlp = HunYuanMLP( - hidden_size=config.hidden_size, - intermediate_size=config.intermediate_size * num_shared_expert, - hidden_act=config.hidden_act, - quant_config=quant_config, - reduce_results=False, - prefix=f"{prefix}.shared_mlp", - ) - else: - self.shared_mlp = None - - self.experts = FusedMoEFactory( - shared_experts=self.shared_mlp, - num_experts=self.n_routed_experts, - top_k=top_k, - hidden_size=config.hidden_size, - intermediate_size=intermediate_size, - renormalize=top_k > 1, - quant_config=quant_config, - prefix=f"{prefix}.experts", - enable_eplb=self.enable_eplb, - num_redundant_experts=self.n_redundant_experts, - ) - - def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: - # NOTE: hidden_states can have either 1D or 2D shape. - orig_shape = hidden_states.shape - hidden_dim = hidden_states.shape[-1] - hidden_states = hidden_states.view(-1, hidden_dim) - - # router_logits: (num_tokens, n_experts) - router_logits, _ = self.gate(hidden_states) - final_hidden_states = self.experts( - hidden_states=hidden_states, router_logits=router_logits - ) - - return final_hidden_states.view(orig_shape) - - -class HunYuanDecoderLayer(nn.Module): - def __init__( - self, - config: PretrainedConfig, - cache_config: CacheConfig | None = None, - quant_config: QuantizationConfig | None = None, - prefix: str = "", - layer_id: int = -1, - enable_eplb: bool = False, - ) -> None: - super().__init__() - assert layer_id >= 0 - self.layer_id = layer_id - self.hidden_size = config.hidden_size - self.intermediate_size = ( - config.intermediate_size - if isinstance(config.intermediate_size, int) - else config.intermediate_size[layer_id] - ) - max_position_embeddings = getattr(config, "max_position_embeddings", 8192) - attention_bias = getattr(config, "attention_bias", False) or getattr( - config, "bias", False - ) - cla_factor = _get_cla_factor(config) - attention_type = ( - AttentionType.ENCODER_DECODER - if layer_id >= 0 and layer_id % cla_factor != 0 - else AttentionType.DECODER - ) - if attention_type == AttentionType.DECODER: - self.self_attn = HunYuanAttention( - config=config, - hidden_size=self.hidden_size, - num_heads=config.num_attention_heads, - num_kv_heads=getattr( - config, "num_key_value_heads", config.num_attention_heads - ), - max_position_embeddings=max_position_embeddings, - quant_config=quant_config, - bias=attention_bias, - cache_config=cache_config, - prefix=f"{prefix}.self_attn", - layer_id=layer_id, - ) - elif attention_type == AttentionType.ENCODER_DECODER: - self.self_attn = HunYuanCrossAttention( - config=config, - hidden_size=self.hidden_size, - num_heads=config.num_attention_heads, - num_kv_heads=getattr( - config, "num_key_value_heads", config.num_attention_heads - ), - max_position_embeddings=max_position_embeddings, - quant_config=quant_config, - bias=attention_bias, - cache_config=cache_config, - prefix=f"{prefix}.self_attn", - layer_id=layer_id, - ) - else: - raise RuntimeError(f"Unsupported attention type: {attention_type}") - - if _is_moe(config): - self.mlp = HunYuanSparseMoeBlock( - config=config, - quant_config=quant_config, - layer_id=layer_id, - prefix=f"{prefix}.mlp", - enable_eplb=enable_eplb, - ) - else: - self.mlp = HunYuanMLP( - hidden_size=self.hidden_size, - intermediate_size=self.intermediate_size, - hidden_act=config.hidden_act, - quant_config=quant_config, - bias=getattr(config, "mlp_bias", False), - prefix=f"{prefix}.mlp", - ) - - self.input_layernorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps) - self.post_attention_layernorm = RMSNorm( - config.hidden_size, eps=config.rms_norm_eps - ) - - def forward( - self, - positions: torch.Tensor, - hidden_states: torch.Tensor, - residual: torch.Tensor | None, - kv_states: tuple[torch.Tensor] | None = None, - ) -> tuple[torch.Tensor, torch.Tensor]: - # Self Attention - if residual is None: - residual = hidden_states - hidden_states = self.input_layernorm(hidden_states) - else: - hidden_states, residual = self.input_layernorm(hidden_states, residual) - hidden_states, ori_kv_states = self.self_attn( - positions=positions, - hidden_states=hidden_states, - kv_states=kv_states, - ) - - # Fully Connected - hidden_states, residual = self.post_attention_layernorm(hidden_states, residual) - hidden_states = self.mlp(hidden_states) - return hidden_states, residual, ori_kv_states - - -@support_torch_compile( - dynamic_arg_dims={ - "input_ids": 0, - # positions is of shape (xd, seq_len) if xdrope is enabled for hunyuan-vl, - # otherwise (seq_len, ). - "positions": -1, - "intermediate_tensors": 0, - "inputs_embeds": 0, - } -) -class HunYuanModel(nn.Module, EagleModelMixin): - def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""): - super().__init__() - - config = vllm_config.model_config.hf_config - cache_config = vllm_config.cache_config - quant_config = vllm_config.quant_config - - eplb_config = vllm_config.parallel_config.eplb_config - enable_eplb = vllm_config.parallel_config.enable_eplb - self.num_redundant_experts = eplb_config.num_redundant_experts - - self.config = config - self.quant_config = quant_config - - self.vocab_size = config.vocab_size - - if get_pp_group().is_first_rank or ( - config.tie_word_embeddings and get_pp_group().is_last_rank - ): - self.embed_tokens = VocabParallelEmbedding( - self.vocab_size, - config.hidden_size, - quant_config=quant_config, - ) - else: - self.embed_tokens = PPMissingLayer() - self.start_layer, self.end_layer, self.layers = make_layers( - config.num_hidden_layers, - lambda prefix: HunYuanDecoderLayer( - config=config, - layer_id=int(prefix.split(".")[-1]), - cache_config=cache_config, - quant_config=quant_config, - prefix=prefix, - enable_eplb=enable_eplb, - ), - prefix=f"{prefix}.layers", - ) - if get_pp_group().is_last_rank: - self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps) - else: - self.norm = PPMissingLayer() - - def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor: - return self.embed_tokens(input_ids) - - def forward( - self, - input_ids: torch.Tensor | None, - positions: torch.Tensor, - intermediate_tensors: IntermediateTensors | None, - inputs_embeds: torch.Tensor | None = None, - ) -> torch.Tensor | IntermediateTensors: - if get_pp_group().is_first_rank: - if inputs_embeds is not None: - hidden_states = inputs_embeds - else: - hidden_states = self.embed_input_ids(input_ids) - residual = None - else: - assert intermediate_tensors is not None - hidden_states = intermediate_tensors["hidden_states"] - residual = intermediate_tensors["residual"] - - cla_factor = _get_cla_factor(self.config) - prev_kv_states = None - aux_hidden_states = self._maybe_add_hidden_state([], 0, hidden_states, residual) - for i, layer in enumerate( - islice(self.layers, self.start_layer, self.end_layer) - ): - hidden_states, residual, kv_states = layer( - positions, - hidden_states, - residual, - prev_kv_states, - ) - - if getattr(self.config, "use_cla", False) and i % cla_factor == 0: - prev_kv_states = kv_states - else: - prev_kv_states = None - - self._maybe_add_hidden_state( - aux_hidden_states, i + 1, hidden_states, residual - ) - - if not get_pp_group().is_last_rank: - return IntermediateTensors( - {"hidden_states": hidden_states, "residual": residual} - ) - - hidden_states, _ = self.norm(hidden_states, residual) - - if len(aux_hidden_states) > 0: - return hidden_states, aux_hidden_states - return hidden_states - - def _split_qkv_weight(self, qkv: torch.Tensor): - num_attention_heads = self.config.num_attention_heads - num_kv_heads = getattr( - self.config, "num_key_value_heads", self.config.num_attention_heads - ) - num_key_value_groups = num_attention_heads // num_kv_heads - hidden_size = self.config.hidden_size - - if hasattr(self.config, "head_dim"): - attention_head_dim = self.config.head_dim - elif hasattr(self.config, "attention_head_dim"): - attention_head_dim = self.config.attention_head_dim - else: - attention_head_dim = self.config.hidden_size // num_attention_heads - - qkv = qkv.reshape( - num_kv_heads, num_key_value_groups + 2, attention_head_dim, hidden_size - ) - q, k, v = torch.split(qkv, (num_key_value_groups, 1, 1), dim=1) - q = q.reshape(-1, hidden_size) - k = k.reshape(-1, hidden_size) - v = v.reshape(-1, hidden_size) - return torch.concat((q, k, v)) - - def get_expert_mapping(self) -> list[tuple[str, str, int, str]]: - if _is_moe(self.config): - # Params for weights, fp8 weight scales, fp8 activation scales - # (param_name, weight_name, expert_id, shard_id) - return fused_moe_make_expert_params_mapping( - self, - ckpt_gate_proj_name="gate_proj", - ckpt_down_proj_name="down_proj", - ckpt_up_proj_name="up_proj", - num_experts=self.config.num_experts, - num_redundant_experts=self.num_redundant_experts, - ) - else: - return [] - - def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]): - cla_factor = _get_cla_factor(self.config) - stacked_params_mapping = [ - # (param_name, shard_name, shard_id) - (".qkv_proj", ".q_proj", "q"), - (".qkv_proj", ".k_proj", "k"), - (".qkv_proj", ".v_proj", "v"), - (".gate_up_proj", ".gate_proj", 0), - (".gate_up_proj", ".up_proj", 1), - ] - - num_attention_heads = self.config.num_attention_heads - num_kv_heads = getattr( - self.config, "num_key_value_heads", self.config.num_attention_heads - ) - split_params_mapping = [ - (".gate_up_proj", ".gate_and_up_proj", 2, [(1, 1), (0, 1)], None), - ( - ".qkv_proj", - ".qkv_proj", - num_attention_heads + num_kv_heads * 2, - [("q", num_attention_heads), ("k", num_kv_heads), ("v", num_kv_heads)], - self._split_qkv_weight, - ), - ] - - params_dict = dict(self.named_parameters()) - loaded_params: set[str] = set() - expert_params_mapping = self.get_expert_mapping() - for name, loaded_weight in weights: - if "rotary_emb.inv_freq" in name: - continue - if "gate_proj_bias" in name: - name = name.replace("gate_proj_bias", "gate_proj.bias") - if "up_proj_bias" in name: - name = name.replace("up_proj_bias", "up_proj.bias") - if "rotary_emb.cos_cached" in name or "rotary_emb.sin_cached" in name: - # Models trained using ColossalAI may include these tensors in - # the checkpoint. Skip them. - continue - # With tie_word_embeddings, we can skip lm_head.weight - # The weight might appear unnecessarily in the files if the model is - # processed with quantization, LoRA, fine-tuning, etc. - if self.config.tie_word_embeddings and "lm_head.weight" in name: - continue - - is_found = False - for param_name, weight_name, shard_id in stacked_params_mapping: - if weight_name not in name: - continue - if "mlp.experts" in name: - continue - # cross layer only have q_proj, skip qkv pack - if weight_name == ".q_proj": - match = re.search(r"layers\.\d+", name) - if match: - layer_id = int(match.group(0).split(".")[-1]) - if cla_factor > 1 and layer_id % cla_factor != 0: - continue - name = name.replace(weight_name, param_name) - # Skip loading extra bias for GPTQ models. - if name.endswith(".bias") and name not in params_dict: - continue - - if is_pp_missing_parameter(name, self): - continue - - param = params_dict[name] - weight_loader = param.weight_loader - weight_loader(param, loaded_weight, shard_id) - loaded_params.add(name) - is_found = True - break - if is_found: - continue - - for ( - param_name, - weight_name, - den, - split_param, - func, - ) in split_params_mapping: - if weight_name not in name: - continue - name = name.replace(weight_name, param_name) - # Skip loading extra bias for GPTQ models. - if name.endswith(".bias") and name not in params_dict: - continue - - if is_pp_missing_parameter(name, self): - continue - - assert loaded_weight.shape[0] % den == 0 - units = loaded_weight.shape[0] // den - - param = params_dict[name] - weight_loader = param.weight_loader - offset = 0 - for shard_id, num in split_param: - new_offset = offset + num * units - if func: - weight_loader( - param, func(loaded_weight)[offset:new_offset], shard_id - ) - else: - weight_loader(param, loaded_weight[offset:new_offset], shard_id) - offset = new_offset - - break - else: - # Skip loading extra bias for GPTQ models. - if name.endswith(".bias") and name not in params_dict: - continue - is_expert_weight = False - for mapping in expert_params_mapping: - param_name, weight_name, expert_id, shard_id = mapping - if weight_name not in name: - continue - # this is an expert weight and should not be - # attempted to load as other weights later - is_expert_weight = True - - # Do not modify `name` since the loop may continue here - # Instead, create a new variable - name_mapped = name.replace(weight_name, param_name) - if is_pp_missing_parameter(name_mapped, self): - continue - param = params_dict[name_mapped] - # We should ask the weight loader to return success or not - # here since otherwise we may skip experts with other - # available replicas. - weight_loader = typing.cast( - Callable[..., bool], param.weight_loader - ) - success = weight_loader( - param, - loaded_weight, - name_mapped, - shard_id=shard_id, - expert_id=expert_id, - return_success=True, - ) - if success: - name = name_mapped - break - else: - if is_expert_weight: - # We've checked that this is an expert weight - # However it's not mapped locally to this rank - # So we simply skip it - continue - # Remapping the name of FP8 kv-scale. - name = maybe_remap_kv_scale_name(name, params_dict) - if name is None: - continue - - if is_pp_missing_parameter(name, self): - continue - - if "mlp.gate.wg." in name: - name = name.replace("wg.", "") - - param = params_dict[name] - weight_loader = getattr( - param, "weight_loader", default_weight_loader - ) - weight_loader(param, loaded_weight) - loaded_params.add(name) - return loaded_params - - -class HunyuanV1ModelBase( - nn.Module, SupportsLoRA, SupportsPP, SupportsEagle, SupportsEagle3 -): - packed_modules_mapping = { - "qkv_proj": [ - "q_proj", - "k_proj", - "v_proj", - ], - "gate_up_proj": [ - "gate_proj", - "up_proj", - ], - } - - def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""): - super().__init__() - - config = vllm_config.model_config.hf_config - quant_config = vllm_config.quant_config - self.config = config - self.quant_config = quant_config - - self.model = HunYuanModel( - vllm_config=vllm_config, - prefix=maybe_prefix(prefix, "model"), - ) - if get_pp_group().is_last_rank: - self.lm_head = ParallelLMHead( - config.vocab_size, - config.hidden_size, - quant_config=quant_config, - prefix=maybe_prefix(prefix, "lm_head"), - ) - if config.tie_word_embeddings: - self.lm_head = self.lm_head.tie_weights(self.model.embed_tokens) - - logit_scale = getattr(config, "logit_scale", 1.0) - self.logits_processor = LogitsProcessor( - config.vocab_size, scale=logit_scale - ) - else: - self.lm_head = PPMissingLayer() - - def forward( - self, - input_ids: torch.Tensor | None, - positions: torch.Tensor, - intermediate_tensors: IntermediateTensors | None = None, - inputs_embeds: torch.Tensor | None = None, - ) -> torch.Tensor | IntermediateTensors: - model_output = self.model( - input_ids, positions, intermediate_tensors, inputs_embeds - ) - return model_output - - def compute_logits( - self, - hidden_states: torch.Tensor, - ) -> torch.Tensor | None: - logits = self.logits_processor(self.lm_head, hidden_states) - return logits - - def make_empty_intermediate_tensors( - self, batch_size: int, dtype: torch.dtype, device: torch.device - ) -> IntermediateTensors: - return IntermediateTensors( - { - "hidden_states": torch.zeros( - (batch_size, self.config.hidden_size), dtype=dtype, device=device - ), - "residual": torch.zeros( - (batch_size, self.config.hidden_size), dtype=dtype, device=device - ), - } - ) - - def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: - loader = AutoWeightsLoader(self) - return loader.load_weights(weights) - - def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor: - return self.model.embed_input_ids(input_ids) - - -class HunYuanMoEV1Base(HunyuanV1ModelBase, MixtureOfExperts): - def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""): - super().__init__(vllm_config=vllm_config, prefix=prefix) - - # Set MoE hyperparameters - self.num_expert_groups = 1 - self.moe_layers = [] - example_layer = None - for layer in self.model.layers: - if isinstance(layer, PPMissingLayer): - continue - - assert isinstance(layer, HunYuanDecoderLayer) - if isinstance(layer.mlp, HunYuanSparseMoeBlock): - example_layer = layer.mlp - self.moe_layers.append(layer.mlp.experts) - - if example_layer is None: - raise RuntimeError("No HunYuanMoE layer found in model.layers.") - - self.num_moe_layers = len(self.moe_layers) - self.num_logical_experts = example_layer.n_logical_experts - self.num_physical_experts = example_layer.n_physical_experts - self.num_local_physical_experts = example_layer.n_local_physical_experts - self.num_routed_experts = example_layer.n_routed_experts - self.num_redundant_experts = example_layer.n_redundant_experts - - def update_physical_experts_metadata( - self, - num_physical_experts: int, - num_local_physical_experts: int, - ) -> None: - assert self.num_local_physical_experts == num_local_physical_experts - self.num_physical_experts = num_physical_experts - self.num_local_physical_experts = num_local_physical_experts - self.num_redundant_experts = num_physical_experts - self.num_logical_experts - for layer in self.model.layers: - if isinstance(layer.mlp, HunYuanSparseMoeBlock): - moe = layer.mlp - moe.n_local_physical_experts = num_local_physical_experts - moe.n_physical_experts = num_physical_experts - moe.n_redundant_experts = self.num_redundant_experts - moe.experts.update_expert_map() - - def get_expert_mapping(self) -> list[tuple[str, str, int, str]]: - return self.model.get_expert_mapping() - - -class HunYuanDenseV1Base(HunyuanV1ModelBase): - def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""): - super().__init__(vllm_config=vllm_config, prefix=prefix) - - -class HunYuanDenseV1ForCausalLM(HunYuanDenseV1Base): - pass - - -class HunYuanMoEV1ForCausalLM(HunYuanMoEV1Base): - pass diff --git a/vllm/model_executor/models/hunyuan_vision.py b/vllm/model_executor/models/hunyuan_vision.py deleted file mode 100644 index 4454ce47c1..0000000000 --- a/vllm/model_executor/models/hunyuan_vision.py +++ /dev/null @@ -1,1047 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project - -# coding=utf-8 -# Copyright 2025 The HunYuan team. -# Copyright 2025 The vLLM team. -# Copyright 2025 EleutherAI and the HuggingFace Inc. team. All rights reserved. -# -# This code is based on EleutherAI's GPT-NeoX library and the GPT-NeoX -# and OPT implementations in this library. It has been modified from its -# original forms to accommodate minor architectural differences compared -# to GPT-NeoX and OPT used by the Meta AI team that trained the model. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Inference-only HunYuan-VL model compatible with HuggingFace weights.""" - -from collections.abc import Callable, Iterable, Mapping, Sequence -from functools import partial -from typing import Annotated, Any, Literal, TypeAlias - -import torch -import torch.nn as nn -import torch.nn.functional as F -from transformers import BatchFeature, HunYuanVLProcessor -from transformers.models.hunyuan_vl.image_processing_hunyuan_vl import ( - HunYuanVLImageProcessor, - smart_resize, -) - -from vllm.config import VllmConfig -from vllm.config.multimodal import BaseDummyOptions -from vllm.distributed import parallel_state -from vllm.distributed import utils as dist_utils -from vllm.inputs import ModalityData, MultiModalDataDict -from vllm.logger import init_logger -from vllm.model_executor.layers.activation import get_act_fn -from vllm.model_executor.layers.attention import MMEncoderAttention -from vllm.model_executor.layers.layernorm import RMSNorm -from vllm.model_executor.layers.linear import ( - ColumnParallelLinear, - QKVParallelLinear, - RowParallelLinear, -) -from vllm.model_executor.layers.quantization import QuantizationConfig -from vllm.model_executor.models.module_mapping import MultiModelKeys -from vllm.multimodal import MULTIMODAL_REGISTRY -from vllm.multimodal.inputs import ( - ImageItem, - MultiModalFeatureSpec, - MultiModalFieldConfig, - MultiModalKwargsItems, -) -from vllm.multimodal.parse import ( - DictEmbeddingItems, - ImageSize, - ModalityDataItems, - MultiModalDataItems, - MultiModalDataParser, -) -from vllm.multimodal.processing import ( - BaseDummyInputsBuilder, - BaseMultiModalProcessor, - BaseProcessingInfo, - PromptReplacement, - PromptUpdate, - PromptUpdateDetails, -) -from vllm.sequence import IntermediateTensors -from vllm.transformers_utils.configs.hunyuan_vl import ( - HunYuanVLConfig, - HunYuanVLVisionConfig, -) -from vllm.utils.tensor_schema import TensorSchema, TensorShape - -from .interfaces import ( - MultiModalEmbeddings, - SupportsEagle, - SupportsEagle3, - SupportsLoRA, - SupportsMultiModal, - SupportsPP, - SupportsQuant, - SupportsXDRoPE, -) -from .utils import ( - AutoWeightsLoader, - WeightsMapper, - init_vllm_registered_model, - maybe_prefix, -) -from .vision import is_vit_use_data_parallel - -logger = init_logger(__name__) - -# === Vision Inputs === # - - -class HunYuanVLImagePixelInputs(TensorSchema): - """ - Dimensions: - - np: Number of patches - - ni: Number of images - - cps: Number of channels * patch_size * patch_size - """ - - type: Literal["pixel_values"] - - pixel_values: Annotated[ - torch.Tensor, - TensorShape("np", "cps"), - ] - - image_grid_thw: Annotated[ - torch.Tensor, - TensorShape("ni", 3), - ] - - -class HunYuanVLImageEmbeddingInputs(TensorSchema): - """ - Dimensions: - - nf: Number of image features - - hs: Hidden size - - ni: Number of images - """ - - type: Literal["image_embeds"] - - image_embeds: Annotated[ - torch.Tensor, - TensorShape("nf", "hs"), - ] - - image_grid_thw: Annotated[ - torch.Tensor, - TensorShape("ni", 3), - ] - - -HunYuanVLImageInputs: TypeAlias = ( - HunYuanVLImagePixelInputs | HunYuanVLImageEmbeddingInputs -) - -# === Vision Encoder === # - - -class HunYuanVisionMLP(nn.Module): - def __init__( - self, - in_features: int, - hidden_features: int, - bias: bool = True, - act_fn: Callable[[torch.Tensor], torch.Tensor] = F.gelu, - quant_config: QuantizationConfig | None = None, - prefix: str = "", - ): - super().__init__() - use_data_parallel = is_vit_use_data_parallel() - self.dense_h_to_4h = ColumnParallelLinear( - in_features, - hidden_features, - bias=bias, - quant_config=quant_config, - prefix=f"{prefix}.dense_h_to_4h", - disable_tp=use_data_parallel, - ) - self.dense_4h_to_h = RowParallelLinear( - hidden_features, - in_features, - bias=bias, - quant_config=quant_config, - prefix=f"{prefix}.dense_4h_to_h", - disable_tp=use_data_parallel, - ) - self.act_fn = act_fn - - def forward(self, x: torch.Tensor): - x_up, _ = self.dense_h_to_4h(x) - x_down, _ = self.dense_4h_to_h(self.act_fn(x_up)) - return x_down - - -class HunYuanVisionAttention(nn.Module): - def __init__( - self, - embed_dim: int, - num_heads: int, - projection_size: int, - quant_config: QuantizationConfig | None = None, - prefix: str = "", - ) -> None: - super().__init__() - # Per attention head and per partition values. - use_data_parallel = is_vit_use_data_parallel() - self.tp_size = ( - 1 - if use_data_parallel - else parallel_state.get_tensor_model_parallel_world_size() - ) - self.hidden_size_per_attention_head = dist_utils.divide( - projection_size, num_heads - ) - self.num_attention_heads_per_partition = dist_utils.divide( - num_heads, self.tp_size - ) - - self.qkv = QKVParallelLinear( - hidden_size=embed_dim, - head_size=self.hidden_size_per_attention_head, - total_num_heads=num_heads, - total_num_kv_heads=num_heads, - bias=True, - quant_config=quant_config, - prefix=f"{prefix}.qkv", - disable_tp=use_data_parallel, - ) - - self.o_proj = RowParallelLinear( - input_size=projection_size, - output_size=embed_dim, - quant_config=quant_config, - prefix=f"{prefix}.o_proj", - disable_tp=use_data_parallel, - ) - - self.scale = self.hidden_size_per_attention_head**-0.5 - self.attn = MMEncoderAttention( - self.num_attention_heads_per_partition, - self.hidden_size_per_attention_head, - self.scale, - prefix=f"{prefix}.attn", - ) - - def forward( - self, - x: torch.Tensor, - ) -> torch.Tensor: - qkv, _ = self.qkv(x) - q, k, v = qkv.chunk(3, dim=-1) - out = self.attn(q, k, v) - output, _ = self.o_proj(out) - return output - - -class HunYuanVisionBlock(nn.Module): - def __init__( - self, - dim: int, - num_heads: int, - mlp_hidden_dim: int, - act_fn: Callable[[torch.Tensor], torch.Tensor] = F.gelu, - norm_layer: Callable[[int], nn.Module] | None = None, - quant_config: QuantizationConfig | None = None, - prefix: str = "", - ) -> None: - super().__init__() - if norm_layer is None: - norm_layer = partial(nn.LayerNorm, eps=1e-6) - self.input_layernorm = norm_layer(dim) - self.post_attention_layernorm = norm_layer(dim) - self.self_attn = HunYuanVisionAttention( - embed_dim=dim, - num_heads=num_heads, - projection_size=dim, - quant_config=quant_config, - prefix=f"{prefix}.self_attn", - ) - self.mlp = HunYuanVisionMLP( - dim, - mlp_hidden_dim, - act_fn=act_fn, - bias=True, - quant_config=quant_config, - prefix=f"{prefix}.mlp", - ) - - def forward( - self, - x: torch.Tensor, - ) -> torch.Tensor: - x = x + self.self_attn(self.input_layernorm(x)) - x = x + self.mlp(self.post_attention_layernorm(x)) - return x - - -class HunYuanVisionPatchEmbed(nn.Module): - def __init__(self, config: HunYuanVLVisionConfig): - super().__init__() - - self.config = config - self.embed_dim = config.hidden_size - self.patch_size = config.patch_size - self.num_channels = config.num_channels - self.spatial_merge_size = config.spatial_merge_size - self.interpolate_mode = config.interpolate_mode - - self.patch_embedding = nn.Conv2d( - in_channels=config.num_channels, - out_channels=self.embed_dim, - kernel_size=self.patch_size, - stride=self.patch_size, - bias=True, - ) - - self.max_num_patches = (config.max_image_size // self.patch_size) ** 2 - - self.num_positions = self.max_num_patches + 1 - self.position_edge = int(self.num_positions**0.5) - # first token is cls token, skip it - self.position_embedding = nn.Embedding(self.num_positions, self.embed_dim) - - self.patch_pos_embed = None - - def forward( - self, pixel_values: torch.Tensor, grid_thw: list[list[int]] - ) -> torch.Tensor: - num_patches = pixel_values.size(0) - pixel_values = pixel_values.reshape( - num_patches, self.num_channels, self.patch_size, self.patch_size - ) - - patch_embeds = self.patch_embedding(pixel_values) - patch_embeds = patch_embeds.squeeze(-1).squeeze(-1).unsqueeze(0) - - if self.patch_pos_embed is None: - patch_pos_shape = ( - 1, - self.position_edge, - self.position_edge, - self.embed_dim, - ) - self.patch_pos_embed = ( - self.position_embedding.weight[1:, :] - .reshape(patch_pos_shape) - .permute(0, 3, 1, 2) - .float() - ) - - patch_pos_embed_list = [] - for grid in grid_thw: - _, h0, w0 = grid - # we add a small number to avoid floating point error in the interpolation - # see discussion at https://github.com/facebookresearch/dino/issues/8 - h0, w0 = h0 + 0.1, w0 + 0.1 - patch_pos_embed = nn.functional.interpolate( - self.patch_pos_embed, - scale_factor=(h0 / self.position_edge, w0 / self.position_edge), - mode=self.interpolate_mode, - align_corners=False, - ) - - patch_pos_embed = ( - patch_pos_embed.reshape(self.embed_dim, -1) - .transpose(0, 1) - .unsqueeze(0) - .to(patch_embeds.dtype) - ) - patch_pos_embed_list.append(patch_pos_embed) - - patch_pos_embed = torch.cat(patch_pos_embed_list, dim=1) - embeddings = patch_embeds + patch_pos_embed - - return embeddings - - -class HunYuanVisionPatchMerger(nn.Module): - def __init__( - self, - in_channels, - out_channels, - spatial_merge_size=2, - rms_norm_eps=1e-5, - prefix="", - ): - super().__init__() - self.spatial_merge_size = spatial_merge_size - embed_std = out_channels**-0.5 - - self.proj = nn.Sequential( - nn.Conv2d( - in_channels, - in_channels * 2, - kernel_size=spatial_merge_size, - stride=spatial_merge_size, - ), - nn.GELU(), - nn.Conv2d(in_channels * 2, in_channels * 4, kernel_size=1), - ) - self.mlp = nn.Linear(in_channels * 4, out_channels) - - self.image_newline = nn.Parameter(torch.randn(in_channels * 4) * embed_std) - self.image_begin = nn.Parameter(torch.randn(out_channels) * embed_std) - self.image_end = nn.Parameter(torch.randn(out_channels) * embed_std) - self.image_sep = nn.Parameter(torch.randn(out_channels) * embed_std) - - self.before_rms = RMSNorm(in_channels, eps=rms_norm_eps) - self.after_rms = RMSNorm(out_channels, eps=rms_norm_eps) - - def forward(self, x, size=(16, 16)): - x = self.before_rms(x) - - h, w = size - dtype = x.dtype - x = x.permute(0, 2, 1).reshape(x.shape[0], -1, h, w) - - x = self.proj(x) # b,c,h,w - b, c, h, w = x.shape - x = torch.cat( - [x, self.image_newline.reshape(1, c, 1, 1).expand(b, c, h, 1).to(dtype)], - dim=-1, - ) - x = x.reshape(b, c, -1).permute(0, 2, 1) - x = self.mlp(x) - - begin = self.image_begin.reshape(1, 1, -1).expand(b, 1, x.shape[-1]).to(dtype) - end = self.image_end.reshape(1, 1, -1).expand(b, 1, x.shape[-1]).to(dtype) - x = torch.cat([begin, x, end], dim=1) - - return self.after_rms(x) - - -class HunYuanVisionTransformer(nn.Module): - hf_to_vllm_mapper = WeightsMapper( - orig_to_new_stacked={ - ".q_proj": (".qkv", "q"), - ".k_proj": (".qkv", "k"), - ".v_proj": (".qkv", "v"), - } - ) - - def __init__( - self, - vision_config: HunYuanVLVisionConfig, - quant_config: QuantizationConfig | None = None, - prefix: str = "", - ) -> None: - super().__init__() - - num_hidden_layers = vision_config.num_hidden_layers - self.hidden_size = vision_config.hidden_size - self.num_heads = vision_config.num_attention_heads - self.spatial_merge_size = vision_config.spatial_merge_size - - from vllm.compilation.backends import set_model_tag - - with set_model_tag("HunYuanVisionPatchEmbed"): - self.embeddings = HunYuanVisionPatchEmbed(vision_config) - - norm_layer = partial(nn.LayerNorm, eps=vision_config.rms_norm_eps) - - with set_model_tag("HunYuanVisionBlock"): - self.layers = nn.ModuleList( - [ - HunYuanVisionBlock( - dim=vision_config.hidden_size, - num_heads=vision_config.num_attention_heads, - mlp_hidden_dim=vision_config.intermediate_size, - act_fn=get_act_fn(vision_config.hidden_act), - norm_layer=norm_layer, - quant_config=quant_config, - prefix=f"{prefix}.layers.{layer_idx}", - ) - for layer_idx in range(num_hidden_layers) - ] - ) - - with set_model_tag("HunYuanVisionPatchMerger"): - self.perceive = HunYuanVisionPatchMerger( - vision_config.hidden_size, - vision_config.out_hidden_size, - spatial_merge_size=vision_config.spatial_merge_size, - rms_norm_eps=vision_config.rms_norm_eps, - prefix=f"{prefix}.perceive", - ) - - @property - def dtype(self) -> torch.dtype: - return self.embeddings.patch_embedding.weight.dtype - - @property - def device(self) -> torch.device: - return self.embeddings.patch_embedding.weight.device - - def forward( - self, - x: torch.Tensor, - grid_thw: list[list[int]], - ) -> torch.Tensor: - # patchify - seq_len = x.size(0) - cu_seqlens: list = [0] - - hidden_states = x.to(device=self.device, dtype=self.dtype) - # embeddings = patch_embeds + patch_pos_embed - hidden_states = self.embeddings(hidden_states, grid_thw) - - for t, h, w in grid_thw: - t, h, w = int(t), int(h), int(w) - cu_seqlens.append(h * w) - - cu_seqlens = torch.tensor(cu_seqlens, dtype=torch.int32) - cu_seqlens = torch.cumsum(cu_seqlens, dim=0, dtype=torch.int32) - - cu_seqlens = cu_seqlens.to(device=self.device, non_blocking=True) - - hidden_states = hidden_states.reshape(seq_len, -1) - hidden_states = hidden_states.unsqueeze(0) - - # build per-image lengths once - split_lengths = [int(h) * int(w) for (_, h, w) in grid_thw] - for layer in self.layers: - # hidden_states: (1, T_total, D) - parts = hidden_states.split(split_lengths, dim=1) # list of (1, L_i, D) - parts = [layer(p) for p in parts] - hidden_states = torch.cat(parts, dim=1) - - # adapter - split_lengths = (cu_seqlens[1:] - cu_seqlens[:-1]).tolist() - split_items = hidden_states.split(split_lengths, dim=1) - image_embeds_list = [] - for grid, split_item in zip(grid_thw, split_items): - image_embeds_list.append( - self.perceive(split_item.contiguous(), size=grid[1:]).squeeze(0) - ) - - return image_embeds_list - - def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: - loader = AutoWeightsLoader(self) - return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper) - - -def _hunyuan_vl_field_config(hf_inputs: Mapping[str, torch.Tensor]): - image_grid_thw = hf_inputs.get("image_grid_thw", torch.empty((0, 3))) - image_grid_sizes = image_grid_thw.prod(-1) - return dict( - pixel_values=MultiModalFieldConfig.flat_from_sizes("image", image_grid_sizes), - image_embeds=MultiModalFieldConfig.flat_from_sizes("image", image_grid_sizes), - image_grid_thw=MultiModalFieldConfig.batched("image", keep_on_cpu=True), - ) - - -class HunYuanVLMultiModalDataParser(MultiModalDataParser): - # The patch grid is what sizes the placeholder range. - embedding_fields = { - "image": {"image_embeds": "values", "image_grid_thw": "metadata"}, - } - - def _parse_image_data( - self, - data: dict[str, torch.Tensor] | ModalityData[ImageItem], - ) -> ModalityDataItems[Any, Any] | None: - if isinstance(data, dict): - required, optional = self.embedding_field_sets("image") - return DictEmbeddingItems( - data, - modality="image", - required_fields=required, - optional_fields=optional, - fields_factory=_hunyuan_vl_field_config, - ) - - return super()._parse_image_data(data) - - -class HunYuanVLProcessingInfo(BaseProcessingInfo): - def get_hf_config(self): - return self.ctx.get_hf_config(HunYuanVLConfig) - - def get_hf_processor( - self, - **kwargs: object, - ) -> HunYuanVLProcessor: - # transformers>=5.13 replaced `use_fast` with `backend`; pin the - # PIL backend to match the released HunyuanOCR checkpoint packing. - kwargs.pop("use_fast", None) - kwargs.setdefault("backend", "pil") - return self.ctx.get_hf_processor( - HunYuanVLProcessor, - **kwargs, - ) - - def get_image_processor( - self, - **kwargs: object, - ) -> HunYuanVLImageProcessor: - return self.get_hf_processor(**kwargs).image_processor - - def get_data_parser(self): - return HunYuanVLMultiModalDataParser( - expected_hidden_size=self._get_expected_hidden_size(), - allow_missing_mm_embeddings=self.allow_missing_mm_embeddings, - ) - - def get_supported_mm_limits(self) -> Mapping[str, int | None]: - return {"image": None} - - def get_mm_max_tokens_per_item( - self, - seq_len: int, - mm_counts: Mapping[str, int], - ) -> Mapping[str, int]: - max_image_tokens = self.get_max_image_tokens() - # TODO: support video - max_video_tokens = 0 - return {"image": max_image_tokens, "video": max_video_tokens} - - def _get_vision_info( - self, - *, - image_width: int, - image_height: int, - num_frames: int = 1, - do_resize: bool = True, - image_processor: HunYuanVLImageProcessor, - mm_kwargs: Mapping[str, object], - ) -> tuple[ImageSize, int]: - hf_config = self.get_hf_config() - vision_config = hf_config.vision_config - patch_size = vision_config.patch_size - spatial_merge_size = vision_config.spatial_merge_size - - mm_kwargs = self.ctx.get_merged_mm_kwargs(mm_kwargs) - size = image_processor.size - if override_size := mm_kwargs.get("size"): - size = size | override_size - if (override_min_pixels := mm_kwargs.get("min_pixels")) is not None: - size = size | {"shortest_edge": override_min_pixels} - if (override_max_pixels := mm_kwargs.get("max_pixels")) is not None: - size = size | {"longest_edge": override_max_pixels} - - if do_resize: - resized_height, resized_width = smart_resize( - height=image_height, - width=image_width, - factor=patch_size * spatial_merge_size, - min_pixels=size["shortest_edge"], - max_pixels=size["longest_edge"], - ) - preprocessed_size = ImageSize(width=resized_width, height=resized_height) - else: - preprocessed_size = ImageSize(width=image_width, height=image_height) - - grid_t = 1 - grid_h = preprocessed_size.height // patch_size - grid_w = preprocessed_size.width // patch_size - - num_vision_tokens = ( - grid_t * grid_h // spatial_merge_size * (grid_w // spatial_merge_size + 1) - + 2 - ) - - return preprocessed_size, num_vision_tokens - - def get_num_image_tokens( - self, - *, - image_width: int, - image_height: int, - image_processor: HunYuanVLImageProcessor, - mm_kwargs: Mapping[str, object], - ) -> int: - _, num_image_tokens = self._get_vision_info( - image_width=image_width, - image_height=image_height, - image_processor=image_processor, - mm_kwargs=mm_kwargs, - ) - return num_image_tokens - - def get_image_size_with_most_features(self) -> ImageSize: - image_processor = self.get_image_processor() - - max_image_size, _ = self._get_vision_info( - image_width=512, - image_height=8192, - image_processor=image_processor, - mm_kwargs={}, - ) - return max_image_size - - def get_max_image_tokens(self) -> int: - image_processor = self.get_image_processor() - target_width, target_height = self.get_image_size_with_most_features() - - return self.get_num_image_tokens( - image_width=target_width, - image_height=target_height, - image_processor=image_processor, - mm_kwargs={}, - ) - - -class HunYuanVLDummyInputsBuilder(BaseDummyInputsBuilder[HunYuanVLProcessingInfo]): - def get_dummy_text(self, mm_counts: Mapping[str, int]) -> str: - num_images = mm_counts.get("image", 0) - - hf_processor = self.info.get_hf_processor(typ=HunYuanVLProcessor) - image_placeholder = ( - f"{hf_processor.image_start_token}{hf_processor.image_token}" - f"{hf_processor.image_end_token}" - ) - - return image_placeholder * num_images - - def get_dummy_mm_data( - self, - seq_len: int, - mm_counts: Mapping[str, int], - mm_options: Mapping[str, BaseDummyOptions], - ) -> MultiModalDataDict: - num_images = mm_counts.get("image", 1) - - target_width, target_height = self.info.get_image_size_with_most_features() - - return { - "image": self._get_dummy_images( - width=target_width, height=target_height, num_images=num_images - ), - } - - -class HunYuanVLMultiModalProcessor(BaseMultiModalProcessor[HunYuanVLProcessingInfo]): - def _call_hf_processor( - self, - prompt: str, - mm_data: Mapping[str, object], - mm_kwargs: Mapping[str, object], - ) -> BatchFeature: - hf_processor = self.info.get_hf_processor(**mm_kwargs) - # HunYuanVLProcessor requires image placeholders wrapped with start/end tokens. - if mm_data.get("images") is not None and prompt: - img_tok = hf_processor.image_token - wrapped = ( - f"{hf_processor.image_start_token}{img_tok}" - f"{hf_processor.image_end_token}" - ) - if img_tok in prompt and wrapped not in prompt: - prompt = prompt.replace(img_tok, wrapped) - return self.info.ctx.call_hf_processor( - hf_processor, - dict(text=prompt, **mm_data), - mm_kwargs, - ) - - def _get_prompt_updates( - self, - mm_items: MultiModalDataItems, - hf_processor_mm_kwargs: Mapping[str, Any], - out_mm_kwargs: MultiModalKwargsItems, - ) -> Sequence[PromptUpdate]: - hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs) - image_processor = self.info.get_image_processor(**hf_processor_mm_kwargs) - - token_ids = { - "image": hf_processor.image_token_id, - "image_start": hf_processor.image_start_token_id, - "image_end": hf_processor.image_end_token_id, - } - - merge_size = image_processor.merge_size - - def get_replacement_hunyuan_vl(item_idx: int, modality: str): - out_item = out_mm_kwargs[modality][item_idx] - grid_thw = out_item[f"{modality}_grid_thw"].data - assert isinstance(grid_thw, torch.Tensor) - - _, grid_h, grid_w = grid_thw - num_tokens = (int(grid_h) // merge_size) * ( - int(grid_w) // merge_size + 1 - ) + 2 - tokens = ( - [token_ids[f"{modality}_start"]] - + [token_ids[modality]] * num_tokens - + [token_ids[f"{modality}_end"]] - ) - return PromptUpdateDetails.select_token_id(tokens, token_ids[modality]) - - return [ - PromptReplacement( - modality=modality, - target=[ - token_ids[f"{modality}_start"], - token_ids[modality], - token_ids[f"{modality}_end"], - ], - replacement=partial(get_replacement_hunyuan_vl, modality=modality), - ) - for modality in ("image",) - ] - - def _get_mm_fields_config( - self, - hf_inputs: BatchFeature, - hf_processor_mm_kwargs: Mapping[str, object], - ) -> Mapping[str, MultiModalFieldConfig]: - return _hunyuan_vl_field_config(hf_inputs) - - -@MULTIMODAL_REGISTRY.register_processor( - HunYuanVLMultiModalProcessor, - info=HunYuanVLProcessingInfo, - dummy_inputs=HunYuanVLDummyInputsBuilder, -) -class HunYuanVLForConditionalGeneration( - nn.Module, - SupportsMultiModal, - SupportsLoRA, - SupportsPP, - SupportsQuant, - SupportsXDRoPE, - SupportsEagle, - SupportsEagle3, -): - # To ensure correct weight loading and mapping. - hf_to_vllm_mapper = WeightsMapper( - orig_to_new_prefix={ - # mapping for new names in checkpoint saved after transformers v4.52 - "vit.vit.": "visual.", - "vit.": "visual.", - "model.": "language_model.model.", - } - ) - - supports_encoder_tp_data = True - - def get_xdrope_input_positions( - self, - input_tokens: list[int], - mm_features: list[MultiModalFeatureSpec], - ) -> torch.Tensor: - kwargs = MultiModalFeatureSpec.gather_kwargs( - mm_features, - {"image_grid_thw"}, - ) - image_grid_thw = [item.tolist() for item in kwargs.get("image_grid_thw", [])] - - hf_config = self.config - image_start_token_id = hf_config.image_start_token_id - spatial_merge_size = hf_config.vision_config.spatial_merge_size - xd_num = len(hf_config.rope_scaling["xdrope_section"]) - - input_tokens_tensor = torch.tensor(input_tokens) - image_start_indices = torch.argwhere( - input_tokens_tensor == image_start_token_id - ).squeeze(1) - - p_index = torch.arange(len(input_tokens_tensor)) - w_index = torch.arange(len(input_tokens_tensor)) - h_index = torch.arange(len(input_tokens_tensor)) - t_index = torch.arange(len(input_tokens_tensor)) - for image_index in range(len(image_start_indices)): - # +1 : first image_token, +2: for xdrope positions - pos = image_start_indices[image_index] + 2 - t, h, w = image_grid_thw[image_index] - _, llm_grid_h, llm_grid_w = ( - t, - h // spatial_merge_size, - w // spatial_merge_size, - ) - - token_num = (llm_grid_w + 1) * llm_grid_h - w_index[pos : pos + token_num].copy_( - torch.arange(0, llm_grid_w + 1) - .reshape(1, -1) - .expand(llm_grid_h, -1) - .reshape(-1) - ) - h_index[pos : pos + token_num].copy_( - torch.arange(0, llm_grid_h) - .reshape(-1, 1) - .expand(-1, llm_grid_w + 1) - .reshape(-1) - ) - t_index[pos : pos + token_num] = image_index - - if xd_num == 4: - llm_positions = torch.stack([p_index, w_index, h_index, t_index]) - elif xd_num == 3: - llm_positions = torch.stack([w_index, h_index, t_index]) - - return llm_positions - - @classmethod - def get_placeholder_str(cls, modality: str, i: int) -> str | None: - if modality.startswith("image"): - return "<｜hy_place▁holder▁no▁100｜><｜hy_place▁holder▁no▁102｜><｜hy_place▁holder▁no▁101｜>" # noqa: E501 - - raise ValueError("Only image modality is supported") - - def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""): - super().__init__() - config: HunYuanVLConfig = vllm_config.model_config.hf_config - - self.config = config - - with self._mark_tower_model(vllm_config, {"image"}): - self.visual = HunYuanVisionTransformer( - config.vision_config, - quant_config=vllm_config.quant_config, - prefix=maybe_prefix(prefix, "visual"), - ) - - with self._mark_language_model(vllm_config): - self.language_model = init_vllm_registered_model( - vllm_config=vllm_config, - prefix=maybe_prefix(prefix, "language_model.model"), - architectures=[ - "HunYuanDenseV1ForCausalLM", - "HunYuanMoEV1ForCausalLM", - ], - ) - - self.make_empty_intermediate_tensors = ( - self.language_model.make_empty_intermediate_tensors - ) - - def _parse_and_validate_image_input( - self, **kwargs: object - ) -> HunYuanVLImageInputs | None: - pixel_values = kwargs.pop("pixel_values", None) - image_embeds = kwargs.pop("image_embeds", None) - image_grid_thw = kwargs.pop("image_grid_thw", None) - - if pixel_values is None and image_embeds is None: - return None - - # TODO: refine - if isinstance(pixel_values, list): - pixel_values = torch.cat(pixel_values, dim=0) - if len(pixel_values.shape) == 3: - last_dim = pixel_values.shape[-1] - pixel_values = pixel_values.reshape(-1, last_dim) - image_grid_thw = image_grid_thw.reshape(-1, 3) - - if pixel_values is not None: - return HunYuanVLImagePixelInputs( - type="pixel_values", - pixel_values=pixel_values, - image_grid_thw=image_grid_thw, - ) - - if image_embeds is not None: - return HunYuanVLImageEmbeddingInputs( - type="image_embeds", - image_embeds=image_embeds, - image_grid_thw=image_grid_thw, - ) - - def _process_image_input( - self, image_input: HunYuanVLImageInputs - ) -> tuple[torch.Tensor, ...]: - grid_thw = image_input["image_grid_thw"] - assert grid_thw.ndim == 2 - grid_thw_list = grid_thw.tolist() - - if image_input["type"] == "image_embeds": - image_embeds = image_input["image_embeds"].type(self.visual.dtype) - else: - pixel_values = image_input["pixel_values"] - - # TODO: use_data_parallel (split image_embeds in visual) - image_embeds = self.visual(pixel_values, grid_thw=grid_thw_list) - - return image_embeds - - def _parse_and_validate_multimodal_inputs(self, **kwargs: object) -> dict: - mm_input_by_modality = {} - - # Preserve the order of modalities if there are multiple of them - # from the order of kwargs. - for input_key in kwargs: - if ( - input_key in ("pixel_values", "image_embeds") - and "image" not in mm_input_by_modality - ): - mm_input_by_modality["image"] = self._parse_and_validate_image_input( - **kwargs - ) - return mm_input_by_modality - - def embed_multimodal(self, **kwargs: object) -> MultiModalEmbeddings: - mm_input_by_modality = self._parse_and_validate_multimodal_inputs(**kwargs) - if not mm_input_by_modality: - return [] - - # The result multimodal_embeddings is tuple of tensors, with each - # tensor correspoending to a multimodal data item (image or video). - multimodal_embeddings: tuple[torch.Tensor, ...] = () - - # NOTE: It is important to iterate over the keys in this dictionary - # to preserve the order of the modalities. - for modality in mm_input_by_modality: - multimodal_input = mm_input_by_modality[modality] - if modality == "image": - image_embeddings = self._process_image_input(multimodal_input) - multimodal_embeddings += tuple(image_embeddings) - return multimodal_embeddings - - def forward( - self, - input_ids: torch.Tensor | None, - positions: torch.Tensor, - intermediate_tensors: IntermediateTensors | None, - inputs_embeds: torch.Tensor | None, - **kwargs: object, - ) -> torch.Tensor | IntermediateTensors: - if intermediate_tensors is not None: - inputs_embeds = None - - hidden_states = self.language_model( - input_ids=input_ids, - positions=positions, - intermediate_tensors=intermediate_tensors, - inputs_embeds=inputs_embeds, - ) - return hidden_states - - def compute_logits( - self, - hidden_states: torch.Tensor, - ) -> torch.Tensor | None: - return self.language_model.compute_logits(hidden_states) - - def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: - loader = AutoWeightsLoader(self) - return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper) - - def get_mm_mapping(self) -> MultiModelKeys: - """ - Get the module prefix in multimodal models - """ - return MultiModelKeys.from_string_field( - language_model="language_model.model", - connector="visual.perceive", - tower_model="visual", - ) diff --git a/vllm/model_executor/models/registry.py b/vllm/model_executor/models/registry.py index 24fc66a089..d8e10ba75f 100644 --- a/vllm/model_executor/models/registry.py +++ b/vllm/model_executor/models/registry.py @@ -131,8 +131,6 @@ _TEXT_GENERATION_MODELS = { "GraniteSWAForCausalLM": ("granite", "GraniteForCausalLM"), "GritLM": ("gritlm", "GritLM"), "HrmTextForCausalLM": ("hrm_text", "HrmTextForCausalLM"), - "HunYuanMoEV1ForCausalLM": ("hunyuan_v1", "HunYuanMoEV1ForCausalLM"), - "HunYuanDenseV1ForCausalLM": ("hunyuan_v1", "HunYuanDenseV1ForCausalLM"), "HYV3ForCausalLM": ("hy_v3", "HYV3ForCausalLM"), "HCXVisionForCausalLM": ("hyperclovax_vision", "HCXVisionForCausalLM"), "HCXVisionV2ForCausalLM": ("hyperclovax_vision_v2", "HCXVisionV2ForCausalLM"), @@ -441,10 +439,6 @@ _MULTIMODAL_MODELS = { "Granite4VisionForConditionalGeneration", ), "H2OVLChatModel": ("h2ovl", "H2OVLChatModel"), - "HunYuanVLForConditionalGeneration": ( - "hunyuan_vision", - "HunYuanVLForConditionalGeneration", - ), "InternVLChatModel": ("internvl", "InternVLChatModel"), "InternS1ForConditionalGeneration": ( "interns1", @@ -700,6 +694,8 @@ _SPECULATIVE_DECODING_MODELS = { _TRANSFORMERS_SUPPORTED_MODELS = { # Text generation models "GPTBigCodeForCausalLM": ("transformers", "TransformersForCausalLM"), + "HunYuanDenseV1ForCausalLM": ("transformers", "TransformersForCausalLM"), + "HunYuanMoEV1ForCausalLM": ("transformers", "TransformersMoEForCausalLM"), "OlmoForCausalLM": ("transformers", "TransformersForCausalLM"), "Olmo2ForCausalLM": ("transformers", "TransformersForCausalLM"), "SmolLM3ForCausalLM": ("transformers", "TransformersForCausalLM"), @@ -710,6 +706,10 @@ _TRANSFORMERS_SUPPORTED_MODELS = { "transformers", "TransformersMultiModalForCausalLM", ), + "HunYuanVLForConditionalGeneration": ( + "transformers", + "TransformersMultiModalForCausalLM", + ), "VibeVoiceAsrForConditionalGeneration": ( "transformers", "TransformersMultiModalForCausalLM", diff --git a/vllm/transformers_utils/config.py b/vllm/transformers_utils/config.py index 26d4bd6e1b..d58a386f58 100644 --- a/vllm/transformers_utils/config.py +++ b/vllm/transformers_utils/config.py @@ -95,7 +95,6 @@ _CONFIG_REGISTRY: dict[str, type[PretrainedConfig]] = LazyConfigDict( granite4_vision="Granite4VisionConfig", hyperclovax="HyperCLOVAXConfig", hyperclovax_vlm="HCXVisionConfig", - hunyuan_vl="HunYuanVLConfig", hy_v3="HYV3Config", isaac="IsaacConfig", kimi_k2="DeepseekV3Config", # Kimi K2 uses same architecture as DeepSeek V3 diff --git a/vllm/transformers_utils/configs/__init__.py b/vllm/transformers_utils/configs/__init__.py index c38cc4b4b8..d0b3b962e4 100644 --- a/vllm/transformers_utils/configs/__init__.py +++ b/vllm/transformers_utils/configs/__init__.py @@ -44,9 +44,6 @@ _CLASS_TO_MODULE: dict[str, str] = { "FunAudioChatConfig": "vllm.transformers_utils.configs.funaudiochat", "FunAudioChatAudioEncoderConfig": "vllm.transformers_utils.configs.funaudiochat", "Granite4VisionConfig": "vllm.transformers_utils.configs.granite4_vision", - "HunYuanVLConfig": "vllm.transformers_utils.configs.hunyuan_vl", - "HunYuanVLTextConfig": "vllm.transformers_utils.configs.hunyuan_vl", - "HunYuanVLVisionConfig": "vllm.transformers_utils.configs.hunyuan_vl", "HCXVisionConfig": "vllm.transformers_utils.configs.hyperclovax", "HYV3Config": "vllm.transformers_utils.configs.hy_v3", "HyperCLOVAXConfig": "vllm.transformers_utils.configs.hyperclovax", @@ -141,9 +138,6 @@ __all__ = [ "FunAudioChatConfig", "FunAudioChatAudioEncoderConfig", "Granite4VisionConfig", - "HunYuanVLConfig", - "HunYuanVLTextConfig", - "HunYuanVLVisionConfig", "HCXVisionConfig", "HYV3Config", "HyperCLOVAXConfig", diff --git a/vllm/transformers_utils/configs/hunyuan_vl.py b/vllm/transformers_utils/configs/hunyuan_vl.py deleted file mode 100644 index 548dcfefca..0000000000 --- a/vllm/transformers_utils/configs/hunyuan_vl.py +++ /dev/null @@ -1,281 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project - -# adapted from https://github.com/ManaEstras/transformers/blob/v4.57.1.hyvl/src/transformers/models/hunyuan_vl/configuration_hunyuan_vl.py - -from transformers import PretrainedConfig - - -class HunYuanVLVisionConfig(PretrainedConfig): - model_type = "hunyuan_vl" - base_config_key = "vision_config" - - def __init__( - self, - hidden_act="gelu", - hidden_size=1152, - intermediate_size=4304, - interpolate_mode="bilinear", - rms_norm_eps=1e-05, - learnable_mlp_pooling_size=0, - num_attention_heads=16, - num_key_value_heads=None, - num_channels=3, - num_hidden_layers=27, - out_hidden_size=4096, - patch_size=16, - remove_prenorm=True, - spatial_merge_size=2, - temporal_patch_size=1, - resize_resolution=2048, - img_max_token_num=4096, - max_image_size=2048, - video_max_image_size=768, - video_min_image_size=256, - min_image_size=512, - anyres_vit_max_image_size=2048, - max_vit_seq_len=16384, - text_hidden_size=3072, - **kwargs, - ): - super().__init__(**kwargs) - - self.hidden_act = hidden_act - self.hidden_size = hidden_size - self.intermediate_size = intermediate_size - self.interpolate_mode = interpolate_mode - self.learnable_mlp_pooling_size = learnable_mlp_pooling_size - self.num_attention_heads = num_attention_heads - if not num_key_value_heads: - self.num_key_value_heads = num_attention_heads - else: - self.num_key_value_heads = num_key_value_heads - self.num_channels = num_channels - self.num_hidden_layers = num_hidden_layers - self.out_hidden_size = out_hidden_size - self.patch_size = patch_size - self.remove_prenorm = remove_prenorm - self.spatial_merge_size = spatial_merge_size - self.temporal_patch_size = temporal_patch_size - self.rms_norm_eps = rms_norm_eps - - self.resize_resolution = resize_resolution - self.img_max_token_num = img_max_token_num - self.max_image_size = max_image_size - self.min_image_size = min_image_size - self.video_max_image_size = video_max_image_size - self.video_min_image_size = video_min_image_size - self.anyres_vit_max_image_size = anyres_vit_max_image_size - self.max_vit_seq_len = max_vit_seq_len - self.text_hidden_size = text_hidden_size - - -class HunYuanVLTextConfig(PretrainedConfig): - r""" - This is the configuration class to store the configuration of a [`HunYuanVLTextConfig`]. It is used to instantiate an - HunYuan model according to the specified arguments, defining the model architecture. Instantiating a configuration - with the defaults will yield a similar configuration to that of the HunYuan-7B. - Hunyuan-7B-Instruct [tencent/Hunyuan-7B-Instruct](https://huggingface.co/tencent/Hunyuan-7B-Instruct). - - Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the - documentation from [`PretrainedConfig`] for more information. - - - Args: - vocab_size (`int`, *optional*, defaults to 290943): - Vocabulary size of the HunYuan model. Defines the number of different tokens that can be represented by the - `inputs_ids` passed when calling [`HunYuanVLTextConfig`] - hidden_size (`int`, *optional*, defaults to 4096): - Dimension of the hidden representations. - intermediate_size (`int`, *optional*, defaults to 11008): - Dimension of the MLP representations or shared MLP representations. - num_hidden_layers (`int`, *optional*, defaults to 32): - Number of hidden layers in the Transformer decoder. - num_attention_heads (`int`, *optional*, defaults to 32): - Number of attention heads for each attention layer in the Transformer decoder. - num_key_value_heads (`int`, *optional*): - This is the number of key_value heads that should be used to implement Grouped Query Attention. If - `num_key_value_heads=num_attention_heads`, the model will use Multi Head Attention (MHA), if - `num_key_value_heads=1 the model will use Multi Query Attention (MQA) otherwise GQA is used. When - converting a multi-head checkpoint to a GQA checkpoint, each group key and value head should be constructed - by meanpooling all the original heads within that group. For more details checkout [this - paper](https://huggingface.co/papers/2305.13245). If it is not specified, will default to - `num_attention_heads`. - hidden_act (`str` or `function`, *optional*, defaults to `"silu"`): - The non-linear activation function (function or string) in the decoder. - max_position_embeddings (`int`, *optional*, defaults to 2048): - The maximum sequence length that this model might ever be used with. - initializer_range (`float`, *optional*, defaults to 0.02): - The standard deviation of the truncated_normal_initializer for initializing all weight matrices. - rms_norm_eps (`float`, *optional*, defaults to 1e-05): - The epsilon used by the rms normalization layers. - use_cache (`bool`, *optional*, defaults to `True`): - Whether or not the model should return the last key/values attentions (not used by all models). Only - relevant if `config.is_decoder=True`. - pad_token_id (`int`, *optional*, defaults to 0): - Padding token id. - bos_token_id (`int`, *optional*, defaults to 1): - Beginning of stream token id. - eos_token_id (`int`, *optional*, defaults to 2): - End of stream token id. - eod_token_id (int, *optional*, defaults to 3): - Token ID representing the end-of-document marker. Used to indicate the termination of a text sequence. - Example: In multi-document processing, this token helps the model distinguish between separate documents. - pretraining_tp (`int`, *optional*, defaults to 1): - Experimental feature. Tensor parallelism rank used during pretraining. Please refer to [this - document](https://huggingface.co/docs/transformers/parallelism) to understand more about it. This value is - necessary to ensure exact reproducibility of the pretraining results. Please refer to [this - issue](https://github.com/pytorch/pytorch/issues/76232). - tie_word_embeddings (`bool`, *optional*, defaults to `False`): - Whether to tie weight embeddings - rope_theta (`float`, *optional*, defaults to 10000.0): - The base period of the RoPE embeddings. - rope_scaling (`Dict`, *optional*): - Dictionary containing the scaling configuration for the RoPE embeddings. Currently supports two scaling - strategies: linear and dynamic. Their scaling factor must be a float greater than 1. The expected format is - `{"type": strategy name, "factor": scaling factor}`. When using this flag, don't update - `max_position_embeddings` to the expected new maximum. See the following thread for more information on how - these scaling strategies behave: - https://www.reddit.com/r/LocalLLaMA/comments/14mrgpr/dynamically_scaled_rope_further_increases/. This is an - experimental feature, subject to breaking API changes in future versions. - attention_bias (`bool`, defaults to `False`, *optional*, defaults to `False`): - Whether to use a bias in the query, key, value and output projection layers during self-attention. - attention_dropout (`float`, *optional*, defaults to 0.0): - The dropout ratio for the attention probabilities. - head_dim (`int`, *optional*, defaults to 128): - The attention head dimension. - """ # noqa: E501 - - model_type = "hunyuan_vl_text" - keys_to_ignore_at_inference = ["past_key_values"] - - def __init__( - self, - vocab_size=290943, - hidden_size=4096, - intermediate_size: int = 11008, - num_hidden_layers=32, - num_attention_heads=32, - num_key_value_heads=None, - hidden_act="silu", - max_position_embeddings=2048, - initializer_range=0.02, - rms_norm_eps=1e-5, - use_cache=True, - pad_token_id=0, - bos_token_id=1, - eos_token_id=2, - eod_token_id=3, - pretraining_tp=1, - tie_word_embeddings=False, - rope_theta=10000.0, - rope_scaling=None, - attention_bias=False, - attention_dropout=0.0, - head_dim=None, - **kwargs, - ): - self.vocab_size = vocab_size - self.max_position_embeddings = max_position_embeddings - self.hidden_size = hidden_size - self.intermediate_size = intermediate_size - self.num_hidden_layers = num_hidden_layers - self.num_attention_heads = num_attention_heads - self.head_dim = head_dim - # for backward compatibility - if num_key_value_heads is None: - num_key_value_heads = num_attention_heads - - self.num_key_value_heads = num_key_value_heads - self.hidden_act = hidden_act - self.initializer_range = initializer_range - self.rms_norm_eps = rms_norm_eps - self.pretraining_tp = pretraining_tp - self.use_cache = use_cache - self.rope_theta = rope_theta - self.rope_scaling = rope_scaling - self.attention_bias = attention_bias - self.attention_dropout = attention_dropout - - super().__init__( - pad_token_id=pad_token_id, - bos_token_id=bos_token_id, - eos_token_id=eos_token_id, - tie_word_embeddings=tie_word_embeddings, - **kwargs, - ) - - -class HunYuanVLConfig(PretrainedConfig): - model_type = "hunyuan_vl" - sub_configs = { - "vision_config": HunYuanVLVisionConfig, - "text_config": HunYuanVLTextConfig, - } - keys_to_ignore_at_inference = ["past_key_values"] - - def __init__( - self, - text_config=None, - vision_config=None, - im_start_id=120118, - im_end_id=120119, - image_token_id=120120, - im_newline_id=120121, - video_start_id=120122, - video_end_id=120123, - **kwargs, - ): - # We need to init super() here so that it does not reset values - # that are in text config to the BaseClass defaults. The Base - # config has many text related defaults and not all defaults are - # same as for `HunYuanVLTextConfig`. - super().__init__(**kwargs) - - if isinstance(vision_config, dict): - self.vision_config = self.sub_configs["vision_config"](**vision_config) - elif vision_config is None: - self.vision_config = self.sub_configs["vision_config"]() - - if isinstance(text_config, dict): - self.text_config = self.sub_configs["text_config"](**text_config) - elif text_config is None: - # For BC use all kwargs to init `TextConfig` - self.text_config = self.sub_configs["text_config"](**kwargs) - - self.image_token_id = image_token_id - self.im_start_id = im_start_id - self.im_end_id = im_end_id - self.im_newline_id = im_newline_id - self.video_start_id = video_start_id - self.video_end_id = video_end_id - - self.vision_config.text_hidden_size = self.text_config.hidden_size - - # Attention implementation to use. It sets it recursively on sub-configs - # so we call it again in the end. - self._attn_implementation = kwargs.pop("attn_implementation", None) - - def __setattr__(self, key, value): - if ( - (text_config := super().__getattribute__("__dict__").get("text_config")) - is not None - and key not in ["dtype", "_attn_implementation_internal"] - and key in text_config.__dict__ - ): - setattr(text_config, key, value) - else: - super().__setattr__(key, value) - - def __getattribute__(self, key): - if "text_config" in super().__getattribute__("__dict__") and key not in [ - "_name_or_path", - "model_type", - "dtype", - "_attn_implementation_internal", - ]: - text_config = super().__getattribute__("text_config") - if key in text_config.__dict__: - return getattr(text_config, key) - - return super().__getattribute__(key)