From 48d7132962ef3e2e6982a316013294c0029e3a64 Mon Sep 17 00:00:00 2001 From: Harry Mellor <19981378+hmellor@users.noreply.github.com> Date: Mon, 24 Aug 2026 22:39:05 -0700 Subject: [PATCH] [Model] Migrate FlexOlmo, Olmo3 and Hunyuan V1/VL to the Transformers modeling backend (#53615) Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com> Signed-off-by: Cyrus Leung Co-authored-by: Xianbao QIAN Co-authored-by: Claude Opus 5 (1M context) Co-authored-by: Cyrus Leung --- docs/models/supported_models.md | 10 +- tests/models/registry.py | 5 +- vllm/model_executor/models/flex_olmo.py | 156 --- vllm/model_executor/models/hunyuan_v1.py | 1038 ----------------- vllm/model_executor/models/hunyuan_vision.py | 1025 ---------------- vllm/model_executor/models/olmo3.py | 412 ------- vllm/model_executor/models/registry.py | 16 +- .../model_executor/models/transformers/moe.py | 25 +- .../models/transformers/multimodal.py | 5 + .../models/transformers/utils.py | 10 +- vllm/transformers_utils/config.py | 2 - vllm/transformers_utils/configs/__init__.py | 8 - vllm/transformers_utils/configs/flex_olmo.py | 82 -- vllm/transformers_utils/configs/hunyuan_vl.py | 281 ----- 14 files changed, 47 insertions(+), 3028 deletions(-) delete mode 100644 vllm/model_executor/models/flex_olmo.py delete mode 100644 vllm/model_executor/models/hunyuan_v1.py delete mode 100644 vllm/model_executor/models/hunyuan_vision.py delete mode 100644 vllm/model_executor/models/olmo3.py delete mode 100644 vllm/transformers_utils/configs/flex_olmo.py delete mode 100644 vllm/transformers_utils/configs/hunyuan_vl.py diff --git a/docs/models/supported_models.md b/docs/models/supported_models.md index 38eebf0e01..a696b02ede 100644 --- a/docs/models/supported_models.md +++ b/docs/models/supported_models.md @@ -384,7 +384,6 @@ th { | `FalconForCausalLM` | Falcon | `tiiuae/falcon-7b`, `tiiuae/falcon-40b`, `tiiuae/falcon-rw-7b`, etc. | | ✅︎ | | `FalconMambaForCausalLM` | FalconMamba | `tiiuae/falcon-mamba-7b`, `tiiuae/falcon-mamba-7b-instruct`, etc. | | ✅︎ | | `FalconH1ForCausalLM` | Falcon-H1 | `tiiuae/Falcon-H1-34B-Base`, `tiiuae/Falcon-H1-34B-Instruct`, etc. | ✅︎ | ✅︎ | -| `FlexOlmoForCausalLM` | FlexOlmo | `allenai/FlexOlmo-7x7B-1T`, `allenai/FlexOlmo-7x7B-1T-RT`, etc. | | ✅︎ | | `GemmaForCausalLM` | Gemma | `google/gemma-2b`, `google/gemma-1.1-2b-it`, etc. | ✅︎ | ✅︎ | | `Gemma2ForCausalLM` | Gemma 2 | `google/gemma-2-9b`, `google/gemma-2-27b`, etc. | ✅︎ | ✅︎ | | `Gemma3ForCausalLM` | Gemma 3 | `google/gemma-3-1b-it`, etc. | ✅︎ | ✅︎ | @@ -406,8 +405,6 @@ th { | `GraniteMoeSWAForCausalLM` | Granite MoE SWA | `ibm-granite/granite-swash-3b-a600m` | ✅︎ | ✅︎ | | `GraniteSWAForCausalLM` | Granite SWA | `ibm-granite/granite-swash-2b` | ✅︎ | ✅︎ | | `HrmTextForCausalLM` | HRM-Text | `sapientinc/HRM-Text-1B`, etc. | | | -| `HunYuanDenseV1ForCausalLM` | Hunyuan Dense | `tencent/Hunyuan-7B-Instruct` | ✅︎ | ✅︎ | -| `HunYuanMoEV1ForCausalLM` | Hunyuan-A13B | `tencent/Hunyuan-A13B-Instruct`, `tencent/Hunyuan-A13B-Pretrain`, `tencent/Hunyuan-A13B-Instruct-FP8`, etc. | ✅︎ | ✅︎ | | `HYV3ForCausalLM` | HY3 | `tencent/Hy3-preview-Base`, `tencent/Hy3-preview` | ✅︎ | ✅︎ | | `HyperCLOVAXForCausalLM` | HyperCLOVAX-SEED-Think-14B | `naver-hyperclovax/HyperCLOVAX-SEED-Think-14B` | ✅︎ | ✅︎ | | `InternLM2ForCausalLM` | InternLM2 | `internlm/internlm2-7b`, `internlm/internlm2-chat-7b`, etc. | ✅︎ | ✅︎ | @@ -436,7 +433,6 @@ th { | `MixtralForCausalLM` | Mixtral-8x7B, Mixtral-8x7B-Instruct | `mistralai/Mixtral-8x7B-v0.1`, `mistralai/Mixtral-8x7B-Instruct-v0.1`, `mistral-community/Mixtral-8x22B-v0.1`, etc. | ✅︎ | ✅︎ | | `NemotronForCausalLM` | Nemotron-3, Nemotron-4, Minitron | `nvidia/Minitron-8B-Base`, `mgoin/Nemotron-4-340B-Base-hf-FP8`, etc. | ✅︎ | ✅︎ | | `NemotronHForCausalLM` | Nemotron-H | `nvidia/Nemotron-H-8B-Base-8K`, `nvidia/Nemotron-H-47B-Base-8K`, `nvidia/Nemotron-H-56B-Base-8K`, etc. | ✅︎ | ✅︎ | -| `Olmo3ForCausalLM` | OLMo3 | `allenai/Olmo-3-7B-Instruct`, `allenai/Olmo-3-32B-Think`, etc. | ✅︎ | ✅︎ | | `OlmoHybridForCausalLM` | OLMo Hybrid | `allenai/Olmo-Hybrid-7B` | ✅︎ | ✅︎ | | `OlmoeForCausalLM` | OLMoE | `allenai/OLMoE-1B-7B-0924`, `allenai/OLMoE-1B-7B-0924-Instruct`, etc. | | ✅︎ | | `OPTForCausalLM` | OPT, OPT-IML | `facebook/opt-66b`, `facebook/opt-iml-max-30b`, etc. | ✅︎ | ✅︎ | @@ -471,9 +467,13 @@ Some models are supported only via the [Transformers modeling backend](#transfor | Architecture | Models | Example HF Models | [LoRA](../features/lora.md) | [PP](../serving/parallelism_scaling.md) | | ------------ | ------ | ----------------- | -------------------- | ------------------------- | +| `FlexOlmoForCausalLM` | FlexOlmo | `allenai/FlexOlmo-7x7B-1T`, `allenai/FlexOlmo-7x7B-1T-RT`, etc. | | ✅︎ | | `GPTBigCodeForCausalLM` | StarCoder, SantaCoder, WizardCoder | `bigcode/starcoder`, `bigcode/gpt_bigcode-santacoder`, `WizardLM/WizardCoder-15B-V1.0`, etc. | ✅︎ | | +| `HunYuanDenseV1ForCausalLM` | Hunyuan Dense | `tencent/Hunyuan-7B-Instruct` | ✅︎ | ✅︎ | +| `HunYuanMoEV1ForCausalLM` | Hunyuan-A13B | `tencent/Hunyuan-A13B-Instruct`, `tencent/Hunyuan-A13B-Pretrain`, `tencent/Hunyuan-A13B-Instruct-FP8`, etc. | ✅︎ | ✅︎ | | `OlmoForCausalLM` | OLMo | `allenai/OLMo-1B-hf`, `allenai/OLMo-7B-hf`, etc. | ✅︎ | ✅︎ | | `Olmo2ForCausalLM` | OLMo2 | `allenai/OLMo-2-0425-1B`, etc. | ✅︎ | ✅︎ | +| `Olmo3ForCausalLM` | OLMo3 | `allenai/Olmo-3-7B-Instruct`, `allenai/Olmo-3-32B-Think`, etc. | ✅︎ | ✅︎ | | `SmolLM3ForCausalLM` | SmolLM3 | `HuggingFaceTB/SmolLM3-3B` | ✅︎ | ✅︎ | | `Starcoder2ForCausalLM` | Starcoder2 | `bigcode/starcoder2-3b`, `bigcode/starcoder2-7b`, `bigcode/starcoder2-15b`, etc. | ✅︎ | ✅︎ | | `VaultGemmaForCausalLM` | VaultGemma | `google/vaultgemma-1b` | ✅︎ | ✅︎ | @@ -543,7 +543,6 @@ These models primarily accept the [`LLM.generate`](./generative_models.md#llmgen | `GraniteSpeechPlusForConditionalGeneration` | Granite Speech Plus | T + A | `ibm-granite/granite-speech-4.1-2b-plus` | ✅︎ | ✅︎ | | `HCXVisionV2ForCausalLM` | HyperCLOVAX-SEED-Think-32B | T + I+ + V+ | `naver-hyperclovax/HyperCLOVAX-SEED-Think-32B` | | | | `H2OVLChatModel` | H2OVL | T + IE+ | `h2oai/h2ovl-mississippi-800m`, `h2oai/h2ovl-mississippi-2b`, etc. | ✅︎ | ✅︎ | -| `HunYuanVLForConditionalGeneration` | HunyuanOCR | T + IE+ | `tencent/HunyuanOCR`, etc. | ✅︎ | ✅︎ | | `Idefics3ForConditionalGeneration` | Idefics3 | T + I | `HuggingFaceM4/Idefics3-8B-Llama3`, etc. | ✅︎ | | | `IsaacForConditionalGeneration` | Isaac | T + I+ | `PerceptronAI/Isaac-0.1` | ✅︎ | ✅︎ | | `InternS1ForConditionalGeneration` | Intern-S1 | T + IE+ + VE+ | `internlm/Intern-S1`, `internlm/Intern-S1-mini`, etc. | ✅︎ | ✅︎ | @@ -620,6 +619,7 @@ Some models are supported only via the [Transformers modeling backend](#transfor | Architecture | Models | Inputs | Example HF Models | [LoRA](../features/lora.md) | [PP](../serving/parallelism_scaling.md) | | ------------ | ------ | ------ | ----------------- | --------------------------- | --------------------------------------- | | `Emu3ForConditionalGeneration` | Emu3 | T + I+ | `BAAI/Emu3-Chat-hf` | ✅︎ | ✅︎ | +| `HunYuanVLForConditionalGeneration` | HunyuanOCR | T + IE+ | `tencent/HunyuanOCR`, etc. | ✅︎ | ✅︎ | | `VibeVoiceAsrForConditionalGeneration` | VibeVoice-ASR | T + A+ | `microsoft/VibeVoice-ASR-HF` | ✅︎ | ✅︎ | ^ You need to set the architecture name via `--hf-overrides` to match the one in vLLM.
diff --git a/tests/models/registry.py b/tests/models/registry.py index c46772db9d..819c7c4ff8 100644 --- a/tests/models/registry.py +++ b/tests/models/registry.py @@ -329,9 +329,7 @@ _TEXT_GENERATION_EXAMPLE_MODELS = { min_transformers_version="5.9.0", ), "HunYuanDenseV1ForCausalLM": _HfExamplesInfo("tencent/Hunyuan-7B-Instruct"), - "HunYuanMoEV1ForCausalLM": _HfExamplesInfo( - "tencent/Hunyuan-A13B-Instruct", trust_remote_code=True - ), + "HunYuanMoEV1ForCausalLM": _HfExamplesInfo("tencent/Hunyuan-A13B-Instruct"), "HYV3ForCausalLM": _HfExamplesInfo("tencent/Hy3-preview", trust_remote_code=True), "HyperCLOVAXForCausalLM": _HfExamplesInfo( "naver-hyperclovax/HyperCLOVAX-SEED-Think-14B", @@ -938,7 +936,6 @@ _MULTIMODAL_EXAMPLE_MODELS = { ), "HunYuanVLForConditionalGeneration": _HfExamplesInfo( "tencent/HunyuanOCR", - hf_overrides={"num_experts": 0}, min_transformers_version="5.13", ), "Idefics3ForConditionalGeneration": _HfExamplesInfo( diff --git a/vllm/model_executor/models/flex_olmo.py b/vllm/model_executor/models/flex_olmo.py deleted file mode 100644 index a0abf9e0fa..0000000000 --- a/vllm/model_executor/models/flex_olmo.py +++ /dev/null @@ -1,156 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project - -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Inference-only FlexOlmo model compatible with HuggingFace weights.""" - -import torch -from torch import nn - -from vllm.config import VllmConfig -from vllm.distributed import get_tensor_model_parallel_world_size -from vllm.logger import init_logger -from vllm.model_executor.layers.fused_moe import ( - FusedMoEFactory, -) -from vllm.model_executor.layers.layernorm import RMSNorm -from vllm.model_executor.layers.linear import ReplicatedLinear -from vllm.model_executor.models.olmoe import OlmoeAttention, OlmoeForCausalLM -from vllm.transformers_utils.configs.flex_olmo import FlexOlmoConfig - -logger = init_logger(__name__) - - -class FlexOlmoAttention(OlmoeAttention): - def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""): - super().__init__(vllm_config=vllm_config, prefix=prefix) - - hf_config = vllm_config.model_config.hf_config - assert isinstance(hf_config, FlexOlmoConfig) - - self.k_norm = RMSNorm( - self.total_num_kv_heads * self.head_dim, eps=hf_config.rms_norm_eps - ) - self.q_norm = RMSNorm( - self.total_num_heads * self.head_dim, eps=hf_config.rms_norm_eps - ) - - -class FlexOlmoMoE(nn.Module): - """A tensor-parallel MoE implementation for FlexOlmo that shards each expert - across all ranks. - - Each expert's weights are sharded across all ranks and a fused MoE - kernel is used for the forward pass, and finally we reduce the outputs - across ranks. - """ - - def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""): - super().__init__() - - hf_config = vllm_config.model_config.hf_config - assert isinstance(hf_config, FlexOlmoConfig) - - tp_size = get_tensor_model_parallel_world_size() - - # Gate always runs at half / full precision for now. - self.gate = ReplicatedLinear( - hf_config.hidden_size, - hf_config.num_experts, - bias=False, - return_bias=False, - quant_config=None, - prefix=f"{prefix}.gate", - ) - - self.experts = FusedMoEFactory( - num_experts=hf_config.num_experts, - top_k=hf_config.num_experts_per_tok, - hidden_size=hf_config.hidden_size, - intermediate_size=hf_config.intermediate_size, - renormalize=False, - quant_config=None, - tp_size=tp_size, - prefix=f"{prefix}.experts", - router_logits_dtype=torch.float32, - ) - - self.top_k = hf_config.num_experts_per_tok - - def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: - # NOTE: hidden_states can have either 1D or 2D shape. - orig_shape = hidden_states.shape - hidden_dim = hidden_states.shape[-1] - hidden_states = hidden_states.view(-1, hidden_dim) - - # router_logits: (num_tokens, n_experts) - router_logits = self.gate(hidden_states) - # Warning: The experts mutate the hidden state input! This messes up - # basic things like the residual stream. - final_hidden_states = self.experts( - hidden_states=hidden_states.detach().clone(), - router_logits=router_logits.float(), - ) - - return final_hidden_states.view(orig_shape) - - -class FlexOlmoDecoderLayer(nn.Module): - def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None: - super().__init__() - hf_config = vllm_config.model_config.hf_config - assert isinstance(hf_config, FlexOlmoConfig) - - self.self_attn = FlexOlmoAttention( - vllm_config=vllm_config, prefix=f"{prefix}.self_attn" - ) - self.post_attention_layernorm = RMSNorm( - hf_config.hidden_size, eps=hf_config.rms_norm_eps - ) - self.post_feedforward_layernorm = RMSNorm( - hf_config.hidden_size, eps=hf_config.rms_norm_eps - ) - - self.mlp = FlexOlmoMoE(vllm_config=vllm_config, prefix=f"{prefix}.mlp") - - def forward( - self, - positions: torch.Tensor, - hidden_states: torch.Tensor, - residual: torch.Tensor | None, - ) -> tuple[torch.Tensor, torch.Tensor | None]: - # Attention block. - residual = hidden_states - hidden_states = self.self_attn(positions, hidden_states) - hidden_states = self.post_attention_layernorm(hidden_states) - hidden_states = hidden_states + residual - - # MLP block. - residual = hidden_states - hidden_states = self.mlp(hidden_states) - hidden_states = self.post_feedforward_layernorm(hidden_states) - hidden_states = residual + hidden_states - return hidden_states, None - - -class FlexOlmoForCausalLM(OlmoeForCausalLM): - fall_back_to_pt_during_load = False - - def __init__( - self, - *, - vllm_config: VllmConfig, - prefix: str = "", - layer_type: type[nn.Module] = FlexOlmoDecoderLayer, - ): - super().__init__(vllm_config=vllm_config, prefix=prefix, layer_type=layer_type) diff --git a/vllm/model_executor/models/hunyuan_v1.py b/vllm/model_executor/models/hunyuan_v1.py deleted file mode 100644 index cf1933bd7d..0000000000 --- a/vllm/model_executor/models/hunyuan_v1.py +++ /dev/null @@ -1,1038 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project - -# coding=utf-8 -# Copyright 2024 The HunYuan team. -# Copyright 2023 The vLLM team. -# Copyright 2022 EleutherAI and the HuggingFace Inc. team. All rights reserved. -# -# This code is based on EleutherAI's GPT-NeoX library and the GPT-NeoX -# and OPT implementations in this library. It has been modified from its -# original forms to accommodate minor architectural differences compared -# to GPT-NeoX and OPT used by the Meta AI team that trained the model. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Inference-only HunYuan model compatible with HuggingFace weights.""" - -import typing -from collections.abc import Callable, Iterable -from itertools import islice - -import regex as re -import torch -from torch import nn -from transformers import PretrainedConfig - -from vllm.compilation.decorators import support_torch_compile -from vllm.config import CacheConfig, VllmConfig, get_current_vllm_config -from vllm.distributed import ( - get_ep_group, - get_pp_group, - get_tensor_model_parallel_world_size, -) -from vllm.model_executor.layers.activation import SiluAndMul -from vllm.model_executor.layers.attention import Attention -from vllm.model_executor.layers.fused_moe import ( - FusedMoEFactory, - fused_moe_make_expert_params_mapping, -) -from vllm.model_executor.layers.layernorm import RMSNorm -from vllm.model_executor.layers.linear import ( - ColumnParallelLinear, - MergedColumnParallelLinear, - QKVParallelLinear, - ReplicatedLinear, - RowParallelLinear, -) -from vllm.model_executor.layers.logits_processor import LogitsProcessor -from vllm.model_executor.layers.quantization import QuantizationConfig -from vllm.model_executor.layers.rotary_embedding import get_rope -from vllm.model_executor.layers.vocab_parallel_embedding import ( - ParallelLMHead, - VocabParallelEmbedding, -) -from vllm.model_executor.model_loader.weight_utils import ( - default_weight_loader, - maybe_remap_kv_scale_name, -) -from vllm.sequence import IntermediateTensors -from vllm.v1.attention.backend import AttentionType - -from .interfaces import ( - EagleModelMixin, - MixtureOfExperts, - SupportsEagle, - SupportsEagle3, - SupportsLoRA, - SupportsPP, -) -from .utils import ( - AutoWeightsLoader, - PPMissingLayer, - is_pp_missing_parameter, - make_layers, - maybe_prefix, -) - - -def _is_moe(config: PretrainedConfig) -> bool: - num_experts = getattr(config, "num_experts", None) - if isinstance(num_experts, int): - return num_experts > 1 - if isinstance(num_experts, list) and num_experts: - # Ensure all elements are integers before calling max. - if all(isinstance(e, int) for e in num_experts): - return max(num_experts) > 1 - else: - return False - return False - - -def _get_cla_factor(config: PretrainedConfig) -> int: - if not getattr(config, "use_cla", False): - return 1 - return getattr(config, "cla_share_factor", 1) - - -class HunYuanMLP(nn.Module): - def __init__( - self, - hidden_size: int, - intermediate_size: int, - hidden_act: str, - quant_config: QuantizationConfig | None = None, - bias: bool = False, - prefix: str = "", - reduce_results: bool = True, - ) -> None: - super().__init__() - self.gate_up_proj = MergedColumnParallelLinear( - input_size=hidden_size, - output_sizes=[intermediate_size] * 2, - bias=bias, - quant_config=quant_config, - prefix=f"{prefix}.gate_up_proj", - ) - self.down_proj = RowParallelLinear( - input_size=intermediate_size, - output_size=hidden_size, - bias=bias, - quant_config=quant_config, - prefix=f"{prefix}.down_proj", - reduce_results=reduce_results, - ) - if hidden_act != "silu": - raise ValueError( - f"Unsupported activation: {hidden_act}. Only silu is supported for now." - ) - self.act_fn = SiluAndMul() - - def forward(self, x): - gate_up, _ = self.gate_up_proj(x) - x = self.act_fn(gate_up) - x, _ = self.down_proj(x) - return x - - -class HunYuanAttention(nn.Module): - def __init__( - self, - config: PretrainedConfig, - hidden_size: int, - num_heads: int, - num_kv_heads: int, - max_position_embeddings: int = 8192, - quant_config: QuantizationConfig | None = None, - bias: bool = False, - cache_config: CacheConfig | None = None, - prefix: str = "", - layer_id: int = -1, - ) -> None: - super().__init__() - self.hidden_size = hidden_size - tp_size = get_tensor_model_parallel_world_size() - self.total_num_heads = num_heads - assert self.total_num_heads % tp_size == 0 - self.num_heads = self.total_num_heads // tp_size - self.total_num_kv_heads = num_kv_heads - if self.total_num_kv_heads >= tp_size: - # Number of KV heads is greater than TP size, so we partition - # the KV heads across multiple tensor parallel GPUs. - assert self.total_num_kv_heads % tp_size == 0 - else: - # Number of KV heads is less than TP size, so we replicate - # the KV heads across multiple tensor parallel GPUs. - assert tp_size % self.total_num_kv_heads == 0 - self.num_kv_heads = max(1, self.total_num_kv_heads // tp_size) - - if hasattr(config, "head_dim") and config.head_dim: - self.head_dim = config.head_dim - elif hasattr(config, "attention_head_dim"): - self.head_dim = config.attention_head_dim - else: - self.head_dim = self.hidden_size // self.total_num_heads - self.q_size = self.num_heads * self.head_dim - self.kv_size = self.num_kv_heads * self.head_dim - self.scaling = self.head_dim**-0.5 - self.max_position_embeddings = max_position_embeddings - self.use_qk_norm = getattr(config, "use_qk_norm", False) - self.layer_id = layer_id - - self.qkv_proj = QKVParallelLinear( - hidden_size=hidden_size, - head_size=self.head_dim, - total_num_heads=self.total_num_heads, - total_num_kv_heads=self.total_num_kv_heads, - bias=bias, - quant_config=quant_config, - prefix=f"{prefix}.qkv_proj", - ) - - self.o_proj = RowParallelLinear( - input_size=self.total_num_heads * self.head_dim, - output_size=hidden_size, - bias=bias, - quant_config=quant_config, - prefix=f"{prefix}.o_proj", - ) - - self.rotary_emb = get_rope( - self.head_dim, - max_position=max_position_embeddings, - rope_parameters=config.rope_parameters, - is_neox_style=True, - ) - self.attn = Attention( - self.num_heads, - self.head_dim, - self.scaling, - num_kv_heads=self.num_kv_heads, - cache_config=cache_config, - quant_config=quant_config, - prefix=f"{prefix}.attn", - ) - - if self.use_qk_norm: - self.query_layernorm = RMSNorm(self.head_dim, eps=config.rms_norm_eps) - self.key_layernorm = RMSNorm(self.head_dim, eps=config.rms_norm_eps) - - def forward( - self, - positions: torch.Tensor, - hidden_states: torch.Tensor, - kv_states: tuple[torch.Tensor] | None = None, - ) -> torch.Tensor: - qkv, _ = self.qkv_proj(hidden_states) - q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1) - q, k = self.rotary_emb(positions, q, k) - ori_k = k - if self.use_qk_norm: - q = self.query_layernorm( - q.view(-1, self.num_heads, self.head_dim), - ) - k = self.key_layernorm( - k.view(-1, self.num_kv_heads, self.head_dim), - ) - - attn_output = self.attn(q, k, v) - # For o_proj - attn_output = attn_output.view(q.shape[0], -1) - output, _ = self.o_proj(attn_output) - return output, (ori_k, v) - - -class HunYuanCrossAttention(nn.Module): - def __init__( - self, - config: PretrainedConfig, - hidden_size: int, - num_heads: int, - num_kv_heads: int, - max_position_embeddings: int = 8192, - quant_config: QuantizationConfig | None = None, - bias: bool = False, - cache_config: CacheConfig | None = None, - prefix: str = "", - layer_id: int = -1, - ) -> None: - super().__init__() - self.hidden_size = hidden_size - tp_size = get_tensor_model_parallel_world_size() - self.total_num_heads = num_heads - assert self.total_num_heads % tp_size == 0 - self.num_heads = self.total_num_heads // tp_size - self.total_num_kv_heads = num_kv_heads - if self.total_num_kv_heads >= tp_size: - # Number of KV heads is greater than TP size, so we partition - # the KV heads across multiple tensor parallel GPUs. - assert self.total_num_kv_heads % tp_size == 0 - else: - # Number of KV heads is less than TP size, so we replicate - # the KV heads across multiple tensor parallel GPUs. - assert tp_size % self.total_num_kv_heads == 0 - self.num_kv_heads = max(1, self.total_num_kv_heads // tp_size) - # MistralConfig has an optional head_dim introduced by Mistral-Nemo - if hasattr(config, "head_dim"): - self.head_dim = config.head_dim - elif hasattr(config, "attention_head_dim"): - self.head_dim = config.attention_head_dim - else: - self.head_dim = self.hidden_size // self.total_num_heads - self.q_size = self.num_heads * self.head_dim - self.kv_size = self.num_kv_heads * self.head_dim - self.scaling = self.head_dim**-0.5 - self.max_position_embeddings = max_position_embeddings - self.use_qk_norm = getattr(config, "use_qk_norm", False) - self.layer_id = layer_id - - self.q_proj = ColumnParallelLinear( - hidden_size, - hidden_size, - bias=bias, - quant_config=quant_config, - prefix=f"{prefix}.q_proj", - ) - - self.o_proj = RowParallelLinear( - input_size=self.total_num_heads * self.head_dim, - output_size=hidden_size, - bias=bias, - quant_config=quant_config, - prefix=f"{prefix}.o_proj", - ) - - self.rotary_emb = get_rope( - self.head_dim, - max_position=max_position_embeddings, - rope_parameters=config.rope_parameters, - is_neox_style=True, - ) - self.attn = Attention( - self.num_heads, - self.head_dim, - self.scaling, - num_kv_heads=self.num_kv_heads, - cache_config=cache_config, - quant_config=quant_config, - prefix=f"{prefix}.attn", - attn_type=AttentionType.ENCODER_DECODER, - ) - - if self.use_qk_norm: - self.query_layernorm = RMSNorm(self.head_dim, eps=config.rms_norm_eps) - self.key_layernorm = RMSNorm(self.head_dim, eps=config.rms_norm_eps) - - def forward( - self, - positions: torch.Tensor, - hidden_states: torch.Tensor, - kv_states: tuple[torch.Tensor] | None = None, - ) -> torch.Tensor: - assert kv_states is not None - ori_k, v = kv_states # use last layer kv, - k = ori_k - q, _ = self.q_proj(hidden_states) - k_tmp = torch.empty_like(k) # Todo: reduant rotary embedding - q, _ = self.rotary_emb(positions, q, k_tmp) - if self.use_qk_norm: - q = self.query_layernorm( - q.view(-1, self.num_heads, self.head_dim), - ) - k = self.key_layernorm( - k.view(-1, self.num_kv_heads, self.head_dim), - ) - - attn_output = self.attn(q, k, v) - # For o_proj - attn_output = attn_output.view(q.shape[0], -1) - output, _ = self.o_proj(attn_output) - return output, (ori_k, v) - - -class HunYuanSparseMoeBlock(nn.Module): - def __init__( - self, - config: PretrainedConfig, - quant_config: QuantizationConfig | None = None, - layer_id: int = -1, - prefix: str = "", - enable_eplb: bool = False, - ): - super().__init__() - self.tp_size = get_tensor_model_parallel_world_size() - - self.ep_group = get_ep_group().device_group - self.ep_size = self.ep_group.size() - self.n_routed_experts = config.num_experts - - if self.tp_size > config.num_experts: - raise ValueError( - f"Tensor parallel size {self.tp_size} is greater than " - f"the number of experts {config.num_experts}." - ) - - # Get layer_id topk if config.moe_topk is a list - if isinstance(config.moe_topk, list): - assert layer_id >= 0 - assert len(config.moe_topk) > layer_id - top_k = config.moe_topk[layer_id] - else: - top_k = config.moe_topk - - # If it is moe, moe_intermediate_size is preferred - intermediate_size = config.intermediate_size - if config.moe_intermediate_size is not None: - intermediate_size = ( - config.moe_intermediate_size - if isinstance(config.moe_intermediate_size, int) - else config.moe_intermediate_size[layer_id] - ) - - # Load balancing settings. - vllm_config = get_current_vllm_config() - eplb_config = vllm_config.parallel_config.eplb_config - self.enable_eplb = enable_eplb - - self.n_logical_experts = self.n_routed_experts - self.n_redundant_experts = eplb_config.num_redundant_experts - self.n_physical_experts = self.n_logical_experts + self.n_redundant_experts - self.n_local_physical_experts = self.n_physical_experts // self.ep_size - self.gate = ReplicatedLinear( - config.hidden_size, - config.num_experts, - bias=False, - quant_config=None, - prefix=f"{prefix}.gate", - ) - if config.use_mixed_mlp_moe > 0: - # Get layer_id num_shared_expert if config.num_shared_expert is - # a list. - if isinstance(config.num_shared_expert, list): - assert layer_id >= 0 - assert len(config.num_shared_expert) > layer_id - num_shared_expert = config.num_shared_expert[layer_id] - else: - num_shared_expert = config.num_shared_expert - - self.shared_mlp = HunYuanMLP( - hidden_size=config.hidden_size, - intermediate_size=config.intermediate_size * num_shared_expert, - hidden_act=config.hidden_act, - quant_config=quant_config, - reduce_results=False, - prefix=f"{prefix}.shared_mlp", - ) - else: - self.shared_mlp = None - - self.experts = FusedMoEFactory( - shared_experts=self.shared_mlp, - num_experts=self.n_routed_experts, - top_k=top_k, - hidden_size=config.hidden_size, - intermediate_size=intermediate_size, - renormalize=top_k > 1, - quant_config=quant_config, - prefix=f"{prefix}.experts", - enable_eplb=self.enable_eplb, - num_redundant_experts=self.n_redundant_experts, - ) - - def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: - # NOTE: hidden_states can have either 1D or 2D shape. - orig_shape = hidden_states.shape - hidden_dim = hidden_states.shape[-1] - hidden_states = hidden_states.view(-1, hidden_dim) - - # router_logits: (num_tokens, n_experts) - router_logits, _ = self.gate(hidden_states) - final_hidden_states = self.experts( - hidden_states=hidden_states, router_logits=router_logits - ) - - return final_hidden_states.view(orig_shape) - - -class HunYuanDecoderLayer(nn.Module): - def __init__( - self, - config: PretrainedConfig, - cache_config: CacheConfig | None = None, - quant_config: QuantizationConfig | None = None, - prefix: str = "", - layer_id: int = -1, - enable_eplb: bool = False, - ) -> None: - super().__init__() - assert layer_id >= 0 - self.layer_id = layer_id - self.hidden_size = config.hidden_size - self.intermediate_size = ( - config.intermediate_size - if isinstance(config.intermediate_size, int) - else config.intermediate_size[layer_id] - ) - max_position_embeddings = getattr(config, "max_position_embeddings", 8192) - attention_bias = getattr(config, "attention_bias", False) or getattr( - config, "bias", False - ) - cla_factor = _get_cla_factor(config) - attention_type = ( - AttentionType.ENCODER_DECODER - if layer_id >= 0 and layer_id % cla_factor != 0 - else AttentionType.DECODER - ) - if attention_type == AttentionType.DECODER: - self.self_attn = HunYuanAttention( - config=config, - hidden_size=self.hidden_size, - num_heads=config.num_attention_heads, - num_kv_heads=getattr( - config, "num_key_value_heads", config.num_attention_heads - ), - max_position_embeddings=max_position_embeddings, - quant_config=quant_config, - bias=attention_bias, - cache_config=cache_config, - prefix=f"{prefix}.self_attn", - layer_id=layer_id, - ) - elif attention_type == AttentionType.ENCODER_DECODER: - self.self_attn = HunYuanCrossAttention( - config=config, - hidden_size=self.hidden_size, - num_heads=config.num_attention_heads, - num_kv_heads=getattr( - config, "num_key_value_heads", config.num_attention_heads - ), - max_position_embeddings=max_position_embeddings, - quant_config=quant_config, - bias=attention_bias, - cache_config=cache_config, - prefix=f"{prefix}.self_attn", - layer_id=layer_id, - ) - else: - raise RuntimeError(f"Unsupported attention type: {attention_type}") - - if _is_moe(config): - self.mlp = HunYuanSparseMoeBlock( - config=config, - quant_config=quant_config, - layer_id=layer_id, - prefix=f"{prefix}.mlp", - enable_eplb=enable_eplb, - ) - else: - self.mlp = HunYuanMLP( - hidden_size=self.hidden_size, - intermediate_size=self.intermediate_size, - hidden_act=config.hidden_act, - quant_config=quant_config, - bias=getattr(config, "mlp_bias", False), - prefix=f"{prefix}.mlp", - ) - - self.input_layernorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps) - self.post_attention_layernorm = RMSNorm( - config.hidden_size, eps=config.rms_norm_eps - ) - - def forward( - self, - positions: torch.Tensor, - hidden_states: torch.Tensor, - residual: torch.Tensor | None, - kv_states: tuple[torch.Tensor] | None = None, - ) -> tuple[torch.Tensor, torch.Tensor]: - # Self Attention - if residual is None: - residual = hidden_states - hidden_states = self.input_layernorm(hidden_states) - else: - hidden_states, residual = self.input_layernorm(hidden_states, residual) - hidden_states, ori_kv_states = self.self_attn( - positions=positions, - hidden_states=hidden_states, - kv_states=kv_states, - ) - - # Fully Connected - hidden_states, residual = self.post_attention_layernorm(hidden_states, residual) - hidden_states = self.mlp(hidden_states) - return hidden_states, residual, ori_kv_states - - -@support_torch_compile( - dynamic_arg_dims={ - "input_ids": 0, - # positions is of shape (xd, seq_len) if xdrope is enabled for hunyuan-vl, - # otherwise (seq_len, ). - "positions": -1, - "intermediate_tensors": 0, - "inputs_embeds": 0, - } -) -class HunYuanModel(nn.Module, EagleModelMixin): - def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""): - super().__init__() - - config = vllm_config.model_config.hf_config - cache_config = vllm_config.cache_config - quant_config = vllm_config.quant_config - - eplb_config = vllm_config.parallel_config.eplb_config - enable_eplb = vllm_config.parallel_config.enable_eplb - self.num_redundant_experts = eplb_config.num_redundant_experts - - self.config = config - self.quant_config = quant_config - - self.vocab_size = config.vocab_size - - if get_pp_group().is_first_rank or ( - config.tie_word_embeddings and get_pp_group().is_last_rank - ): - self.embed_tokens = VocabParallelEmbedding( - self.vocab_size, - config.hidden_size, - quant_config=quant_config, - ) - else: - self.embed_tokens = PPMissingLayer() - self.start_layer, self.end_layer, self.layers = make_layers( - config.num_hidden_layers, - lambda prefix: HunYuanDecoderLayer( - config=config, - layer_id=int(prefix.split(".")[-1]), - cache_config=cache_config, - quant_config=quant_config, - prefix=prefix, - enable_eplb=enable_eplb, - ), - prefix=f"{prefix}.layers", - ) - if get_pp_group().is_last_rank: - self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps) - else: - self.norm = PPMissingLayer() - - def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor: - return self.embed_tokens(input_ids) - - def forward( - self, - input_ids: torch.Tensor | None, - positions: torch.Tensor, - intermediate_tensors: IntermediateTensors | None, - inputs_embeds: torch.Tensor | None = None, - ) -> torch.Tensor | IntermediateTensors: - if get_pp_group().is_first_rank: - if inputs_embeds is not None: - hidden_states = inputs_embeds - else: - hidden_states = self.embed_input_ids(input_ids) - residual = None - else: - assert intermediate_tensors is not None - hidden_states = intermediate_tensors["hidden_states"] - residual = intermediate_tensors["residual"] - - cla_factor = _get_cla_factor(self.config) - prev_kv_states = None - aux_hidden_states = self._maybe_add_hidden_state([], 0, hidden_states, residual) - for i, layer in enumerate( - islice(self.layers, self.start_layer, self.end_layer) - ): - hidden_states, residual, kv_states = layer( - positions, - hidden_states, - residual, - prev_kv_states, - ) - - if getattr(self.config, "use_cla", False) and i % cla_factor == 0: - prev_kv_states = kv_states - else: - prev_kv_states = None - - self._maybe_add_hidden_state( - aux_hidden_states, i + 1, hidden_states, residual - ) - - if not get_pp_group().is_last_rank: - return IntermediateTensors( - {"hidden_states": hidden_states, "residual": residual} - ) - - hidden_states, _ = self.norm(hidden_states, residual) - - if len(aux_hidden_states) > 0: - return hidden_states, aux_hidden_states - return hidden_states - - def _split_qkv_weight(self, qkv: torch.Tensor): - num_attention_heads = self.config.num_attention_heads - num_kv_heads = getattr( - self.config, "num_key_value_heads", self.config.num_attention_heads - ) - num_key_value_groups = num_attention_heads // num_kv_heads - hidden_size = self.config.hidden_size - - if hasattr(self.config, "head_dim"): - attention_head_dim = self.config.head_dim - elif hasattr(self.config, "attention_head_dim"): - attention_head_dim = self.config.attention_head_dim - else: - attention_head_dim = self.config.hidden_size // num_attention_heads - - qkv = qkv.reshape( - num_kv_heads, num_key_value_groups + 2, attention_head_dim, hidden_size - ) - q, k, v = torch.split(qkv, (num_key_value_groups, 1, 1), dim=1) - q = q.reshape(-1, hidden_size) - k = k.reshape(-1, hidden_size) - v = v.reshape(-1, hidden_size) - return torch.concat((q, k, v)) - - def get_expert_mapping(self) -> list[tuple[str, str, int, str]]: - if _is_moe(self.config): - # Params for weights, fp8 weight scales, fp8 activation scales - # (param_name, weight_name, expert_id, shard_id) - return fused_moe_make_expert_params_mapping( - self, - ckpt_gate_proj_name="gate_proj", - ckpt_down_proj_name="down_proj", - ckpt_up_proj_name="up_proj", - num_experts=self.config.num_experts, - num_redundant_experts=self.num_redundant_experts, - ) - else: - return [] - - def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]): - cla_factor = _get_cla_factor(self.config) - stacked_params_mapping = [ - # (param_name, shard_name, shard_id) - (".qkv_proj", ".q_proj", "q"), - (".qkv_proj", ".k_proj", "k"), - (".qkv_proj", ".v_proj", "v"), - (".gate_up_proj", ".gate_proj", 0), - (".gate_up_proj", ".up_proj", 1), - ] - - num_attention_heads = self.config.num_attention_heads - num_kv_heads = getattr( - self.config, "num_key_value_heads", self.config.num_attention_heads - ) - split_params_mapping = [ - (".gate_up_proj", ".gate_and_up_proj", 2, [(1, 1), (0, 1)], None), - ( - ".qkv_proj", - ".qkv_proj", - num_attention_heads + num_kv_heads * 2, - [("q", num_attention_heads), ("k", num_kv_heads), ("v", num_kv_heads)], - self._split_qkv_weight, - ), - ] - - params_dict = dict(self.named_parameters()) - loaded_params: set[str] = set() - expert_params_mapping = self.get_expert_mapping() - for name, loaded_weight in weights: - if "rotary_emb.inv_freq" in name: - continue - if "gate_proj_bias" in name: - name = name.replace("gate_proj_bias", "gate_proj.bias") - if "up_proj_bias" in name: - name = name.replace("up_proj_bias", "up_proj.bias") - if "rotary_emb.cos_cached" in name or "rotary_emb.sin_cached" in name: - # Models trained using ColossalAI may include these tensors in - # the checkpoint. Skip them. - continue - # With tie_word_embeddings, we can skip lm_head.weight - # The weight might appear unnecessarily in the files if the model is - # processed with quantization, LoRA, fine-tuning, etc. - if self.config.tie_word_embeddings and "lm_head.weight" in name: - continue - - is_found = False - for param_name, weight_name, shard_id in stacked_params_mapping: - if weight_name not in name: - continue - if "mlp.experts" in name: - continue - # cross layer only have q_proj, skip qkv pack - if weight_name == ".q_proj": - match = re.search(r"layers\.\d+", name) - if match: - layer_id = int(match.group(0).split(".")[-1]) - if cla_factor > 1 and layer_id % cla_factor != 0: - continue - name = name.replace(weight_name, param_name) - # Skip loading extra bias for GPTQ models. - if name.endswith(".bias") and name not in params_dict: - continue - - if is_pp_missing_parameter(name, self): - continue - - param = params_dict[name] - weight_loader = param.weight_loader - weight_loader(param, loaded_weight, shard_id) - loaded_params.add(name) - is_found = True - break - if is_found: - continue - - for ( - param_name, - weight_name, - den, - split_param, - func, - ) in split_params_mapping: - if weight_name not in name: - continue - name = name.replace(weight_name, param_name) - # Skip loading extra bias for GPTQ models. - if name.endswith(".bias") and name not in params_dict: - continue - - if is_pp_missing_parameter(name, self): - continue - - assert loaded_weight.shape[0] % den == 0 - units = loaded_weight.shape[0] // den - - param = params_dict[name] - weight_loader = param.weight_loader - offset = 0 - for shard_id, num in split_param: - new_offset = offset + num * units - if func: - weight_loader( - param, func(loaded_weight)[offset:new_offset], shard_id - ) - else: - weight_loader(param, loaded_weight[offset:new_offset], shard_id) - offset = new_offset - - break - else: - # Skip loading extra bias for GPTQ models. - if name.endswith(".bias") and name not in params_dict: - continue - is_expert_weight = False - for mapping in expert_params_mapping: - param_name, weight_name, expert_id, shard_id = mapping - if weight_name not in name: - continue - # this is an expert weight and should not be - # attempted to load as other weights later - is_expert_weight = True - - # Do not modify `name` since the loop may continue here - # Instead, create a new variable - name_mapped = name.replace(weight_name, param_name) - if is_pp_missing_parameter(name_mapped, self): - continue - param = params_dict[name_mapped] - # We should ask the weight loader to return success or not - # here since otherwise we may skip experts with other - # available replicas. - weight_loader = typing.cast( - Callable[..., bool], param.weight_loader - ) - success = weight_loader( - param, - loaded_weight, - name_mapped, - shard_id=shard_id, - expert_id=expert_id, - return_success=True, - ) - if success: - name = name_mapped - break - else: - if is_expert_weight: - # We've checked that this is an expert weight - # However it's not mapped locally to this rank - # So we simply skip it - continue - # Remapping the name of FP8 kv-scale. - name = maybe_remap_kv_scale_name(name, params_dict) - if name is None: - continue - - if is_pp_missing_parameter(name, self): - continue - - if "mlp.gate.wg." in name: - name = name.replace("wg.", "") - - param = params_dict[name] - weight_loader = getattr( - param, "weight_loader", default_weight_loader - ) - weight_loader(param, loaded_weight) - loaded_params.add(name) - return loaded_params - - -class HunyuanV1ModelBase( - nn.Module, SupportsLoRA, SupportsPP, SupportsEagle, SupportsEagle3 -): - packed_modules_mapping = { - "qkv_proj": [ - "q_proj", - "k_proj", - "v_proj", - ], - "gate_up_proj": [ - "gate_proj", - "up_proj", - ], - } - - def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""): - super().__init__() - - config = vllm_config.model_config.hf_config - quant_config = vllm_config.quant_config - self.config = config - self.quant_config = quant_config - - self.model = HunYuanModel( - vllm_config=vllm_config, - prefix=maybe_prefix(prefix, "model"), - ) - if get_pp_group().is_last_rank: - self.lm_head = ParallelLMHead( - config.vocab_size, - config.hidden_size, - quant_config=quant_config, - prefix=maybe_prefix(prefix, "lm_head"), - ) - if config.tie_word_embeddings: - self.lm_head = self.lm_head.tie_weights(self.model.embed_tokens) - - logit_scale = getattr(config, "logit_scale", 1.0) - self.logits_processor = LogitsProcessor( - config.vocab_size, scale=logit_scale - ) - else: - self.lm_head = PPMissingLayer() - - def forward( - self, - input_ids: torch.Tensor | None, - positions: torch.Tensor, - intermediate_tensors: IntermediateTensors | None = None, - inputs_embeds: torch.Tensor | None = None, - ) -> torch.Tensor | IntermediateTensors: - model_output = self.model( - input_ids, positions, intermediate_tensors, inputs_embeds - ) - return model_output - - def compute_logits( - self, - hidden_states: torch.Tensor, - ) -> torch.Tensor | None: - logits = self.logits_processor(self.lm_head, hidden_states) - return logits - - def make_empty_intermediate_tensors( - self, batch_size: int, dtype: torch.dtype, device: torch.device - ) -> IntermediateTensors: - return IntermediateTensors( - { - "hidden_states": torch.zeros( - (batch_size, self.config.hidden_size), dtype=dtype, device=device - ), - "residual": torch.zeros( - (batch_size, self.config.hidden_size), dtype=dtype, device=device - ), - } - ) - - def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: - loader = AutoWeightsLoader(self) - return loader.load_weights(weights) - - def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor: - return self.model.embed_input_ids(input_ids) - - -class HunYuanMoEV1Base(HunyuanV1ModelBase, MixtureOfExperts): - def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""): - super().__init__(vllm_config=vllm_config, prefix=prefix) - - # Set MoE hyperparameters - self.num_expert_groups = 1 - self.moe_layers = [] - example_layer = None - for layer in self.model.layers: - if isinstance(layer, PPMissingLayer): - continue - - assert isinstance(layer, HunYuanDecoderLayer) - if isinstance(layer.mlp, HunYuanSparseMoeBlock): - example_layer = layer.mlp - self.moe_layers.append(layer.mlp.experts) - - if example_layer is None: - raise RuntimeError("No HunYuanMoE layer found in model.layers.") - - self.num_moe_layers = len(self.moe_layers) - self.num_logical_experts = example_layer.n_logical_experts - self.num_physical_experts = example_layer.n_physical_experts - self.num_local_physical_experts = example_layer.n_local_physical_experts - self.num_routed_experts = example_layer.n_routed_experts - self.num_redundant_experts = example_layer.n_redundant_experts - - def update_physical_experts_metadata( - self, - num_physical_experts: int, - num_local_physical_experts: int, - ) -> None: - assert self.num_local_physical_experts == num_local_physical_experts - self.num_physical_experts = num_physical_experts - self.num_local_physical_experts = num_local_physical_experts - self.num_redundant_experts = num_physical_experts - self.num_logical_experts - for layer in self.model.layers: - if isinstance(layer.mlp, HunYuanSparseMoeBlock): - moe = layer.mlp - moe.n_local_physical_experts = num_local_physical_experts - moe.n_physical_experts = num_physical_experts - moe.n_redundant_experts = self.num_redundant_experts - moe.experts.update_expert_map() - - def get_expert_mapping(self) -> list[tuple[str, str, int, str]]: - return self.model.get_expert_mapping() - - -class HunYuanDenseV1Base(HunyuanV1ModelBase): - def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""): - super().__init__(vllm_config=vllm_config, prefix=prefix) - - -class HunYuanDenseV1ForCausalLM(HunYuanDenseV1Base): - pass - - -class HunYuanMoEV1ForCausalLM(HunYuanMoEV1Base): - pass diff --git a/vllm/model_executor/models/hunyuan_vision.py b/vllm/model_executor/models/hunyuan_vision.py deleted file mode 100644 index ccc3f8fee9..0000000000 --- a/vllm/model_executor/models/hunyuan_vision.py +++ /dev/null @@ -1,1025 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project - -# coding=utf-8 -# Copyright 2025 The HunYuan team. -# Copyright 2025 The vLLM team. -# Copyright 2025 EleutherAI and the HuggingFace Inc. team. All rights reserved. -# -# This code is based on EleutherAI's GPT-NeoX library and the GPT-NeoX -# and OPT implementations in this library. It has been modified from its -# original forms to accommodate minor architectural differences compared -# to GPT-NeoX and OPT used by the Meta AI team that trained the model. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Inference-only HunYuan-VL model compatible with HuggingFace weights.""" - -from collections.abc import Callable, Iterable, Mapping, Sequence -from functools import partial -from typing import Annotated, Any, Literal, TypeAlias - -import torch -import torch.nn as nn -import torch.nn.functional as F -from transformers import BatchFeature, HunYuanVLProcessor -from transformers.models.hunyuan_vl.image_processing_hunyuan_vl import ( - HunYuanVLImageProcessor, - smart_resize, -) - -from vllm.config import VllmConfig -from vllm.config.multimodal import BaseDummyOptions -from vllm.distributed import parallel_state -from vllm.distributed import utils as dist_utils -from vllm.inputs import ModalityData, MultiModalDataDict -from vllm.logger import init_logger -from vllm.model_executor.layers.activation import get_act_fn -from vllm.model_executor.layers.attention import MMEncoderAttention -from vllm.model_executor.layers.layernorm import RMSNorm -from vllm.model_executor.layers.linear import ( - ColumnParallelLinear, - QKVParallelLinear, - RowParallelLinear, -) -from vllm.model_executor.layers.quantization import QuantizationConfig -from vllm.model_executor.models.module_mapping import MultiModelKeys -from vllm.multimodal import MULTIMODAL_REGISTRY -from vllm.multimodal.inputs import ( - ImageItem, - MultiModalFeatureSpec, - MultiModalFieldConfig, - MultiModalKwargsItems, -) -from vllm.multimodal.parse import ( - DictEmbeddingItems, - ImageSize, - ModalityDataItems, - MultiModalDataItems, - MultiModalDataParser, -) -from vllm.multimodal.processing import ( - BaseDummyInputsBuilder, - BaseMultiModalProcessor, - BaseProcessingInfo, - PromptReplacement, - PromptUpdate, - PromptUpdateDetails, -) -from vllm.sequence import IntermediateTensors -from vllm.transformers_utils.configs.hunyuan_vl import ( - HunYuanVLConfig, - HunYuanVLVisionConfig, -) -from vllm.utils.tensor_schema import TensorSchema, TensorShape - -from .interfaces import ( - MultiModalEmbeddings, - SupportsEagle, - SupportsEagle3, - SupportsLoRA, - SupportsMultiModal, - SupportsPP, - SupportsQuant, - SupportsXDRoPE, -) -from .utils import ( - AutoWeightsLoader, - WeightsMapper, - init_vllm_registered_model, - maybe_prefix, -) -from .vision import is_vit_use_data_parallel - -logger = init_logger(__name__) - -# === Vision Inputs === # - - -class HunYuanVLImagePixelInputs(TensorSchema): - """ - Dimensions: - - np: Number of patches - - ni: Number of images - - cps: Number of channels * patch_size * patch_size - """ - - type: Literal["pixel_values"] - - pixel_values: Annotated[ - torch.Tensor, - TensorShape("np", "cps"), - ] - - image_grid_thw: Annotated[ - torch.Tensor, - TensorShape("ni", 3), - ] - - -class HunYuanVLImageEmbeddingInputs(TensorSchema): - """ - Dimensions: - - nf: Number of image features - - hs: Hidden size - - ni: Number of images - """ - - type: Literal["image_embeds"] - - image_embeds: Annotated[ - torch.Tensor, - TensorShape("nf", "hs"), - ] - - image_grid_thw: Annotated[ - torch.Tensor, - TensorShape("ni", 3), - ] - - -HunYuanVLImageInputs: TypeAlias = ( - HunYuanVLImagePixelInputs | HunYuanVLImageEmbeddingInputs -) - -# === Vision Encoder === # - - -class HunYuanVisionMLP(nn.Module): - def __init__( - self, - in_features: int, - hidden_features: int, - bias: bool = True, - act_fn: Callable[[torch.Tensor], torch.Tensor] = F.gelu, - quant_config: QuantizationConfig | None = None, - prefix: str = "", - ): - super().__init__() - use_data_parallel = is_vit_use_data_parallel() - self.dense_h_to_4h = ColumnParallelLinear( - in_features, - hidden_features, - bias=bias, - quant_config=quant_config, - prefix=f"{prefix}.dense_h_to_4h", - disable_tp=use_data_parallel, - ) - self.dense_4h_to_h = RowParallelLinear( - hidden_features, - in_features, - bias=bias, - quant_config=quant_config, - prefix=f"{prefix}.dense_4h_to_h", - disable_tp=use_data_parallel, - ) - self.act_fn = act_fn - - def forward(self, x: torch.Tensor): - x_up, _ = self.dense_h_to_4h(x) - x_down, _ = self.dense_4h_to_h(self.act_fn(x_up)) - return x_down - - -class HunYuanVisionAttention(nn.Module): - def __init__( - self, - embed_dim: int, - num_heads: int, - projection_size: int, - quant_config: QuantizationConfig | None = None, - prefix: str = "", - ) -> None: - super().__init__() - # Per attention head and per partition values. - use_data_parallel = is_vit_use_data_parallel() - self.tp_size = ( - 1 - if use_data_parallel - else parallel_state.get_tensor_model_parallel_world_size() - ) - self.hidden_size_per_attention_head = dist_utils.divide( - projection_size, num_heads - ) - self.num_attention_heads_per_partition = dist_utils.divide( - num_heads, self.tp_size - ) - - self.qkv = QKVParallelLinear( - hidden_size=embed_dim, - head_size=self.hidden_size_per_attention_head, - total_num_heads=num_heads, - total_num_kv_heads=num_heads, - bias=True, - quant_config=quant_config, - prefix=f"{prefix}.qkv", - disable_tp=use_data_parallel, - ) - - self.o_proj = RowParallelLinear( - input_size=projection_size, - output_size=embed_dim, - quant_config=quant_config, - prefix=f"{prefix}.o_proj", - disable_tp=use_data_parallel, - ) - - self.scale = self.hidden_size_per_attention_head**-0.5 - self.attn = MMEncoderAttention( - self.num_attention_heads_per_partition, - self.hidden_size_per_attention_head, - self.scale, - prefix=f"{prefix}.attn", - ) - - def forward( - self, - x: torch.Tensor, - ) -> torch.Tensor: - qkv, _ = self.qkv(x) - q, k, v = qkv.chunk(3, dim=-1) - out = self.attn(q, k, v) - output, _ = self.o_proj(out) - return output - - -class HunYuanVisionBlock(nn.Module): - def __init__( - self, - dim: int, - num_heads: int, - mlp_hidden_dim: int, - act_fn: Callable[[torch.Tensor], torch.Tensor] = F.gelu, - norm_layer: Callable[[int], nn.Module] | None = None, - quant_config: QuantizationConfig | None = None, - prefix: str = "", - ) -> None: - super().__init__() - if norm_layer is None: - norm_layer = partial(nn.LayerNorm, eps=1e-6) - self.input_layernorm = norm_layer(dim) - self.post_attention_layernorm = norm_layer(dim) - self.self_attn = HunYuanVisionAttention( - embed_dim=dim, - num_heads=num_heads, - projection_size=dim, - quant_config=quant_config, - prefix=f"{prefix}.self_attn", - ) - self.mlp = HunYuanVisionMLP( - dim, - mlp_hidden_dim, - act_fn=act_fn, - bias=True, - quant_config=quant_config, - prefix=f"{prefix}.mlp", - ) - - def forward( - self, - x: torch.Tensor, - ) -> torch.Tensor: - x = x + self.self_attn(self.input_layernorm(x)) - x = x + self.mlp(self.post_attention_layernorm(x)) - return x - - -class HunYuanVisionPatchEmbed(nn.Module): - def __init__(self, config: HunYuanVLVisionConfig): - super().__init__() - - self.config = config - self.embed_dim = config.hidden_size - self.patch_size = config.patch_size - self.num_channels = config.num_channels - self.spatial_merge_size = config.spatial_merge_size - self.interpolate_mode = config.interpolate_mode - - self.patch_embedding = nn.Conv2d( - in_channels=config.num_channels, - out_channels=self.embed_dim, - kernel_size=self.patch_size, - stride=self.patch_size, - bias=True, - ) - - self.max_num_patches = (config.max_image_size // self.patch_size) ** 2 - - self.num_positions = self.max_num_patches + 1 - self.position_edge = int(self.num_positions**0.5) - # first token is cls token, skip it - self.position_embedding = nn.Embedding(self.num_positions, self.embed_dim) - - self.patch_pos_embed = None - - def forward( - self, pixel_values: torch.Tensor, grid_thw: list[list[int]] - ) -> torch.Tensor: - num_patches = pixel_values.size(0) - pixel_values = pixel_values.reshape( - num_patches, self.num_channels, self.patch_size, self.patch_size - ) - - patch_embeds = self.patch_embedding(pixel_values) - patch_embeds = patch_embeds.squeeze(-1).squeeze(-1).unsqueeze(0) - - if self.patch_pos_embed is None: - patch_pos_shape = ( - 1, - self.position_edge, - self.position_edge, - self.embed_dim, - ) - self.patch_pos_embed = ( - self.position_embedding.weight[1:, :] - .reshape(patch_pos_shape) - .permute(0, 3, 1, 2) - .float() - ) - - patch_pos_embed_list = [] - for grid in grid_thw: - _, h0, w0 = grid - # we add a small number to avoid floating point error in the interpolation - # see discussion at https://github.com/facebookresearch/dino/issues/8 - h0, w0 = h0 + 0.1, w0 + 0.1 - patch_pos_embed = nn.functional.interpolate( - self.patch_pos_embed, - scale_factor=(h0 / self.position_edge, w0 / self.position_edge), - mode=self.interpolate_mode, - align_corners=False, - ) - - patch_pos_embed = ( - patch_pos_embed.reshape(self.embed_dim, -1) - .transpose(0, 1) - .unsqueeze(0) - .to(patch_embeds.dtype) - ) - patch_pos_embed_list.append(patch_pos_embed) - - patch_pos_embed = torch.cat(patch_pos_embed_list, dim=1) - embeddings = patch_embeds + patch_pos_embed - - return embeddings - - -class HunYuanVisionPatchMerger(nn.Module): - def __init__( - self, - in_channels, - out_channels, - spatial_merge_size=2, - rms_norm_eps=1e-5, - prefix="", - ): - super().__init__() - self.spatial_merge_size = spatial_merge_size - embed_std = out_channels**-0.5 - - self.proj = nn.Sequential( - nn.Conv2d( - in_channels, - in_channels * 2, - kernel_size=spatial_merge_size, - stride=spatial_merge_size, - ), - nn.GELU(), - nn.Conv2d(in_channels * 2, in_channels * 4, kernel_size=1), - ) - self.mlp = nn.Linear(in_channels * 4, out_channels) - - self.image_newline = nn.Parameter(torch.randn(in_channels * 4) * embed_std) - self.image_begin = nn.Parameter(torch.randn(out_channels) * embed_std) - self.image_end = nn.Parameter(torch.randn(out_channels) * embed_std) - self.image_sep = nn.Parameter(torch.randn(out_channels) * embed_std) - - self.before_rms = RMSNorm(in_channels, eps=rms_norm_eps) - self.after_rms = RMSNorm(out_channels, eps=rms_norm_eps) - - def forward(self, x, size=(16, 16)): - x = self.before_rms(x) - - h, w = size - dtype = x.dtype - x = x.permute(0, 2, 1).reshape(x.shape[0], -1, h, w) - - x = self.proj(x) # b,c,h,w - b, c, h, w = x.shape - x = torch.cat( - [x, self.image_newline.reshape(1, c, 1, 1).expand(b, c, h, 1).to(dtype)], - dim=-1, - ) - x = x.reshape(b, c, -1).permute(0, 2, 1) - x = self.mlp(x) - - begin = self.image_begin.reshape(1, 1, -1).expand(b, 1, x.shape[-1]).to(dtype) - end = self.image_end.reshape(1, 1, -1).expand(b, 1, x.shape[-1]).to(dtype) - x = torch.cat([begin, x, end], dim=1) - - return self.after_rms(x) - - -class HunYuanVisionTransformer(nn.Module): - hf_to_vllm_mapper = WeightsMapper( - orig_to_new_stacked={ - ".q_proj": (".qkv", "q"), - ".k_proj": (".qkv", "k"), - ".v_proj": (".qkv", "v"), - } - ) - - def __init__( - self, - vision_config: HunYuanVLVisionConfig, - quant_config: QuantizationConfig | None = None, - prefix: str = "", - ) -> None: - super().__init__() - - num_hidden_layers = vision_config.num_hidden_layers - self.hidden_size = vision_config.hidden_size - self.num_heads = vision_config.num_attention_heads - self.spatial_merge_size = vision_config.spatial_merge_size - - from vllm.compilation.backends import set_model_tag - - with set_model_tag("HunYuanVisionPatchEmbed"): - self.embeddings = HunYuanVisionPatchEmbed(vision_config) - - norm_layer = partial(nn.LayerNorm, eps=vision_config.rms_norm_eps) - - with set_model_tag("HunYuanVisionBlock"): - self.layers = nn.ModuleList( - [ - HunYuanVisionBlock( - dim=vision_config.hidden_size, - num_heads=vision_config.num_attention_heads, - mlp_hidden_dim=vision_config.intermediate_size, - act_fn=get_act_fn(vision_config.hidden_act), - norm_layer=norm_layer, - quant_config=quant_config, - prefix=f"{prefix}.layers.{layer_idx}", - ) - for layer_idx in range(num_hidden_layers) - ] - ) - - with set_model_tag("HunYuanVisionPatchMerger"): - self.perceive = HunYuanVisionPatchMerger( - vision_config.hidden_size, - vision_config.out_hidden_size, - spatial_merge_size=vision_config.spatial_merge_size, - rms_norm_eps=vision_config.rms_norm_eps, - prefix=f"{prefix}.perceive", - ) - - @property - def dtype(self) -> torch.dtype: - return self.embeddings.patch_embedding.weight.dtype - - @property - def device(self) -> torch.device: - return self.embeddings.patch_embedding.weight.device - - def forward( - self, - x: torch.Tensor, - grid_thw: list[list[int]], - ) -> torch.Tensor: - # patchify - seq_len = x.size(0) - cu_seqlens: list = [0] - - hidden_states = x.to(device=self.device, dtype=self.dtype) - # embeddings = patch_embeds + patch_pos_embed - hidden_states = self.embeddings(hidden_states, grid_thw) - - for t, h, w in grid_thw: - t, h, w = int(t), int(h), int(w) - cu_seqlens.append(h * w) - - cu_seqlens = torch.tensor(cu_seqlens, dtype=torch.int32) - cu_seqlens = torch.cumsum(cu_seqlens, dim=0, dtype=torch.int32) - - cu_seqlens = cu_seqlens.to(device=self.device, non_blocking=True) - - hidden_states = hidden_states.reshape(seq_len, -1) - hidden_states = hidden_states.unsqueeze(0) - - # build per-image lengths once - split_lengths = [int(h) * int(w) for (_, h, w) in grid_thw] - for layer in self.layers: - # hidden_states: (1, T_total, D) - parts = hidden_states.split(split_lengths, dim=1) # list of (1, L_i, D) - parts = [layer(p) for p in parts] - hidden_states = torch.cat(parts, dim=1) - - # adapter - split_lengths = (cu_seqlens[1:] - cu_seqlens[:-1]).tolist() - split_items = hidden_states.split(split_lengths, dim=1) - image_embeds_list = [] - for grid, split_item in zip(grid_thw, split_items): - image_embeds_list.append( - self.perceive(split_item.contiguous(), size=grid[1:]).squeeze(0) - ) - - return image_embeds_list - - def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: - loader = AutoWeightsLoader(self) - return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper) - - -def _hunyuan_vl_field_config(hf_inputs: Mapping[str, torch.Tensor]): - image_grid_thw = hf_inputs.get("image_grid_thw", torch.empty((0, 3))) - image_grid_sizes = image_grid_thw.prod(-1) - return dict( - pixel_values=MultiModalFieldConfig.flat_from_sizes("image", image_grid_sizes), - image_embeds=MultiModalFieldConfig.flat_from_sizes("image", image_grid_sizes), - image_grid_thw=MultiModalFieldConfig.batched("image", keep_on_cpu=True), - ) - - -class HunYuanVLMultiModalDataParser(MultiModalDataParser): - # The patch grid is what sizes the placeholder range. - embedding_fields = { - "image": {"image_embeds": "values", "image_grid_thw": "metadata"}, - } - - def _parse_image_data( - self, - data: dict[str, torch.Tensor] | ModalityData[ImageItem], - ) -> ModalityDataItems[Any, Any] | None: - if isinstance(data, dict): - required, optional = self.embedding_field_sets("image") - return DictEmbeddingItems( - data, - modality="image", - required_fields=required, - optional_fields=optional, - fields_factory=_hunyuan_vl_field_config, - ) - - return super()._parse_image_data(data) - - -class HunYuanVLProcessingInfo(BaseProcessingInfo): - def get_hf_config(self): - return self.ctx.get_hf_config(HunYuanVLConfig) - - def get_hf_processor( - self, - **kwargs: object, - ) -> HunYuanVLProcessor: - # transformers>=5.13 replaced `use_fast` with `backend`; pin the - # PIL backend to match the released HunyuanOCR checkpoint packing. - kwargs.pop("use_fast", None) - kwargs.setdefault("backend", "pil") - return self.ctx.get_hf_processor( - HunYuanVLProcessor, - **kwargs, - ) - - def get_image_processor( - self, - **kwargs: object, - ) -> HunYuanVLImageProcessor: - return self.get_hf_processor(**kwargs).image_processor - - def get_data_parser(self): - return HunYuanVLMultiModalDataParser( - expected_hidden_size=self._get_expected_hidden_size(), - allow_missing_mm_embeddings=self.allow_missing_mm_embeddings, - ) - - def get_supported_mm_limits(self) -> Mapping[str, int | None]: - return {"image": None} - - def get_mm_max_tokens_per_item( - self, - seq_len: int, - mm_counts: Mapping[str, int], - ) -> Mapping[str, int]: - max_image_tokens = self.get_max_image_tokens() - # TODO: support video - max_video_tokens = 0 - return {"image": max_image_tokens, "video": max_video_tokens} - - def _get_vision_info( - self, - *, - image_width: int, - image_height: int, - num_frames: int = 1, - do_resize: bool = True, - image_processor: HunYuanVLImageProcessor, - mm_kwargs: Mapping[str, object], - ) -> tuple[ImageSize, int]: - hf_config = self.get_hf_config() - vision_config = hf_config.vision_config - patch_size = vision_config.patch_size - spatial_merge_size = vision_config.spatial_merge_size - - mm_kwargs = self.ctx.get_merged_mm_kwargs(mm_kwargs) - size = image_processor.size - if override_size := mm_kwargs.get("size"): - size = size | override_size - if (override_min_pixels := mm_kwargs.get("min_pixels")) is not None: - size = size | {"shortest_edge": override_min_pixels} - if (override_max_pixels := mm_kwargs.get("max_pixels")) is not None: - size = size | {"longest_edge": override_max_pixels} - - if do_resize: - resized_height, resized_width = smart_resize( - height=image_height, - width=image_width, - factor=patch_size * spatial_merge_size, - min_pixels=size["shortest_edge"], - max_pixels=size["longest_edge"], - ) - preprocessed_size = ImageSize(width=resized_width, height=resized_height) - else: - preprocessed_size = ImageSize(width=image_width, height=image_height) - - grid_t = 1 - grid_h = preprocessed_size.height // patch_size - grid_w = preprocessed_size.width // patch_size - - num_vision_tokens = ( - grid_t * grid_h // spatial_merge_size * (grid_w // spatial_merge_size + 1) - + 2 - ) - - return preprocessed_size, num_vision_tokens - - def get_num_image_tokens( - self, - *, - image_width: int, - image_height: int, - image_processor: HunYuanVLImageProcessor, - mm_kwargs: Mapping[str, object], - ) -> int: - _, num_image_tokens = self._get_vision_info( - image_width=image_width, - image_height=image_height, - image_processor=image_processor, - mm_kwargs=mm_kwargs, - ) - return num_image_tokens - - def get_image_size_with_most_features(self) -> ImageSize: - image_processor = self.get_image_processor() - - max_image_size, _ = self._get_vision_info( - image_width=512, - image_height=8192, - image_processor=image_processor, - mm_kwargs={}, - ) - return max_image_size - - def get_max_image_tokens(self) -> int: - image_processor = self.get_image_processor() - target_width, target_height = self.get_image_size_with_most_features() - - return self.get_num_image_tokens( - image_width=target_width, - image_height=target_height, - image_processor=image_processor, - mm_kwargs={}, - ) - - -class HunYuanVLDummyInputsBuilder(BaseDummyInputsBuilder[HunYuanVLProcessingInfo]): - def get_dummy_text(self, mm_counts: Mapping[str, int]) -> str: - num_images = mm_counts.get("image", 0) - - hf_processor = self.info.get_hf_processor(typ=HunYuanVLProcessor) - image_placeholder = ( - f"{hf_processor.image_start_token}{hf_processor.image_token}" - f"{hf_processor.image_end_token}" - ) - - return image_placeholder * num_images - - def get_dummy_mm_data( - self, - seq_len: int, - mm_counts: Mapping[str, int], - mm_options: Mapping[str, BaseDummyOptions], - ) -> MultiModalDataDict: - num_images = mm_counts.get("image", 1) - - target_width, target_height = self.info.get_image_size_with_most_features() - - return { - "image": self._get_dummy_images( - width=target_width, height=target_height, num_images=num_images - ), - } - - -class HunYuanVLMultiModalProcessor(BaseMultiModalProcessor[HunYuanVLProcessingInfo]): - def _get_prompt_updates( - self, - mm_items: MultiModalDataItems, - hf_processor_mm_kwargs: Mapping[str, Any], - out_mm_kwargs: MultiModalKwargsItems, - ) -> Sequence[PromptUpdate]: - hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs) - image_processor = self.info.get_image_processor(**hf_processor_mm_kwargs) - - token_ids = { - "image": hf_processor.image_token_id, - "image_start": hf_processor.image_start_token_id, - "image_end": hf_processor.image_end_token_id, - } - - merge_size = image_processor.merge_size - - def get_replacement_hunyuan_vl(item_idx: int, modality: str): - out_item = out_mm_kwargs[modality][item_idx] - grid_thw = out_item[f"{modality}_grid_thw"].data - assert isinstance(grid_thw, torch.Tensor) - - _, grid_h, grid_w = grid_thw - num_tokens = (int(grid_h) // merge_size) * ( - int(grid_w) // merge_size + 1 - ) + 2 - tokens = ( - [token_ids[f"{modality}_start"]] - + [token_ids[modality]] * num_tokens - + [token_ids[f"{modality}_end"]] - ) - return PromptUpdateDetails.select_token_id(tokens, token_ids[modality]) - - return [ - PromptReplacement( - modality=modality, - target=[ - token_ids[f"{modality}_start"], - token_ids[modality], - token_ids[f"{modality}_end"], - ], - replacement=partial(get_replacement_hunyuan_vl, modality=modality), - ) - for modality in ("image",) - ] - - def _get_mm_fields_config( - self, - hf_inputs: BatchFeature, - hf_processor_mm_kwargs: Mapping[str, object], - ) -> Mapping[str, MultiModalFieldConfig]: - return _hunyuan_vl_field_config(hf_inputs) - - -@MULTIMODAL_REGISTRY.register_processor( - HunYuanVLMultiModalProcessor, - info=HunYuanVLProcessingInfo, - dummy_inputs=HunYuanVLDummyInputsBuilder, -) -class HunYuanVLForConditionalGeneration( - nn.Module, - SupportsMultiModal, - SupportsLoRA, - SupportsPP, - SupportsQuant, - SupportsXDRoPE, - SupportsEagle, - SupportsEagle3, -): - # To ensure correct weight loading and mapping. - hf_to_vllm_mapper = WeightsMapper( - orig_to_new_prefix={ - # mapping for new names in checkpoint saved after transformers v4.52 - "vit.vit.": "visual.", - "vit.": "visual.", - "model.": "language_model.model.", - } - ) - - supports_encoder_tp_data = True - - def get_xdrope_input_positions( - self, - input_tokens: list[int], - mm_features: list[MultiModalFeatureSpec], - ) -> torch.Tensor: - kwargs = MultiModalFeatureSpec.gather_kwargs( - mm_features, - {"image_grid_thw"}, - ) - image_grid_thw = [item.tolist() for item in kwargs.get("image_grid_thw", [])] - - hf_config = self.config - image_start_token_id = hf_config.image_start_token_id - spatial_merge_size = hf_config.vision_config.spatial_merge_size - xd_num = len(hf_config.rope_scaling["xdrope_section"]) - - input_tokens_tensor = torch.tensor(input_tokens) - image_start_indices = torch.argwhere( - input_tokens_tensor == image_start_token_id - ).squeeze(1) - - p_index = torch.arange(len(input_tokens_tensor)) - w_index = torch.arange(len(input_tokens_tensor)) - h_index = torch.arange(len(input_tokens_tensor)) - t_index = torch.arange(len(input_tokens_tensor)) - for image_index in range(len(image_start_indices)): - # +1 : first image_token, +2: for xdrope positions - pos = image_start_indices[image_index] + 2 - t, h, w = image_grid_thw[image_index] - _, llm_grid_h, llm_grid_w = ( - t, - h // spatial_merge_size, - w // spatial_merge_size, - ) - - token_num = (llm_grid_w + 1) * llm_grid_h - w_index[pos : pos + token_num].copy_( - torch.arange(0, llm_grid_w + 1) - .reshape(1, -1) - .expand(llm_grid_h, -1) - .reshape(-1) - ) - h_index[pos : pos + token_num].copy_( - torch.arange(0, llm_grid_h) - .reshape(-1, 1) - .expand(-1, llm_grid_w + 1) - .reshape(-1) - ) - t_index[pos : pos + token_num] = image_index - - if xd_num == 4: - llm_positions = torch.stack([p_index, w_index, h_index, t_index]) - elif xd_num == 3: - llm_positions = torch.stack([w_index, h_index, t_index]) - - return llm_positions - - @classmethod - def get_placeholder_str(cls, modality: str, i: int) -> str | None: - if modality.startswith("image"): - return "<|hy_place▁holder▁no▁100|><|hy_place▁holder▁no▁102|><|hy_place▁holder▁no▁101|>" # noqa: E501 - - raise ValueError("Only image modality is supported") - - def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""): - super().__init__() - config: HunYuanVLConfig = vllm_config.model_config.hf_config - - self.config = config - - with self._mark_tower_model(vllm_config, {"image"}): - self.visual = HunYuanVisionTransformer( - config.vision_config, - quant_config=vllm_config.quant_config, - prefix=maybe_prefix(prefix, "visual"), - ) - - with self._mark_language_model(vllm_config): - self.language_model = init_vllm_registered_model( - vllm_config=vllm_config, - prefix=maybe_prefix(prefix, "language_model.model"), - architectures=[ - "HunYuanDenseV1ForCausalLM", - "HunYuanMoEV1ForCausalLM", - ], - ) - - self.make_empty_intermediate_tensors = ( - self.language_model.make_empty_intermediate_tensors - ) - - def _parse_and_validate_image_input( - self, **kwargs: object - ) -> HunYuanVLImageInputs | None: - pixel_values = kwargs.pop("pixel_values", None) - image_embeds = kwargs.pop("image_embeds", None) - image_grid_thw = kwargs.pop("image_grid_thw", None) - - if pixel_values is None and image_embeds is None: - return None - - # TODO: refine - if isinstance(pixel_values, list): - pixel_values = torch.cat(pixel_values, dim=0) - if len(pixel_values.shape) == 3: - last_dim = pixel_values.shape[-1] - pixel_values = pixel_values.reshape(-1, last_dim) - image_grid_thw = image_grid_thw.reshape(-1, 3) - - if pixel_values is not None: - return HunYuanVLImagePixelInputs( - type="pixel_values", - pixel_values=pixel_values, - image_grid_thw=image_grid_thw, - ) - - if image_embeds is not None: - return HunYuanVLImageEmbeddingInputs( - type="image_embeds", - image_embeds=image_embeds, - image_grid_thw=image_grid_thw, - ) - - def _process_image_input( - self, image_input: HunYuanVLImageInputs - ) -> tuple[torch.Tensor, ...]: - grid_thw = image_input["image_grid_thw"] - assert grid_thw.ndim == 2 - grid_thw_list = grid_thw.tolist() - - if image_input["type"] == "image_embeds": - image_embeds = image_input["image_embeds"].type(self.visual.dtype) - else: - pixel_values = image_input["pixel_values"] - - # TODO: use_data_parallel (split image_embeds in visual) - image_embeds = self.visual(pixel_values, grid_thw=grid_thw_list) - - return image_embeds - - def _parse_and_validate_multimodal_inputs(self, **kwargs: object) -> dict: - mm_input_by_modality = {} - - # Preserve the order of modalities if there are multiple of them - # from the order of kwargs. - for input_key in kwargs: - if ( - input_key in ("pixel_values", "image_embeds") - and "image" not in mm_input_by_modality - ): - mm_input_by_modality["image"] = self._parse_and_validate_image_input( - **kwargs - ) - return mm_input_by_modality - - def embed_multimodal(self, **kwargs: object) -> MultiModalEmbeddings: - mm_input_by_modality = self._parse_and_validate_multimodal_inputs(**kwargs) - if not mm_input_by_modality: - return [] - - # The result multimodal_embeddings is tuple of tensors, with each - # tensor correspoending to a multimodal data item (image or video). - multimodal_embeddings: tuple[torch.Tensor, ...] = () - - # NOTE: It is important to iterate over the keys in this dictionary - # to preserve the order of the modalities. - for modality in mm_input_by_modality: - multimodal_input = mm_input_by_modality[modality] - if modality == "image": - image_embeddings = self._process_image_input(multimodal_input) - multimodal_embeddings += tuple(image_embeddings) - return multimodal_embeddings - - def forward( - self, - input_ids: torch.Tensor | None, - positions: torch.Tensor, - intermediate_tensors: IntermediateTensors | None, - inputs_embeds: torch.Tensor | None, - **kwargs: object, - ) -> torch.Tensor | IntermediateTensors: - if intermediate_tensors is not None: - inputs_embeds = None - - hidden_states = self.language_model( - input_ids=input_ids, - positions=positions, - intermediate_tensors=intermediate_tensors, - inputs_embeds=inputs_embeds, - ) - return hidden_states - - def compute_logits( - self, - hidden_states: torch.Tensor, - ) -> torch.Tensor | None: - return self.language_model.compute_logits(hidden_states) - - def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: - loader = AutoWeightsLoader(self) - return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper) - - def get_mm_mapping(self) -> MultiModelKeys: - """ - Get the module prefix in multimodal models - """ - return MultiModelKeys.from_string_field( - language_model="language_model.model", - connector="visual.perceive", - tower_model="visual", - ) diff --git a/vllm/model_executor/models/olmo3.py b/vllm/model_executor/models/olmo3.py deleted file mode 100644 index a54dd5749e..0000000000 --- a/vllm/model_executor/models/olmo3.py +++ /dev/null @@ -1,412 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project - -# Adapted from -# https://github.com/huggingface/transformers/blob/main/src/transformers/models/olmo3/modeling_olmo3.py -# Copyright 2024 The vLLM team. -# Copyright 2024 EleutherAI and the HuggingFace Inc. team. All rights reserved. -# -# This code is based on EleutherAI's GPT-NeoX library and the GPT-NeoX -# and OPT implementations in this library. It has been modified from its -# original forms to accommodate minor architectural differences compared -# to GPT-NeoX and OPT used by the Meta AI team that trained the model. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Inference-only OLMo3 model compatible with HuggingFace weights.""" - -from collections.abc import Iterable -from functools import partial -from itertools import islice - -import torch -from torch import nn -from transformers import Olmo3Config - -from vllm.compilation.decorators import support_torch_compile -from vllm.config import VllmConfig -from vllm.distributed import get_pp_group, get_tensor_model_parallel_world_size -from vllm.distributed.communication_op import tensor_model_parallel_all_gather -from vllm.distributed.parallel_state import get_tensor_model_parallel_rank -from vllm.distributed.utils import split_tensor_along_last_dim -from vllm.model_executor.layers.activation import SiluAndMul -from vllm.model_executor.layers.attention import Attention -from vllm.model_executor.layers.layernorm import RMSNorm -from vllm.model_executor.layers.linear import ( - MergedColumnParallelLinear, - QKVParallelLinear, - RowParallelLinear, -) -from vllm.model_executor.layers.logits_processor import LogitsProcessor -from vllm.model_executor.layers.rotary_embedding import get_rope -from vllm.model_executor.layers.vocab_parallel_embedding import ( - ParallelLMHead, - VocabParallelEmbedding, -) -from vllm.model_executor.models.interfaces import SupportsLoRA, SupportsPP -from vllm.model_executor.models.utils import ( - AutoWeightsLoader, - WeightsMapper, - extract_layer_index, - make_empty_intermediate_tensors_factory, - make_layers, - maybe_prefix, -) -from vllm.sequence import IntermediateTensors - - -class Olmo3Attention(nn.Module): - """ - This is the attention block where the output is computed as - `Attention(LN(x))` in `MLP(LN(x + Attention(LN(x))))` - (plus another skip connection). - """ - - def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""): - super().__init__() - self.config = vllm_config.model_config.hf_config - assert isinstance(self.config, Olmo3Config) - - hidden_size = self.config.hidden_size - self.tp_size = get_tensor_model_parallel_world_size() - self.total_num_heads = self.config.num_attention_heads - - assert hidden_size % self.total_num_heads == 0 - assert self.total_num_heads % self.tp_size == 0 - - self.num_heads = self.total_num_heads // self.tp_size - self.total_num_kv_heads = ( - self.config.num_key_value_heads or self.total_num_heads - ) - if self.total_num_kv_heads >= self.tp_size: - assert self.total_num_kv_heads % self.tp_size == 0 - else: - assert self.tp_size % self.total_num_kv_heads == 0 - - self.num_kv_heads = max(1, self.total_num_kv_heads // self.tp_size) - self.head_dim = hidden_size // self.total_num_heads - self.q_size = self.num_heads * self.head_dim - self.kv_size = self.num_kv_heads * self.head_dim - self.max_position_embeddings = self.config.max_position_embeddings - - # Attention input projection. Projects x -> (q, k, v) - self.qkv_proj = QKVParallelLinear( - hidden_size, - self.head_dim, - self.total_num_heads, - self.total_num_kv_heads, - bias=False, - quant_config=vllm_config.quant_config, - prefix=f"{prefix}.qkv_proj", - ) - - self.tp_rank = get_tensor_model_parallel_rank() - self.k_norm = RMSNorm( - self.total_num_kv_heads * self.head_dim, - eps=self.config.rms_norm_eps, - ) - self.q_norm = RMSNorm(self.config.hidden_size, eps=self.config.rms_norm_eps) - - self.scaling = self.head_dim**-0.5 - - layer_idx = extract_layer_index(prefix) - sliding_window = None - if ( - layer_types := getattr(self.config, "layer_types", None) - ) is not None and layer_types[layer_idx] == "sliding_attention": - sliding_window = self.config.sliding_window - - self.attn = Attention( - self.num_heads, - self.head_dim, - self.scaling, - num_kv_heads=self.num_kv_heads, - cache_config=vllm_config.cache_config, - quant_config=vllm_config.quant_config, - per_layer_sliding_window=sliding_window, - prefix=f"{prefix}.attn", - ) - - # Rotary embeddings. Rope scaling is only applied on full attention layers. - rope_parameters = self.config.rope_parameters - attn_type = "full_attention" if sliding_window is None else "sliding_attention" - rope_parameters = rope_parameters.get(attn_type, rope_parameters) - self.rotary_emb = get_rope( - self.head_dim, - max_position=self.max_position_embeddings, - rope_parameters=rope_parameters, - ) - - # Attention output projection. - self.o_proj = RowParallelLinear( - self.total_num_heads * self.head_dim, - hidden_size, - bias=False, - quant_config=vllm_config.quant_config, - prefix=f"{prefix}.o_proj", - ) - - def _apply_qk_norm( - self, q: torch.Tensor, k: torch.Tensor - ) -> tuple[torch.Tensor, torch.Tensor]: - if self.tp_size > 1: - q = tensor_model_parallel_all_gather(q.contiguous()) - k = tensor_model_parallel_all_gather(k.contiguous()) - q = self.q_norm(q) - k = self.k_norm(k) - if self.tp_size > 1: - splitter = partial(split_tensor_along_last_dim, num_partitions=self.tp_size) - q = splitter(q)[self.tp_rank] - k = splitter(k)[self.tp_rank] - return q, k - - def forward( - self, - positions: torch.Tensor, - hidden_states: torch.Tensor, - ) -> torch.Tensor: - qkv, _ = self.qkv_proj(hidden_states) - q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1) - q, k = self._apply_qk_norm(q, k) - q, k = self.rotary_emb(positions, q, k) - attn_output = self.attn(q, k, v) - output, _ = self.o_proj(attn_output) - return output - - -class Olmo3MLP(nn.Module): - """ - This is the MLP block where the output is computed as - `MLP(x)` in `LN(MLP(x + LN(Attention(x))))` - (plus another skip connection). - """ - - def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""): - super().__init__() - config = vllm_config.model_config.hf_config - assert isinstance(config, Olmo3Config) - hidden_size = config.hidden_size - intermediate_size = config.intermediate_size - - # Feed-forward input projection. - self.gate_up_proj = MergedColumnParallelLinear( - hidden_size, - [intermediate_size] * 2, - bias=False, - quant_config=vllm_config.quant_config, - prefix=f"{prefix}.gate_up_proj", - ) - - # Activation function. - self.act_fn = SiluAndMul() - - # Feed-forward output projection. - self.down_proj = RowParallelLinear( - intermediate_size, - hidden_size, - bias=False, - quant_config=vllm_config.quant_config, - prefix=f"{prefix}.down_proj", - ) - - def forward( - self, - x: torch.Tensor, - ) -> torch.Tensor: - gate_up, _ = self.gate_up_proj(x) - x = self.act_fn(gate_up) - x, _ = self.down_proj(x) - return x - - -class Olmo3DecoderLayer(nn.Module): - """ - This is a typical transformer block where the output is - computed as `MLP(LN(x + Attention(LN(x))))` - (plus another skip connection). - """ - - def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""): - super().__init__() - config = vllm_config.model_config.hf_config - assert isinstance(config, Olmo3Config) - # Attention block. - self.self_attn = Olmo3Attention( - vllm_config=vllm_config, prefix=f"{prefix}.self_attn" - ) - - # MLP block. - self.mlp = Olmo3MLP(vllm_config=vllm_config, prefix=f"{prefix}.mlp") - - # LayerNorm - self.post_attention_layernorm = RMSNorm( - config.hidden_size, eps=config.rms_norm_eps - ) - - self.post_feedforward_layernorm = RMSNorm( - config.hidden_size, eps=config.rms_norm_eps - ) - - def forward( - self, - positions: torch.Tensor, - hidden_states: torch.Tensor, - ) -> torch.Tensor: - # Attention block. - residual = hidden_states - hidden_states = self.self_attn(positions, hidden_states) - hidden_states = self.post_attention_layernorm(hidden_states) - hidden_states = hidden_states + residual - - # MLP block. - residual = hidden_states - hidden_states = self.mlp(hidden_states) - hidden_states = self.post_feedforward_layernorm(hidden_states) - hidden_states = residual + hidden_states - return hidden_states - - -@support_torch_compile -class Olmo3Model(nn.Module): - def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""): - super().__init__() - self.config = vllm_config.model_config.hf_config - assert isinstance(self.config, Olmo3Config) - - self.embed_tokens = VocabParallelEmbedding( - self.config.vocab_size, - self.config.hidden_size, - prefix=f"{prefix}.embed_tokens", - ) - self.start_layer, self.end_layer, self.layers = make_layers( - self.config.num_hidden_layers, - lambda prefix: Olmo3DecoderLayer(vllm_config=vllm_config, prefix=prefix), - prefix=f"{prefix}.layers", - ) - self.norm = RMSNorm( - self.config.hidden_size, - eps=self.config.rms_norm_eps, - ) - self.make_empty_intermediate_tensors = make_empty_intermediate_tensors_factory( - ["hidden_states"], self.config.hidden_size - ) - - def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor: - return self.embed_tokens(input_ids) - - def forward( - self, - input_ids: torch.Tensor | None, - positions: torch.Tensor, - intermediate_tensors: IntermediateTensors | None, - inputs_embeds: torch.Tensor | None = None, - ) -> torch.Tensor | IntermediateTensors: - """ - Args: - input_ids: A tensor of shape `(batch_size, seq_len)`. - """ - if get_pp_group().is_first_rank: - if inputs_embeds is not None: - hidden_states = inputs_embeds - # Get embeddings of input. - # shape: (batch_size, seq_len, d_model) - else: - hidden_states = self.embed_tokens(input_ids) - - else: - assert intermediate_tensors is not None - hidden_states = intermediate_tensors["hidden_states"] - assert isinstance(hidden_states, torch.Tensor) - - # Apply blocks one-by-one. - for layer in islice(self.layers, self.start_layer, self.end_layer): - # shape: (batch_size, seq_len, d_model) - hidden_states = layer(positions, hidden_states) - - if not get_pp_group().is_last_rank: - return IntermediateTensors({"hidden_states": hidden_states}) - - # Apply final layer norm. - # shape: (batch_size, seq_len or 1, d_model) - hidden_states = self.norm(hidden_states) - return hidden_states - - -class Olmo3ForCausalLM(nn.Module, SupportsPP, SupportsLoRA): - """ - Extremely barebones HF model wrapper. - """ - - hf_to_vllm_mapper = WeightsMapper( - orig_to_new_stacked={ - # weight_name: (param_name, shard_id) - ".q_proj": (".qkv_proj", "q"), - ".k_proj": (".qkv_proj", "k"), - ".v_proj": (".qkv_proj", "v"), - ".gate_proj": (".gate_up_proj", 0), - ".up_proj": (".gate_up_proj", 1), - } - ) - packed_modules_mapping = { - "qkv_proj": ["q_proj", "k_proj", "v_proj"], - "gate_up_proj": ["gate_proj", "up_proj"], - } - - def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""): - super().__init__() - config = vllm_config.model_config.hf_config - assert isinstance(config, Olmo3Config) - self.config = config - self.model = Olmo3Model( - vllm_config=vllm_config, prefix=maybe_prefix(prefix, "model") - ) - self.lm_head = ParallelLMHead( - config.vocab_size, - config.hidden_size, - quant_config=vllm_config.quant_config, - prefix=maybe_prefix(prefix, "lm_head"), - ) - if config.tie_word_embeddings: - self.lm_head = self.lm_head.tie_weights(self.model.embed_tokens) - self.logits_processor = LogitsProcessor(config.vocab_size) - self.make_empty_intermediate_tensors = ( - self.model.make_empty_intermediate_tensors - ) - - def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor: - return self.model.embed_input_ids(input_ids) - - def forward( - self, - input_ids: torch.Tensor | None, - positions: torch.Tensor, - intermediate_tensors: IntermediateTensors | None = None, - inputs_embeds: torch.Tensor | None = None, - ) -> torch.Tensor | IntermediateTensors: - hidden_states = self.model( - input_ids=input_ids, - positions=positions, - intermediate_tensors=intermediate_tensors, - inputs_embeds=inputs_embeds, - ) - return hidden_states - - def compute_logits( - self, - hidden_states: torch.Tensor, - ) -> torch.Tensor | None: - logits = self.logits_processor(self.lm_head, hidden_states) - return logits - - def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]): - loader = AutoWeightsLoader(self) - return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper) diff --git a/vllm/model_executor/models/registry.py b/vllm/model_executor/models/registry.py index 8c3bf4c07b..cf1361bf40 100644 --- a/vllm/model_executor/models/registry.py +++ b/vllm/model_executor/models/registry.py @@ -104,7 +104,6 @@ _TEXT_GENERATION_MODELS = { "FalconForCausalLM": ("falcon", "FalconForCausalLM"), "FalconMambaForCausalLM": ("mamba", "MambaForCausalLM"), "FalconH1ForCausalLM": ("falcon_h1", "FalconH1ForCausalLM"), - "FlexOlmoForCausalLM": ("flex_olmo", "FlexOlmoForCausalLM"), "GemmaForCausalLM": ("gemma", "GemmaForCausalLM"), "Gemma2ForCausalLM": ("gemma2", "Gemma2ForCausalLM"), "Gemma3ForCausalLM": ("gemma3", "Gemma3ForCausalLM"), @@ -128,8 +127,6 @@ _TEXT_GENERATION_MODELS = { "GraniteMoeSWAForCausalLM": ("granitemoeshared", "GraniteMoeSharedForCausalLM"), "GraniteSWAForCausalLM": ("granite", "GraniteForCausalLM"), "HrmTextForCausalLM": ("hrm_text", "HrmTextForCausalLM"), - "HunYuanMoEV1ForCausalLM": ("hunyuan_v1", "HunYuanMoEV1ForCausalLM"), - "HunYuanDenseV1ForCausalLM": ("hunyuan_v1", "HunYuanDenseV1ForCausalLM"), "HYV3ForCausalLM": ("hy_v3", "HYV3ForCausalLM"), "HCXVisionV2ForCausalLM": ("hyperclovax_vision_v2", "HCXVisionV2ForCausalLM"), "HyperCLOVAXForCausalLM": ("hyperclovax", "HyperCLOVAXForCausalLM"), @@ -180,7 +177,6 @@ _TEXT_GENERATION_MODELS = { "NemotronForCausalLM": ("nemotron", "NemotronForCausalLM"), "NemotronHForCausalLM": ("nemotron_h", "NemotronHForCausalLM"), "NemotronHPuzzleForCausalLM": ("nemotron_h", "NemotronHForCausalLM"), - "Olmo3ForCausalLM": ("olmo3", "Olmo3ForCausalLM"), "MuseGlimmerForCausalLM": ("muse_glimmer", "MuseGlimmerForCausalLM"), "OlmoHybridForCausalLM": ("olmo_hybrid", "OlmoHybridForCausalLM"), "OlmoeForCausalLM": ("olmoe", "OlmoeForCausalLM"), @@ -420,10 +416,6 @@ _MULTIMODAL_MODELS = { "Granite4VisionForConditionalGeneration", ), "H2OVLChatModel": ("h2ovl", "H2OVLChatModel"), - "HunYuanVLForConditionalGeneration": ( - "hunyuan_vision", - "HunYuanVLForConditionalGeneration", - ), "InternVLChatModel": ("internvl", "InternVLChatModel"), "InternS1ForConditionalGeneration": ( "interns1", @@ -679,9 +671,13 @@ _SPECULATIVE_DECODING_MODELS = { _TRANSFORMERS_SUPPORTED_MODELS = { # Text generation models + "FlexOlmoForCausalLM": ("transformers", "TransformersMoEForCausalLM"), "GPTBigCodeForCausalLM": ("transformers", "TransformersForCausalLM"), + "HunYuanDenseV1ForCausalLM": ("transformers", "TransformersForCausalLM"), + "HunYuanMoEV1ForCausalLM": ("transformers", "TransformersMoEForCausalLM"), "OlmoForCausalLM": ("transformers", "TransformersForCausalLM"), "Olmo2ForCausalLM": ("transformers", "TransformersForCausalLM"), + "Olmo3ForCausalLM": ("transformers", "TransformersForCausalLM"), "SmolLM3ForCausalLM": ("transformers", "TransformersForCausalLM"), "Starcoder2ForCausalLM": ("transformers", "TransformersForCausalLM"), "VaultGemmaForCausalLM": ("transformers", "TransformersForCausalLM"), @@ -690,6 +686,10 @@ _TRANSFORMERS_SUPPORTED_MODELS = { "transformers", "TransformersMultiModalForCausalLM", ), + "HunYuanVLForConditionalGeneration": ( + "transformers", + "TransformersMultiModalForCausalLM", + ), "VibeVoiceAsrForConditionalGeneration": ( "transformers", "TransformersMultiModalForCausalLM", diff --git a/vllm/model_executor/models/transformers/moe.py b/vllm/model_executor/models/transformers/moe.py index e1c3733a0f..d59fb2b258 100644 --- a/vllm/model_executor/models/transformers/moe.py +++ b/vllm/model_executor/models/transformers/moe.py @@ -37,10 +37,10 @@ from vllm.model_executor.layers.fused_moe import ( from vllm.model_executor.models.interfaces import MixtureOfExperts from vllm.model_executor.models.transformers.fuser import get_fuser from vllm.model_executor.models.transformers.fusers.moe import MoEBlockFuser -from vllm.model_executor.models.utils import maybe_prefix +from vllm.model_executor.models.utils import extract_layer_index, maybe_prefix from vllm.utils.torch_utils import STR_DTYPE_TO_TORCH_DTYPE, direct_register_custom_op -from .utils import log_replacement +from .utils import log_replacement, maybe_per_layer if TYPE_CHECKING: from vllm.config import VllmConfig @@ -183,7 +183,7 @@ class MoEMixin(MixtureOfExperts): ) # Common kwargs - renormalize = getattr(text_config, "norm_topk_prob", top_k > 1) + norm_topk_prob = getattr(text_config, "norm_topk_prob", None) # Routed scaling factor kwargs routed_scaling_factor = getattr(text_config, "routed_scaling_factor", 1.0) @@ -266,12 +266,25 @@ class MoEMixin(MixtureOfExperts): self.num_shared_experts = 1 break + # Only pay for the layer index if something is per-layer; + # `extract_layer_index` raises for unnumbered module paths. + per_layer = (top_k, intermediate_size, norm_topk_prob) + layer_idx = ( + extract_layer_index(qual_name) + if any(isinstance(v, list) for v in per_layer) + else 0 + ) + layer_top_k = maybe_per_layer(top_k, layer_idx) + layer_renormalize = maybe_per_layer(norm_topk_prob, layer_idx) + if layer_renormalize is None: + layer_renormalize = layer_top_k > 1 + kwargs: dict[str, Any] = dict( num_experts=num_experts, - top_k=top_k, + top_k=layer_top_k, hidden_size=hidden_size, - intermediate_size=intermediate_size, - renormalize=renormalize, + intermediate_size=maybe_per_layer(intermediate_size, layer_idx), + renormalize=layer_renormalize, quant_config=self.quant_config, prefix=qual_name, activation=activation, diff --git a/vllm/model_executor/models/transformers/multimodal.py b/vllm/model_executor/models/transformers/multimodal.py index e0911d8e2c..f4206a3e14 100644 --- a/vllm/model_executor/models/transformers/multimodal.py +++ b/vllm/model_executor/models/transformers/multimodal.py @@ -173,6 +173,11 @@ class MultiModalDummyInputsBuilder(BaseDummyInputsBuilder[MultiModalProcessingIn image_token = processor.boi_token else: image_token = getattr(processor, "image_token", "") + # Some processors (e.g. HunYuanVL) reject a bare image token and + # require each one to be wrapped in its start/end markers. + start_token = getattr(processor, "image_start_token", "") + end_token = getattr(processor, "image_end_token", "") + image_token = f"{start_token}{image_token}{end_token}" text += image_token * num_images return text diff --git a/vllm/model_executor/models/transformers/utils.py b/vllm/model_executor/models/transformers/utils.py index 0d5d5f9a06..ed6c47a86e 100644 --- a/vllm/model_executor/models/transformers/utils.py +++ b/vllm/model_executor/models/transformers/utils.py @@ -22,7 +22,7 @@ from functools import lru_cache from itertools import chain from operator import attrgetter from pathlib import Path -from typing import TYPE_CHECKING, Literal +from typing import TYPE_CHECKING, Literal, TypeVar import torch from torch import nn @@ -306,6 +306,14 @@ def recursive_replace_linear( _recursive_replace(model, prefix=prefix) +T = TypeVar("T") + + +def maybe_per_layer(value: T | list[T], layer_idx: int) -> T: + """Pick `layer_idx`'s entry from a config field that may be sized per layer.""" + return value[layer_idx] if isinstance(value, list) else value + + def named_state(module: nn.Module) -> Iterator[tuple[str, torch.Tensor]]: """`module`'s own state (i.e. named parameters and buffers).""" return chain(module.named_parameters(), module.named_buffers()) diff --git a/vllm/transformers_utils/config.py b/vllm/transformers_utils/config.py index 65b6b133a9..aa11ded482 100644 --- a/vllm/transformers_utils/config.py +++ b/vllm/transformers_utils/config.py @@ -87,11 +87,9 @@ _CONFIG_REGISTRY: dict[str, type[PretrainedConfig]] = LazyConfigDict( deepseek_v4="DeepseekV4Config", dots3_note="Dots3NoteConfig", k3_dspark="K3DSparkConfig", - flex_olmo="FlexOlmoConfig", funaudiochat="FunAudioChatConfig", granite4_vision="Granite4VisionConfig", hyperclovax="HyperCLOVAXConfig", - hunyuan_vl="HunYuanVLConfig", hy_v3="HYV3Config", isaac="IsaacConfig", kimi_k2="DeepseekV3Config", # Kimi K2 uses same architecture as DeepSeek V3 diff --git a/vllm/transformers_utils/configs/__init__.py b/vllm/transformers_utils/configs/__init__.py index 625f0636b8..776eb642bd 100644 --- a/vllm/transformers_utils/configs/__init__.py +++ b/vllm/transformers_utils/configs/__init__.py @@ -37,13 +37,9 @@ _CLASS_TO_MODULE: dict[str, str] = { "K3DSparkConfig": "vllm.transformers_utils.configs.k3_dspark", "DotsOCRConfig": "vllm.transformers_utils.configs.dotsocr", "EAGLEConfig": "vllm.transformers_utils.configs.eagle", - "FlexOlmoConfig": "vllm.transformers_utils.configs.flex_olmo", "FunAudioChatConfig": "vllm.transformers_utils.configs.funaudiochat", "FunAudioChatAudioEncoderConfig": "vllm.transformers_utils.configs.funaudiochat", "Granite4VisionConfig": "vllm.transformers_utils.configs.granite4_vision", - "HunYuanVLConfig": "vllm.transformers_utils.configs.hunyuan_vl", - "HunYuanVLTextConfig": "vllm.transformers_utils.configs.hunyuan_vl", - "HunYuanVLVisionConfig": "vllm.transformers_utils.configs.hunyuan_vl", "HYV3Config": "vllm.transformers_utils.configs.hy_v3", "HyperCLOVAXConfig": "vllm.transformers_utils.configs.hyperclovax", "IsaacConfig": "vllm.transformers_utils.configs.isaac", @@ -130,13 +126,9 @@ __all__ = [ "K3DSparkConfig", "DotsOCRConfig", "EAGLEConfig", - "FlexOlmoConfig", "FunAudioChatConfig", "FunAudioChatAudioEncoderConfig", "Granite4VisionConfig", - "HunYuanVLConfig", - "HunYuanVLTextConfig", - "HunYuanVLVisionConfig", "HYV3Config", "HyperCLOVAXConfig", "IsaacConfig", diff --git a/vllm/transformers_utils/configs/flex_olmo.py b/vllm/transformers_utils/configs/flex_olmo.py deleted file mode 100644 index d283fde3c0..0000000000 --- a/vllm/transformers_utils/configs/flex_olmo.py +++ /dev/null @@ -1,82 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -from typing import Any - -from transformers.configuration_utils import PretrainedConfig - - -class FlexOlmoConfig(PretrainedConfig): - model_type = "flex_olmo" - keys_to_ignore_at_inference = ["past_key_values"] - - def __init__( - self, - vocab_size=100352, - hidden_size=4096, - intermediate_size=11008, - num_hidden_layers=32, - num_attention_heads=32, - num_key_value_heads=None, - hidden_act="silu", - max_position_embeddings=4096, - initializer_range=0.02, - rms_norm_eps=1e-06, - use_cache=True, - pad_token_id=100277, - bos_token_id=None, - eos_token_id=100257, - tie_word_embeddings=False, - rope_parameters: dict[str, Any] | None = None, - attention_bias=False, - attention_dropout=0.0, - num_experts_per_tok=5, - num_experts=7, - output_router_logits=False, - router_aux_loss_coef=0.01, - norm_topk_prob=False, - **kwargs, - ): - if "architectures" not in kwargs: - kwargs["architectures"] = ["FlexOlmoForCausalLM"] - - self.vocab_size = vocab_size - self.max_position_embeddings = max_position_embeddings - self.hidden_size = hidden_size - self.intermediate_size = intermediate_size - self.num_hidden_layers = num_hidden_layers - self.num_attention_heads = num_attention_heads - - # for backward compatibility - if num_key_value_heads is None: - num_key_value_heads = num_attention_heads - - self.num_key_value_heads = num_key_value_heads - self.hidden_act = hidden_act - self.initializer_range = initializer_range - self.rms_norm_eps = rms_norm_eps - self.use_cache = use_cache - # Try to set `rope_scaling` if available, otherwise use `rope_parameters` - rope_scaling = kwargs.pop("rope_scaling", None) - rope_parameters = rope_scaling or rope_parameters or {"rope_type": "default"} - rope_theta = kwargs.pop("rope_theta", 500000.0) - if "rope_theta" not in rope_parameters: - rope_parameters["rope_theta"] = rope_theta - self.rope_parameters = rope_parameters - self.attention_bias = attention_bias - self.attention_dropout = attention_dropout - self.num_experts_per_tok = num_experts_per_tok - self.num_experts = num_experts - self.output_router_logits = output_router_logits - self.router_aux_loss_coef = router_aux_loss_coef - self.norm_topk_prob = norm_topk_prob - # Validate the correctness of rotary position embeddings parameters - # BC: if there is a 'type' field, move it to 'rope_type'. - if self.rope_parameters is not None and "type" in self.rope_parameters: - self.rope_parameters["rope_type"] = self.rope_parameters["type"] - super().__init__( - pad_token_id=pad_token_id, - bos_token_id=bos_token_id, - eos_token_id=eos_token_id, - tie_word_embeddings=tie_word_embeddings, - **kwargs, - ) diff --git a/vllm/transformers_utils/configs/hunyuan_vl.py b/vllm/transformers_utils/configs/hunyuan_vl.py deleted file mode 100644 index 548dcfefca..0000000000 --- a/vllm/transformers_utils/configs/hunyuan_vl.py +++ /dev/null @@ -1,281 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project - -# adapted from https://github.com/ManaEstras/transformers/blob/v4.57.1.hyvl/src/transformers/models/hunyuan_vl/configuration_hunyuan_vl.py - -from transformers import PretrainedConfig - - -class HunYuanVLVisionConfig(PretrainedConfig): - model_type = "hunyuan_vl" - base_config_key = "vision_config" - - def __init__( - self, - hidden_act="gelu", - hidden_size=1152, - intermediate_size=4304, - interpolate_mode="bilinear", - rms_norm_eps=1e-05, - learnable_mlp_pooling_size=0, - num_attention_heads=16, - num_key_value_heads=None, - num_channels=3, - num_hidden_layers=27, - out_hidden_size=4096, - patch_size=16, - remove_prenorm=True, - spatial_merge_size=2, - temporal_patch_size=1, - resize_resolution=2048, - img_max_token_num=4096, - max_image_size=2048, - video_max_image_size=768, - video_min_image_size=256, - min_image_size=512, - anyres_vit_max_image_size=2048, - max_vit_seq_len=16384, - text_hidden_size=3072, - **kwargs, - ): - super().__init__(**kwargs) - - self.hidden_act = hidden_act - self.hidden_size = hidden_size - self.intermediate_size = intermediate_size - self.interpolate_mode = interpolate_mode - self.learnable_mlp_pooling_size = learnable_mlp_pooling_size - self.num_attention_heads = num_attention_heads - if not num_key_value_heads: - self.num_key_value_heads = num_attention_heads - else: - self.num_key_value_heads = num_key_value_heads - self.num_channels = num_channels - self.num_hidden_layers = num_hidden_layers - self.out_hidden_size = out_hidden_size - self.patch_size = patch_size - self.remove_prenorm = remove_prenorm - self.spatial_merge_size = spatial_merge_size - self.temporal_patch_size = temporal_patch_size - self.rms_norm_eps = rms_norm_eps - - self.resize_resolution = resize_resolution - self.img_max_token_num = img_max_token_num - self.max_image_size = max_image_size - self.min_image_size = min_image_size - self.video_max_image_size = video_max_image_size - self.video_min_image_size = video_min_image_size - self.anyres_vit_max_image_size = anyres_vit_max_image_size - self.max_vit_seq_len = max_vit_seq_len - self.text_hidden_size = text_hidden_size - - -class HunYuanVLTextConfig(PretrainedConfig): - r""" - This is the configuration class to store the configuration of a [`HunYuanVLTextConfig`]. It is used to instantiate an - HunYuan model according to the specified arguments, defining the model architecture. Instantiating a configuration - with the defaults will yield a similar configuration to that of the HunYuan-7B. - Hunyuan-7B-Instruct [tencent/Hunyuan-7B-Instruct](https://huggingface.co/tencent/Hunyuan-7B-Instruct). - - Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the - documentation from [`PretrainedConfig`] for more information. - - - Args: - vocab_size (`int`, *optional*, defaults to 290943): - Vocabulary size of the HunYuan model. Defines the number of different tokens that can be represented by the - `inputs_ids` passed when calling [`HunYuanVLTextConfig`] - hidden_size (`int`, *optional*, defaults to 4096): - Dimension of the hidden representations. - intermediate_size (`int`, *optional*, defaults to 11008): - Dimension of the MLP representations or shared MLP representations. - num_hidden_layers (`int`, *optional*, defaults to 32): - Number of hidden layers in the Transformer decoder. - num_attention_heads (`int`, *optional*, defaults to 32): - Number of attention heads for each attention layer in the Transformer decoder. - num_key_value_heads (`int`, *optional*): - This is the number of key_value heads that should be used to implement Grouped Query Attention. If - `num_key_value_heads=num_attention_heads`, the model will use Multi Head Attention (MHA), if - `num_key_value_heads=1 the model will use Multi Query Attention (MQA) otherwise GQA is used. When - converting a multi-head checkpoint to a GQA checkpoint, each group key and value head should be constructed - by meanpooling all the original heads within that group. For more details checkout [this - paper](https://huggingface.co/papers/2305.13245). If it is not specified, will default to - `num_attention_heads`. - hidden_act (`str` or `function`, *optional*, defaults to `"silu"`): - The non-linear activation function (function or string) in the decoder. - max_position_embeddings (`int`, *optional*, defaults to 2048): - The maximum sequence length that this model might ever be used with. - initializer_range (`float`, *optional*, defaults to 0.02): - The standard deviation of the truncated_normal_initializer for initializing all weight matrices. - rms_norm_eps (`float`, *optional*, defaults to 1e-05): - The epsilon used by the rms normalization layers. - use_cache (`bool`, *optional*, defaults to `True`): - Whether or not the model should return the last key/values attentions (not used by all models). Only - relevant if `config.is_decoder=True`. - pad_token_id (`int`, *optional*, defaults to 0): - Padding token id. - bos_token_id (`int`, *optional*, defaults to 1): - Beginning of stream token id. - eos_token_id (`int`, *optional*, defaults to 2): - End of stream token id. - eod_token_id (int, *optional*, defaults to 3): - Token ID representing the end-of-document marker. Used to indicate the termination of a text sequence. - Example: In multi-document processing, this token helps the model distinguish between separate documents. - pretraining_tp (`int`, *optional*, defaults to 1): - Experimental feature. Tensor parallelism rank used during pretraining. Please refer to [this - document](https://huggingface.co/docs/transformers/parallelism) to understand more about it. This value is - necessary to ensure exact reproducibility of the pretraining results. Please refer to [this - issue](https://github.com/pytorch/pytorch/issues/76232). - tie_word_embeddings (`bool`, *optional*, defaults to `False`): - Whether to tie weight embeddings - rope_theta (`float`, *optional*, defaults to 10000.0): - The base period of the RoPE embeddings. - rope_scaling (`Dict`, *optional*): - Dictionary containing the scaling configuration for the RoPE embeddings. Currently supports two scaling - strategies: linear and dynamic. Their scaling factor must be a float greater than 1. The expected format is - `{"type": strategy name, "factor": scaling factor}`. When using this flag, don't update - `max_position_embeddings` to the expected new maximum. See the following thread for more information on how - these scaling strategies behave: - https://www.reddit.com/r/LocalLLaMA/comments/14mrgpr/dynamically_scaled_rope_further_increases/. This is an - experimental feature, subject to breaking API changes in future versions. - attention_bias (`bool`, defaults to `False`, *optional*, defaults to `False`): - Whether to use a bias in the query, key, value and output projection layers during self-attention. - attention_dropout (`float`, *optional*, defaults to 0.0): - The dropout ratio for the attention probabilities. - head_dim (`int`, *optional*, defaults to 128): - The attention head dimension. - """ # noqa: E501 - - model_type = "hunyuan_vl_text" - keys_to_ignore_at_inference = ["past_key_values"] - - def __init__( - self, - vocab_size=290943, - hidden_size=4096, - intermediate_size: int = 11008, - num_hidden_layers=32, - num_attention_heads=32, - num_key_value_heads=None, - hidden_act="silu", - max_position_embeddings=2048, - initializer_range=0.02, - rms_norm_eps=1e-5, - use_cache=True, - pad_token_id=0, - bos_token_id=1, - eos_token_id=2, - eod_token_id=3, - pretraining_tp=1, - tie_word_embeddings=False, - rope_theta=10000.0, - rope_scaling=None, - attention_bias=False, - attention_dropout=0.0, - head_dim=None, - **kwargs, - ): - self.vocab_size = vocab_size - self.max_position_embeddings = max_position_embeddings - self.hidden_size = hidden_size - self.intermediate_size = intermediate_size - self.num_hidden_layers = num_hidden_layers - self.num_attention_heads = num_attention_heads - self.head_dim = head_dim - # for backward compatibility - if num_key_value_heads is None: - num_key_value_heads = num_attention_heads - - self.num_key_value_heads = num_key_value_heads - self.hidden_act = hidden_act - self.initializer_range = initializer_range - self.rms_norm_eps = rms_norm_eps - self.pretraining_tp = pretraining_tp - self.use_cache = use_cache - self.rope_theta = rope_theta - self.rope_scaling = rope_scaling - self.attention_bias = attention_bias - self.attention_dropout = attention_dropout - - super().__init__( - pad_token_id=pad_token_id, - bos_token_id=bos_token_id, - eos_token_id=eos_token_id, - tie_word_embeddings=tie_word_embeddings, - **kwargs, - ) - - -class HunYuanVLConfig(PretrainedConfig): - model_type = "hunyuan_vl" - sub_configs = { - "vision_config": HunYuanVLVisionConfig, - "text_config": HunYuanVLTextConfig, - } - keys_to_ignore_at_inference = ["past_key_values"] - - def __init__( - self, - text_config=None, - vision_config=None, - im_start_id=120118, - im_end_id=120119, - image_token_id=120120, - im_newline_id=120121, - video_start_id=120122, - video_end_id=120123, - **kwargs, - ): - # We need to init super() here so that it does not reset values - # that are in text config to the BaseClass defaults. The Base - # config has many text related defaults and not all defaults are - # same as for `HunYuanVLTextConfig`. - super().__init__(**kwargs) - - if isinstance(vision_config, dict): - self.vision_config = self.sub_configs["vision_config"](**vision_config) - elif vision_config is None: - self.vision_config = self.sub_configs["vision_config"]() - - if isinstance(text_config, dict): - self.text_config = self.sub_configs["text_config"](**text_config) - elif text_config is None: - # For BC use all kwargs to init `TextConfig` - self.text_config = self.sub_configs["text_config"](**kwargs) - - self.image_token_id = image_token_id - self.im_start_id = im_start_id - self.im_end_id = im_end_id - self.im_newline_id = im_newline_id - self.video_start_id = video_start_id - self.video_end_id = video_end_id - - self.vision_config.text_hidden_size = self.text_config.hidden_size - - # Attention implementation to use. It sets it recursively on sub-configs - # so we call it again in the end. - self._attn_implementation = kwargs.pop("attn_implementation", None) - - def __setattr__(self, key, value): - if ( - (text_config := super().__getattribute__("__dict__").get("text_config")) - is not None - and key not in ["dtype", "_attn_implementation_internal"] - and key in text_config.__dict__ - ): - setattr(text_config, key, value) - else: - super().__setattr__(key, value) - - def __getattribute__(self, key): - if "text_config" in super().__getattribute__("__dict__") and key not in [ - "_name_or_path", - "model_type", - "dtype", - "_attn_implementation_internal", - ]: - text_config = super().__getattribute__("text_config") - if key in text_config.__dict__: - return getattr(text_config, key) - - return super().__getattribute__(key)