diff --git a/docs/models/supported_models.md b/docs/models/supported_models.md index 0ed8279629..f9c70adec7 100644 --- a/docs/models/supported_models.md +++ b/docs/models/supported_models.md @@ -405,6 +405,8 @@ th { | `GraniteMoeForCausalLM` | Granite 3.0 MoE, PowerMoE | `ibm-granite/granite-3.0-1b-a400m-base`, `ibm-granite/granite-3.0-3b-a800m-instruct`, `ibm/PowerMoE-3b`, etc. | ✅︎ | ✅︎ | | `GraniteMoeHybridForCausalLM` | Granite 4.0 MoE Hybrid | `ibm-granite/granite-4.0-tiny-preview`, etc. | ✅︎ | ✅︎ | | `GraniteMoeSharedForCausalLM` | Granite MoE Shared | `ibm-research/moe-7b-1b-active-shared-experts` (test model) | ✅︎ | ✅︎ | +| `GraniteMoeSWAForCausalLM` | Granite MoE SWA | `ibm-granite/granite-swash-3b-a600m` | ✅︎ | ✅︎ | +| `GraniteSWAForCausalLM` | Granite SWA | `ibm-granite/granite-swash-2b` | ✅︎ | ✅︎ | | `GritLM` | GritLM | `parasail-ai/GritLM-7B-vllm`. | ✅︎ | ✅︎ | | `HrmTextForCausalLM` | HRM-Text | `sapientinc/HRM-Text-1B`, etc. | | | | `HunYuanDenseV1ForCausalLM` | Hunyuan Dense | `tencent/Hunyuan-7B-Instruct` | ✅︎ | ✅︎ | diff --git a/tests/models/language/generation/test_granite.py b/tests/models/language/generation/test_granite.py index c0498b2f7d..207e798fef 100644 --- a/tests/models/language/generation/test_granite.py +++ b/tests/models/language/generation/test_granite.py @@ -1,14 +1,20 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project import pytest +from transformers import GraniteConfig -from ...utils import check_logprobs_close +from vllm.model_executor.models.granite import granite_layer_attn_params -MODELS = [ +from ...utils import check_logprobs_close, check_transformers_version + +# model -> minimum transformers version, or None if unconstrained +MODELS = { # TODO(sang): Sliding window should be tested separately. - "ibm/PowerLM-3b", - "ibm/PowerMoE-3b", -] + "ibm/PowerLM-3b": None, + "ibm/PowerMoE-3b": None, + "ibm-granite/granite-swash-2b": "5.15.1", + "ibm-granite/granite-swash-3b-a600m": "5.15.1", +} @pytest.mark.parametrize("model", MODELS) @@ -25,6 +31,8 @@ def test_models( max_tokens: int, num_logprobs: int, ) -> None: + check_transformers_version(model, min_transformers_version=MODELS[model]) + with hf_runner(model, dtype=dtype) as hf_model: hf_outputs = hf_model.generate_greedy_logprobs_limit( example_prompts, max_tokens, num_logprobs @@ -40,3 +48,34 @@ def test_models( name_0="hf", name_1="vllm", ) + + +def test_granite_swa_features_are_off_without_swa_config(): + """A plain Granite config must not pick up sliding windows or sinks.""" + config = GraniteConfig(num_hidden_layers=4) + theta = config.rope_parameters["rope_theta"] + + assert [granite_layer_attn_params(config, i) for i in range(4)] == [ + (None, theta, False) + ] * 4 + + +@pytest.mark.parametrize( + "attention_sinks, expected_sink", [(None, True), (False, False)] +) +def test_granite_swa_features_resolve_per_layer(attention_sinks, expected_sink): + """`layer_types`/`layer_rope_theta` apply per layer; theta 0 means NoPE.""" + kwargs = {} if attention_sinks is None else {"attention_sinks": attention_sinks} + config = GraniteConfig( + num_hidden_layers=3, + sliding_window=128, + layer_types=["full_attention", "sliding_attention", "sliding_attention"], + layer_rope_theta=[10000.0, 0.0, 1000000.0], + **kwargs, + ) + + assert [granite_layer_attn_params(config, i) for i in range(3)] == [ + (None, 10000.0, expected_sink), + (128, 0.0, expected_sink), + (128, 1000000.0, expected_sink), + ] diff --git a/tests/models/registry.py b/tests/models/registry.py index 16604ec7ff..64efae5261 100644 --- a/tests/models/registry.py +++ b/tests/models/registry.py @@ -320,6 +320,12 @@ _TEXT_GENERATION_EXAMPLE_MODELS = { "GraniteMoeSharedForCausalLM": _HfExamplesInfo( "ibm-research/moe-7b-1b-active-shared-experts" ), + "GraniteMoeSWAForCausalLM": _HfExamplesInfo( + "ibm-granite/granite-swash-3b-a600m", min_transformers_version="5.15.1" + ), + "GraniteSWAForCausalLM": _HfExamplesInfo( + "ibm-granite/granite-swash-2b", min_transformers_version="5.15.1" + ), "HrmTextForCausalLM": _HfExamplesInfo( "sapientinc/HRM-Text-1B", min_transformers_version="5.9.0", diff --git a/vllm/model_executor/models/granite.py b/vllm/model_executor/models/granite.py index a0bc8350b4..df813fd1e9 100644 --- a/vllm/model_executor/models/granite.py +++ b/vllm/model_executor/models/granite.py @@ -22,7 +22,13 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -"""Inference-only IBM Granite model compatible with HuggingFace weights.""" +"""Inference-only IBM Granite model compatible with HuggingFace weights. + +Also serves the `granite_swa` checkpoints (`GraniteSWAForCausalLM`), supporting +three additional features: per-layer sliding window attention (`layer_types`), a +learnable per-head attention sink (`self_attn.sinks`), and a per-layer RoPE base +(`layer_rope_theta`, with 0 for NoPE). +""" from collections.abc import Iterable from itertools import islice @@ -30,6 +36,7 @@ from itertools import islice import torch from torch import nn from transformers import GraniteConfig +from transformers.configuration_utils import PretrainedConfig from vllm.compilation.decorators import support_torch_compile from vllm.config import CacheConfig, VllmConfig @@ -49,6 +56,8 @@ from vllm.model_executor.layers.vocab_parallel_embedding import ( ParallelLMHead, VocabParallelEmbedding, ) +from vllm.model_executor.model_loader.weight_utils import sharded_weight_loader +from vllm.model_executor.utils import set_weight_attrs from vllm.sequence import IntermediateTensors from .interfaces import SupportsLoRA, SupportsPP, SupportsQuant @@ -56,11 +65,44 @@ from .utils import ( AutoWeightsLoader, PPMissingLayer, WeightsMapper, + extract_layer_index, make_layers, maybe_prefix, ) +def granite_layer_attn_params( + config: PretrainedConfig, layer_idx: int +) -> tuple[int | None, float, bool]: + """Resolve one layer's sliding window, RoPE base and sink usage. + + Plain Granite configs carry no SWA fields and fall back to full + attention, global RoPE base and no sink. HF SWA checkpoints use + sinks without a dedicated flag, so assume true when `layer_types` + is used, and allow `attention_sinks` to override that decision. + + Returns: + Sliding window size (`None` for full attention), RoPE base theta (`0` + for NoPE), and attention sink presence/absence. + """ + layer_types = getattr(config, "layer_types", None) + sliding_window = ( + config.sliding_window + if layer_types is not None and layer_types[layer_idx] == "sliding_attention" + else None + ) + + layer_rope_theta = getattr(config, "layer_rope_theta", None) + rope_theta = ( + layer_rope_theta[layer_idx] + if layer_rope_theta is not None + else config.rope_parameters["rope_theta"] + ) + + has_sink = getattr(config, "attention_sinks", layer_types is not None) + return sliding_window, rope_theta, has_sink + + class GraniteMLP(nn.Module): def __init__( self, @@ -154,11 +196,25 @@ class GraniteAttention(nn.Module): prefix=f"{prefix}.o_proj", ) - self.rotary_emb = get_rope( - self.head_dim, - max_position=max_position_embeddings, - rope_parameters=config.rope_parameters, + sliding_window, rope_theta, has_sink = granite_layer_attn_params( + config, extract_layer_index(prefix) ) + + self.use_rope = rope_theta != 0 + if self.use_rope: + self.rotary_emb = get_rope( + self.head_dim, + max_position=max_position_embeddings, + rope_parameters={**config.rope_parameters, "rope_theta": rope_theta}, + ) + + # Per-head sink, applied in backend as extra logit in the softmax denom + if has_sink: + self.sinks = nn.Parameter(torch.empty(self.num_heads), requires_grad=False) + set_weight_attrs(self.sinks, {"weight_loader": sharded_weight_loader(0)}) + else: + self.sinks = None + self.attn = Attention( self.num_heads, self.head_dim, @@ -166,7 +222,9 @@ class GraniteAttention(nn.Module): num_kv_heads=self.num_kv_heads, cache_config=cache_config, quant_config=quant_config, + per_layer_sliding_window=sliding_window, prefix=f"{prefix}.attn", + sinks=self.sinks, ) def forward( @@ -176,7 +234,8 @@ class GraniteAttention(nn.Module): ) -> torch.Tensor: qkv, _ = self.qkv_proj(hidden_states) q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1) - q, k = self.rotary_emb(positions, q, k) + if self.use_rope: + q, k = self.rotary_emb(positions, q, k) attn_output = self.attn(q, k, v) output, _ = self.o_proj(attn_output) return output @@ -413,8 +472,5 @@ class GraniteForCausalLM(nn.Module, SupportsLoRA, SupportsPP, SupportsQuant): # processed with quantization, LoRA, fine-tuning, etc. skip_prefixes = ["lm_head."] if self.config.tie_word_embeddings else None - loader = AutoWeightsLoader( - self, - skip_prefixes=skip_prefixes, - ) + loader = AutoWeightsLoader(self, skip_prefixes=skip_prefixes) return loader.load_weights(weights) diff --git a/vllm/model_executor/models/granitemoe.py b/vllm/model_executor/models/granitemoe.py index e1219e3337..d775de50ca 100644 --- a/vllm/model_executor/models/granitemoe.py +++ b/vllm/model_executor/models/granitemoe.py @@ -26,10 +26,10 @@ from collections.abc import Iterable from itertools import islice -from typing import Any import torch from torch import nn +from transformers.configuration_utils import PretrainedConfig from vllm.compilation.decorators import support_torch_compile from vllm.config import CacheConfig, VllmConfig @@ -39,10 +39,7 @@ from vllm.distributed import ( tensor_model_parallel_all_gather, ) from vllm.model_executor.layers.attention import Attention -from vllm.model_executor.layers.fused_moe import ( - FusedMoEFactory, - fused_moe_make_expert_params_mapping, -) +from vllm.model_executor.layers.fused_moe import FusedMoEFactory from vllm.model_executor.layers.layernorm import RMSNorm from vllm.model_executor.layers.linear import ( QKVParallelLinear, @@ -56,15 +53,20 @@ from vllm.model_executor.layers.vocab_parallel_embedding import ( ParallelLMHead, VocabParallelEmbedding, ) -from vllm.model_executor.model_loader.weight_utils import ( - default_weight_loader, - maybe_remap_kv_scale_name, -) +from vllm.model_executor.model_loader.weight_utils import sharded_weight_loader from vllm.model_executor.models.utils import sequence_parallel_chunk +from vllm.model_executor.utils import set_weight_attrs from vllm.sequence import IntermediateTensors +from .granite import granite_layer_attn_params from .interfaces import SupportsLoRA, SupportsPP -from .utils import AutoWeightsLoader, is_pp_missing_parameter, make_layers, maybe_prefix +from .utils import ( + AutoWeightsLoader, + WeightsMapper, + extract_layer_index, + make_layers, + maybe_prefix, +) class GraniteMoeMoE(nn.Module): @@ -139,11 +141,11 @@ class GraniteMoeMoE(nn.Module): class GraniteMoeAttention(nn.Module): def __init__( self, + config: PretrainedConfig, hidden_size: int, num_heads: int, num_kv_heads: int, max_position: int = 4096 * 32, - rope_parameters: dict[str, Any] | None = None, cache_config: CacheConfig | None = None, quant_config: QuantizationConfig | None = None, attention_multiplier: float | None = None, @@ -190,12 +192,25 @@ class GraniteMoeAttention(nn.Module): quant_config=quant_config, prefix=f"{prefix}.o_proj", ) - self.rotary_emb = get_rope( - self.head_dim, - max_position=max_position, - rope_parameters=rope_parameters, - is_neox_style=True, + sliding_window, rope_theta, has_sink = granite_layer_attn_params( + config, extract_layer_index(prefix) ) + + self.use_rope = rope_theta != 0 + if self.use_rope: + self.rotary_emb = get_rope( + self.head_dim, + max_position=max_position, + rope_parameters={**config.rope_parameters, "rope_theta": rope_theta}, + is_neox_style=True, + ) + + if has_sink: + self.sinks = nn.Parameter(torch.empty(self.num_heads), requires_grad=False) + set_weight_attrs(self.sinks, {"weight_loader": sharded_weight_loader(0)}) + else: + self.sinks = None + self.attn = Attention( self.num_heads, self.head_dim, @@ -203,7 +218,9 @@ class GraniteMoeAttention(nn.Module): num_kv_heads=self.num_kv_heads, cache_config=cache_config, quant_config=quant_config, + per_layer_sliding_window=sliding_window, prefix=f"{prefix}.attn", + sinks=self.sinks, ) def forward( @@ -213,7 +230,8 @@ class GraniteMoeAttention(nn.Module): ) -> torch.Tensor: qkv, _ = self.qkv_proj(hidden_states) q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1) - q, k = self.rotary_emb(positions, q, k) + if self.use_rope: + q, k = self.rotary_emb(positions, q, k) attn_output = self.attn(q, k, v) output, _ = self.o_proj(attn_output) return output @@ -234,11 +252,11 @@ class GraniteMoeDecoderLayer(nn.Module): self.hidden_size = config.hidden_size self.self_attn = GraniteMoeAttention( + config=config, hidden_size=self.hidden_size, num_heads=config.num_attention_heads, max_position=config.max_position_embeddings, num_kv_heads=config.num_key_value_heads, - rope_parameters=config.rope_parameters, cache_config=cache_config, quant_config=quant_config, prefix=f"{prefix}.self_attn", @@ -284,6 +302,23 @@ class GraniteMoeDecoderLayer(nn.Module): @support_torch_compile class GraniteMoeModel(nn.Module): + hf_to_vllm_mapper: WeightsMapper = WeightsMapper( + orig_to_new_suffix={ + # Legacy names to new names + "moe.input_linear.weight": "moe.experts.gate_up_proj", + "moe.output_linear.weight": "moe.experts.down_proj", + ".router.layer.weight": ".gate.weight", + # Checkpoint name to vLLM name + ".router.weight": ".gate.weight", + }, + orig_to_new_stacked={ + # weight_name: (param_name, shard_id) + ".q_proj": (".qkv_proj", "q"), + ".k_proj": (".qkv_proj", "k"), + ".v_proj": (".qkv_proj", "v"), + }, + ) + def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""): super().__init__() @@ -339,135 +374,9 @@ class GraniteMoeModel(nn.Module): hidden_states = self.norm(hidden_states) return hidden_states - def _load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: - """ - This function is copied from `MixtralModel.load_weights`, mainly to - decouple from mixtral, avoiding impact on support like BNB - quantization. - """ - stacked_params_mapping = [ - # (param_name, shard_name, shard_id) - ("qkv_proj", "q_proj", "q"), - ("qkv_proj", "k_proj", "k"), - ("qkv_proj", "v_proj", "v"), - ] - - # Params for weights, fp8 weight scales, fp8 activation scales - # (param_name, weight_name, expert_id, shard_id) - expert_params_mapping = fused_moe_make_expert_params_mapping( - self, - ckpt_gate_proj_name="w1", - ckpt_down_proj_name="w2", - ckpt_up_proj_name="w3", - num_experts=self.config.num_local_experts, - ) - - params_dict = dict(self.named_parameters()) - loaded_params: set[str] = set() - for name, loaded_weight in weights: - for param_name, weight_name, shard_id in stacked_params_mapping: - if weight_name not in name: - continue - name = name.replace(weight_name, param_name) - # Skip loading extra bias for GPTQ models. - if ( - name.endswith(".bias") or name.endswith("_bias") - ) and name not in params_dict: - continue - # Skip layers on other devices. - if is_pp_missing_parameter(name, self): - continue - if name.endswith("scale"): - # Remapping the name of FP8 kv-scale. - name = maybe_remap_kv_scale_name(name, params_dict) - if name is None: - continue - param = params_dict[name] - weight_loader = param.weight_loader - weight_loader(param, loaded_weight, shard_id) - break - else: - for mapping in expert_params_mapping: - param_name, weight_name, expert_id, shard_id = mapping - if weight_name not in name: - continue - name = name.replace(weight_name, param_name) - # Skip layers on other devices. - if is_pp_missing_parameter(name, self): - continue - if ( - name.endswith(".bias") or name.endswith("_bias") - ) and name not in params_dict: - continue - param = params_dict[name] - weight_loader = param.weight_loader - weight_loader( - param, - loaded_weight, - name, - shard_id=shard_id, - expert_id=expert_id, - ) - break - else: - # Skip loading extra bias for GPTQ models. - if ( - name.endswith(".bias") or name.endswith("_bias") - ) and name not in params_dict: - continue - # Skip layers on other devices. - if is_pp_missing_parameter(name, self): - continue - # Remapping the name of FP8 kv-scale. - name = maybe_remap_kv_scale_name(name, params_dict) - if name is None: - continue - - param = params_dict[name] - weight_loader = getattr( - param, "weight_loader", default_weight_loader - ) - weight_loader(param, loaded_weight) - loaded_params.add(name) - return loaded_params - def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: - new_weights = {} - for n, p in weights: - if n.endswith(".block_sparse_moe.input_linear.weight"): - for e in range(p.size(0)): - w1_name = n.replace( - ".block_sparse_moe.input_linear.weight", - f".block_sparse_moe.experts.{e}.w1.weight", - ) - w3_name = n.replace( - ".block_sparse_moe.input_linear.weight", - f".block_sparse_moe.experts.{e}.w3.weight", - ) - w1_param, w3_param = p[e].chunk(2, dim=0) - assert w1_name not in new_weights - assert w3_name not in new_weights - new_weights[w1_name] = w1_param - new_weights[w3_name] = w3_param - elif n.endswith(".block_sparse_moe.output_linear.weight"): - for e in range(p.size(0)): - w2_name = n.replace( - ".block_sparse_moe.output_linear.weight", - f".block_sparse_moe.experts.{e}.w2.weight", - ) - w2_param = p[e] - assert w2_name not in new_weights - new_weights[w2_name] = w2_param - elif n.endswith(".block_sparse_moe.router.layer.weight"): - gate_name = n.replace( - ".block_sparse_moe.router.layer.weight", - ".block_sparse_moe.gate.weight", - ) - assert gate_name not in new_weights - new_weights[gate_name] = p - else: - new_weights[n] = p - return self._load_weights(new_weights.items()) + loader = AutoWeightsLoader(self) + return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper) class GraniteMoeForCausalLM(nn.Module, SupportsLoRA, SupportsPP): @@ -543,8 +452,6 @@ class GraniteMoeForCausalLM(nn.Module, SupportsLoRA, SupportsPP): ) def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: - loader = AutoWeightsLoader( - self, - skip_prefixes=(["lm_head."] if self.config.tie_word_embeddings else None), - ) + skip_prefixes = ["lm_head."] if self.config.tie_word_embeddings else None + loader = AutoWeightsLoader(self, skip_prefixes=skip_prefixes) return loader.load_weights(weights) diff --git a/vllm/model_executor/models/granitemoeshared.py b/vllm/model_executor/models/granitemoeshared.py index 603afc60d6..ecd942da40 100644 --- a/vllm/model_executor/models/granitemoeshared.py +++ b/vllm/model_executor/models/granitemoeshared.py @@ -4,6 +4,10 @@ The architecture is the same as granitemoe but with the addition of shared experts. + +Also serves the `granitemoe_swa` checkpoints (`GraniteMoeSWAForCausalLM`), which +add the same per-layer sliding window, attention sink and per-layer RoPE support +as `granite_swa` (see `granite.py`). """ from collections.abc import Iterable @@ -85,11 +89,11 @@ class GraniteMoeSharedDecoderLayer(nn.Module): super().__init__() self.hidden_size = config.hidden_size self.self_attn = GraniteMoeAttention( + config=config, hidden_size=self.hidden_size, num_heads=config.num_attention_heads, max_position=config.max_position_embeddings, num_kv_heads=config.num_key_value_heads, - rope_parameters=config.rope_parameters, cache_config=cache_config, quant_config=quant_config, prefix=f"{prefix}.self_attn", @@ -207,43 +211,8 @@ class GraniteMoeSharedModel(nn.Module): hidden_states = self.norm(hidden_states) return hidden_states - def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: - new_weights = {} - for n, p in weights: - if n.endswith(".block_sparse_moe.input_linear.weight"): - for e in range(p.size(0)): - w1_name = n.replace( - ".block_sparse_moe.input_linear.weight", - f".block_sparse_moe.experts.{e}.w1.weight", - ) - w3_name = n.replace( - ".block_sparse_moe.input_linear.weight", - f".block_sparse_moe.experts.{e}.w3.weight", - ) - w1_param, w3_param = p[e].chunk(2, dim=0) - assert w1_name not in new_weights - assert w3_name not in new_weights - new_weights[w1_name] = w1_param - new_weights[w3_name] = w3_param - elif n.endswith(".block_sparse_moe.output_linear.weight"): - for e in range(p.size(0)): - w2_name = n.replace( - ".block_sparse_moe.output_linear.weight", - f".block_sparse_moe.experts.{e}.w2.weight", - ) - w2_param = p[e] - assert w2_name not in new_weights - new_weights[w2_name] = w2_param - elif n.endswith(".block_sparse_moe.router.layer.weight"): - gate_name = n.replace( - ".block_sparse_moe.router.layer.weight", - ".block_sparse_moe.gate.weight", - ) - assert gate_name not in new_weights - new_weights[gate_name] = p - else: - new_weights[n] = p - return GraniteMoeModel._load_weights(self, new_weights.items()) + hf_to_vllm_mapper = GraniteMoeModel.hf_to_vllm_mapper + load_weights = GraniteMoeModel.load_weights class GraniteMoeSharedForCausalLM(nn.Module, SupportsLoRA, SupportsPP): @@ -320,8 +289,6 @@ class GraniteMoeSharedForCausalLM(nn.Module, SupportsLoRA, SupportsPP): ) def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: - loader = AutoWeightsLoader( - self, - skip_prefixes=(["lm_head."] if self.config.tie_word_embeddings else None), - ) + skip_prefixes = ["lm_head."] if self.config.tie_word_embeddings else None + loader = AutoWeightsLoader(self, skip_prefixes=skip_prefixes) return loader.load_weights(weights) diff --git a/vllm/model_executor/models/registry.py b/vllm/model_executor/models/registry.py index 1e4bf6731c..95877c319d 100644 --- a/vllm/model_executor/models/registry.py +++ b/vllm/model_executor/models/registry.py @@ -124,6 +124,8 @@ _TEXT_GENERATION_MODELS = { "GraniteMoeForCausalLM": ("granitemoe", "GraniteMoeForCausalLM"), "GraniteMoeHybridForCausalLM": ("granitemoehybrid", "GraniteMoeHybridForCausalLM"), "GraniteMoeSharedForCausalLM": ("granitemoeshared", "GraniteMoeSharedForCausalLM"), + "GraniteMoeSWAForCausalLM": ("granitemoeshared", "GraniteMoeSharedForCausalLM"), + "GraniteSWAForCausalLM": ("granite", "GraniteForCausalLM"), "GritLM": ("gritlm", "GritLM"), "HrmTextForCausalLM": ("hrm_text", "HrmTextForCausalLM"), "HunYuanMoEV1ForCausalLM": ("hunyuan_v1", "HunYuanMoEV1ForCausalLM"),