diff --git a/docs/models/supported_models.md b/docs/models/supported_models.md index e434c57012..de937ec520 100644 --- a/docs/models/supported_models.md +++ b/docs/models/supported_models.md @@ -364,7 +364,7 @@ th { | `Ernie4_5ForCausalLM` | Ernie4.5 | `baidu/ERNIE-4.5-0.3B-PT`, etc. | ✅︎ | ✅︎ | | `Ernie4_5_MoeForCausalLM` | Ernie4.5MoE | `baidu/ERNIE-4.5-21B-A3B-PT`, `baidu/ERNIE-4.5-300B-A47B-PT`, etc. | ✅︎ | ✅︎ | | `ExaoneForCausalLM` | EXAONE-3 | `LGAI-EXAONE/EXAONE-3.0-7.8B-Instruct`, etc. | ✅︎ | ✅︎ | -| `ExaoneMoEForCausalLM` | K-EXAONE | `LGAI-EXAONE/K-EXAONE-236B-A23B`, etc. | | | +| `ExaoneMoeForCausalLM` | K-EXAONE, K-EXAONE-2 | `LGAI-EXAONE/K-EXAONE-236B-A23B`, `LGAI-EXAONE/K-EXAONE-2.0-750B-A37B`, etc. | | | | `Exaone4ForCausalLM` | EXAONE-4 | `LGAI-EXAONE/EXAONE-4.0-32B`, etc. | ✅︎ | ✅︎ | | `Fairseq2LlamaForCausalLM` | Llama (fairseq2 format) | `mgleize/fairseq2-dummy-Llama-3.2-1B`, etc. | ✅︎ | ✅︎ | | `FalconForCausalLM` | Falcon | `tiiuae/falcon-7b`, `tiiuae/falcon-40b`, `tiiuae/falcon-rw-7b`, etc. | | ✅︎ | diff --git a/tests/models/registry.py b/tests/models/registry.py index f8d0387dff..4ea3120715 100644 --- a/tests/models/registry.py +++ b/tests/models/registry.py @@ -267,7 +267,7 @@ _TEXT_GENERATION_EXAMPLE_MODELS = { "LGAI-EXAONE/EXAONE-3.0-7.8B-Instruct", trust_remote_code=True ), "Exaone4ForCausalLM": _HfExamplesInfo("LGAI-EXAONE/EXAONE-4.0-32B"), - "ExaoneMoEForCausalLM": _HfExamplesInfo( + "ExaoneMoeForCausalLM": _HfExamplesInfo( "LGAI-EXAONE/K-EXAONE-236B-A23B", min_transformers_version="5.1.0" ), "Fairseq2LlamaForCausalLM": _HfExamplesInfo("mgleize/fairseq2-dummy-Llama-3.2-1B"), diff --git a/vllm/model_executor/models/exaone4.py b/vllm/model_executor/models/exaone4.py index dc88c15fc0..8e8252f459 100644 --- a/vllm/model_executor/models/exaone4.py +++ b/vllm/model_executor/models/exaone4.py @@ -31,7 +31,7 @@ from transformers import Exaone4Config from vllm.compilation.decorators import support_torch_compile from vllm.config import CacheConfig, VllmConfig from vllm.distributed import get_pp_group, get_tensor_model_parallel_world_size -from vllm.model_executor.layers.activation import SiluAndMul +from vllm.model_executor.layers.activation import SiluAndMul, SiluAndMulWithClamp from vllm.model_executor.layers.attention import Attention from vllm.model_executor.layers.layernorm import RMSNorm from vllm.model_executor.layers.linear import ( @@ -70,6 +70,7 @@ class Exaone4GatedMLP(nn.Module): quant_config: QuantizationConfig | None = None, reduce_results: bool = True, bias: bool = False, + swiglu_limit: float | None = None, prefix: str = "", use_data_parallel: bool = False, ) -> None: @@ -95,7 +96,9 @@ class Exaone4GatedMLP(nn.Module): raise ValueError( f"Unsupported activation: {hidden_act}. Only silu is supported for now." ) - self.act_fn = SiluAndMul() + self.act_fn = ( + SiluAndMul() if swiglu_limit is None else SiluAndMulWithClamp(swiglu_limit) + ) def forward(self, x): gate_up, _ = self.gate_up_proj(x) diff --git a/vllm/model_executor/models/exaone_moe.py b/vllm/model_executor/models/exaone_moe.py index a4d4e9019d..0d7da9232d 100644 --- a/vllm/model_executor/models/exaone_moe.py +++ b/vllm/model_executor/models/exaone_moe.py @@ -29,19 +29,25 @@ from vllm.distributed import ( get_pp_group, get_tensor_model_parallel_world_size, ) +from vllm.model_executor.layers.attention import Attention from vllm.model_executor.layers.fused_moe import FusedMoEFactory from vllm.model_executor.layers.layernorm import RMSNorm -from vllm.model_executor.layers.linear import ReplicatedLinear +from vllm.model_executor.layers.linear import ( + QKVParallelLinear, + ReplicatedLinear, + RowParallelLinear, +) from vllm.model_executor.layers.logits_processor import LogitsProcessor from vllm.model_executor.layers.quantization import QuantizationConfig +from vllm.model_executor.layers.rotary_embedding import get_rope from vllm.model_executor.layers.vocab_parallel_embedding import ( DEFAULT_VOCAB_PADDING_SIZE, ParallelLMHead, VocabParallelEmbedding, ) from vllm.sequence import IntermediateTensors +from vllm.transformers_utils.config import set_default_rope_theta -from .exaone4 import Exaone4Attention as ExaoneMoeAttention from .exaone4 import Exaone4GatedMLP as ExaoneMoeGatedMLP from .interfaces import SupportsLoRA, SupportsPP from .utils import ( @@ -59,6 +65,7 @@ class ExaoneMoe(nn.Module): def __init__( self, config: PretrainedConfig, + swiglu_limit: float | None = None, quant_config: QuantizationConfig | None = None, prefix: str = "", enable_eplb: bool = False, @@ -119,6 +126,7 @@ class ExaoneMoe(nn.Module): hidden_act=config.hidden_act, quant_config=quant_config, reduce_results=False, + swiglu_limit=swiglu_limit, prefix=f"{prefix}.shared_experts", ) else: @@ -133,6 +141,8 @@ class ExaoneMoe(nn.Module): intermediate_size=config.moe_intermediate_size, renormalize=config.norm_topk_prob, quant_config=quant_config, + activation="silu", + swiglu_limit=swiglu_limit, use_grouped_topk=True, num_expert_group=config.n_group, topk_group=config.topk_group, @@ -157,13 +167,138 @@ class ExaoneMoe(nn.Module): return final_hidden_states.view(orig_shape) +class ExaoneMoeAttention(nn.Module): + def __init__( + self, + config, + hidden_size: int, + num_heads: int, + num_kv_heads: int, + max_position_embeddings: int = 8192, + quant_config: QuantizationConfig | None = None, + bias: bool = False, + cache_config: CacheConfig | None = None, + prefix: str = "", + is_mtp: bool = False, + ) -> None: + super().__init__() + self.hidden_size = hidden_size + tp_size = get_tensor_model_parallel_world_size() + self.total_num_heads = num_heads + assert self.total_num_heads % tp_size == 0 + self.num_heads = self.total_num_heads // tp_size + self.total_num_kv_heads = num_kv_heads + if self.total_num_kv_heads >= tp_size: + # Number of KV heads is greater than TP size, so we partition + # the KV heads across multiple tensor parallel GPUs. + assert self.total_num_kv_heads % tp_size == 0 + else: + # Number of KV heads is less than TP size, so we replicate + # the KV heads across multiple tensor parallel GPUs. + assert tp_size % self.total_num_kv_heads == 0 + self.num_kv_heads = max(1, self.total_num_kv_heads // tp_size) + # MistralConfig has an optional head_dim introduced by Mistral-Nemo + self.head_dim = getattr(config, "head_dim", None) + if self.head_dim is None: + self.head_dim = self.hidden_size // self.total_num_heads + self.q_size = self.num_heads * self.head_dim + self.kv_size = self.num_kv_heads * self.head_dim + self.scaling = self.head_dim**-0.5 + self.max_position_embeddings = max_position_embeddings + + self.qkv_proj = QKVParallelLinear( + hidden_size=hidden_size, + head_size=self.head_dim, + total_num_heads=self.total_num_heads, + total_num_kv_heads=self.total_num_kv_heads, + bias=bias, + quant_config=quant_config, + prefix=f"{prefix}.qkv_proj", + ) + + self.o_proj = RowParallelLinear( + input_size=self.total_num_heads * self.head_dim, + output_size=hidden_size, + bias=bias, + quant_config=quant_config, + prefix=f"{prefix}.o_proj", + ) + + self.q_norm = RMSNorm(self.head_dim, eps=config.rms_norm_eps) + self.k_norm = RMSNorm(self.head_dim, eps=config.rms_norm_eps) + + is_neox_style = True + if quant_config is not None and quant_config.get_name() == "gguf": + is_neox_style = False + + layer_idx = extract_layer_index(prefix) + + if config.sliding_windows is not None: + self.sliding_window_size = config.sliding_windows[layer_idx] + + if config.layer_types[layer_idx] == "full_attention": + self.sliding_window_size = None + + if is_mtp: + self.sliding_window = ( + config.mtp_layer_types[layer_idx] == "sliding_attention" + ) + if config.mtp_sliding_windows is not None: + self.sliding_window_size = config.mtp_sliding_windows[layer_idx] + + if config.mtp_layer_types[layer_idx] == "full_attention": + self.sliding_window_size = None + + # apply rotary embeddings to every layer in full attention models + self.apply_rope_all_layers = "sliding_attention" not in config.layer_types + + set_default_rope_theta(config, default_theta=1000000) + self.rotary_emb = get_rope( + self.head_dim, + max_position=max_position_embeddings, + rope_parameters=config.rope_parameters, + is_neox_style=is_neox_style, + ) + self.attn = Attention( + self.num_heads, + self.head_dim, + self.scaling, + num_kv_heads=self.num_kv_heads, + cache_config=cache_config, + quant_config=quant_config, + per_layer_sliding_window=self.sliding_window_size, + prefix=f"{prefix}.attn", + ) + + def forward( + self, + positions: torch.Tensor, + hidden_states: torch.Tensor, + ) -> torch.Tensor: + qkv, _ = self.qkv_proj(hidden_states) + q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1) + + q = q.unflatten(-1, (self.num_heads, self.head_dim)) + q = self.q_norm(q) + q = q.flatten(-2, -1) + k = k.unflatten(-1, (self.num_kv_heads, self.head_dim)) + k = self.k_norm(k) + k = k.flatten(-2, -1) + + if self.sliding_window_size or self.apply_rope_all_layers: + q, k = self.rotary_emb(positions, q, k) + attn_output = self.attn(q, k, v) + output, _ = self.o_proj(attn_output) + return output + + class ExaoneMoeDecoderLayer(nn.Module): def __init__( self, config: PretrainedConfig, cache_config: CacheConfig | None = None, quant_config: QuantizationConfig | None = None, - mtp_layer: bool = None, + is_mtp: bool = None, prefix: str = "", ) -> None: super().__init__() @@ -187,17 +322,29 @@ class ExaoneMoeDecoderLayer(nn.Module): bias=attention_bias, cache_config=cache_config, prefix=f"{prefix}.self_attn", + is_mtp=is_mtp, ) - if config.is_moe_layer[layer_idx] and not mtp_layer: + swiglu_limits = getattr(config, "swiglu_limits", None) + if swiglu_limits is not None: + swiglu_limit = swiglu_limits[layer_idx] + swiglu_limit = swiglu_limit if swiglu_limit > 0 else None + else: + swiglu_limit = None + + if config.mlp_layer_types[layer_idx] == "sparse" and not is_mtp: self.mlp = ExaoneMoe( - config=config, quant_config=quant_config, prefix=f"{prefix}.mlp" + config=config, + swiglu_limit=swiglu_limit, + quant_config=quant_config, + prefix=f"{prefix}.mlp", ) else: self.mlp = ExaoneMoeGatedMLP( hidden_size=self.hidden_size, intermediate_size=config.intermediate_size, hidden_act=config.hidden_act, + swiglu_limit=swiglu_limit, quant_config=quant_config, bias=getattr(config, "mlp_bias", False), prefix=f"{prefix}.mlp", diff --git a/vllm/model_executor/models/exaone_moe_mtp.py b/vllm/model_executor/models/exaone_moe_mtp.py index a37da487db..ddd570d604 100644 --- a/vllm/model_executor/models/exaone_moe_mtp.py +++ b/vllm/model_executor/models/exaone_moe_mtp.py @@ -81,7 +81,7 @@ class ExaoneMoeMultiTokenPredictor(nn.Module): vllm_config.model_config.hf_config, quant_config=quant_config, prefix=f"{prefix}.layers.{idx}", - mtp_layer=True, + is_mtp=True, ) for idx in range(self.num_mtp_layers) ) diff --git a/vllm/model_executor/models/registry.py b/vllm/model_executor/models/registry.py index 3798d22b6c..545b257794 100644 --- a/vllm/model_executor/models/registry.py +++ b/vllm/model_executor/models/registry.py @@ -97,7 +97,7 @@ _TEXT_GENERATION_MODELS = { "Ernie4_5_MoeForCausalLM": ("ernie45_moe", "Ernie4_5_MoeForCausalLM"), "ExaoneForCausalLM": ("exaone", "ExaoneForCausalLM"), "Exaone4ForCausalLM": ("exaone4", "Exaone4ForCausalLM"), - "ExaoneMoEForCausalLM": ("exaone_moe", "ExaoneMoeForCausalLM"), + "ExaoneMoeForCausalLM": ("exaone_moe", "ExaoneMoeForCausalLM"), "Fairseq2LlamaForCausalLM": ("fairseq2_llama", "Fairseq2LlamaForCausalLM"), "FalconForCausalLM": ("falcon", "FalconForCausalLM"), "FalconMambaForCausalLM": ("mamba", "MambaForCausalLM"),