# SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project # Adapted from: # https://github.com/huggingface/transformers/blob/main/src/transformers/models/olmo_hybrid/modeling_olmo_hybrid.py # Copyright 2026 The vLLM team. # # This code combines OLMo2/OLMo3 attention with Gated DeltaNet linear attention # for the OLMo Hybrid architecture. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Inference-only OLMo Hybrid model compatible with HuggingFace weights.""" from collections.abc import Iterable from functools import partial from itertools import islice import torch from torch import nn from vllm.compilation.decorators import support_torch_compile from vllm.config import ( VllmConfig, ) from vllm.distributed import ( get_pp_group, get_tensor_model_parallel_rank, get_tensor_model_parallel_world_size, tensor_model_parallel_all_gather, ) from vllm.distributed.utils import split_tensor_along_last_dim from vllm.logger import init_logger from vllm.model_executor.layers.activation import SiluAndMul from vllm.model_executor.layers.attention import Attention from vllm.model_executor.layers.layernorm import RMSNorm from vllm.model_executor.layers.linear import ( MergedColumnParallelLinear, QKVParallelLinear, RowParallelLinear, ) from vllm.model_executor.layers.logits_processor import LogitsProcessor from vllm.model_executor.layers.mamba.gdn.olmo_gdn_linear_attn import ( OlmoHybridGatedDeltaNetAttention, ) from vllm.model_executor.layers.mamba.mamba_utils import ( MambaStateCopyFunc, MambaStateCopyFuncCalculator, MambaStateDtypeCalculator, MambaStateShapeCalculator, ) from vllm.model_executor.layers.rotary_embedding import get_rope from vllm.model_executor.layers.vocab_parallel_embedding import ( ParallelLMHead, VocabParallelEmbedding, ) from vllm.sequence import IntermediateTensors from .interfaces import HasInnerState, IsHybrid, SupportsLoRA, SupportsPP from .utils import ( AutoWeightsLoader, WeightsMapper, extract_layer_index, make_empty_intermediate_tensors_factory, make_layers, maybe_prefix, ) logger = init_logger(__name__) class OlmoHybridAttention(nn.Module): def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""): super().__init__() self.config = vllm_config.model_config.hf_config hidden_size = self.config.hidden_size self.tp_size = get_tensor_model_parallel_world_size() self.total_num_heads = self.config.num_attention_heads assert hidden_size % self.total_num_heads == 0 assert self.total_num_heads % self.tp_size == 0 self.num_heads = self.total_num_heads // self.tp_size self.total_num_kv_heads = ( self.config.num_key_value_heads or self.total_num_heads ) if self.total_num_kv_heads >= self.tp_size: assert self.total_num_kv_heads % self.tp_size == 0 else: assert self.tp_size % self.total_num_kv_heads == 0 self.num_kv_heads = max(1, self.total_num_kv_heads // self.tp_size) self.head_dim = hidden_size // self.total_num_heads self.q_size = self.num_heads * self.head_dim self.kv_size = self.num_kv_heads * self.head_dim self.max_position_embeddings = self.config.max_position_embeddings self.qkv_proj = QKVParallelLinear( hidden_size, self.head_dim, self.total_num_heads, self.total_num_kv_heads, bias=False, quant_config=vllm_config.quant_config, prefix=f"{prefix}.qkv_proj", ) self.tp_rank = get_tensor_model_parallel_rank() self.k_norm = RMSNorm( self.total_num_kv_heads * self.head_dim, eps=self.config.rms_norm_eps, ) self.q_norm = RMSNorm( self.config.hidden_size, eps=self.config.rms_norm_eps, ) self.scaling = self.head_dim**-0.5 self.attn = Attention( self.num_heads, self.head_dim, self.scaling, num_kv_heads=self.num_kv_heads, cache_config=vllm_config.cache_config, quant_config=vllm_config.quant_config, prefix=f"{prefix}.attn", ) rope_parameters = getattr(self.config, "rope_parameters", None) self._use_rope = (rope_parameters is not None) and ( rope_parameters["rope_theta"] is not None ) self.rotary_emb: nn.Module | None if self._use_rope: self.rotary_emb = get_rope( self.head_dim, max_position=self.max_position_embeddings, rope_parameters=rope_parameters, ) else: self.rotary_emb = None self.o_proj = RowParallelLinear( self.total_num_heads * self.head_dim, hidden_size, bias=False, quant_config=vllm_config.quant_config, prefix=f"{prefix}.o_proj", ) def _apply_qk_norm( self, q: torch.Tensor, k: torch.Tensor ) -> tuple[torch.Tensor, torch.Tensor]: if self.tp_size > 1: q = tensor_model_parallel_all_gather(q.contiguous()) k = tensor_model_parallel_all_gather(k.contiguous()) q = self.q_norm(q) k = self.k_norm(k) if self.tp_size > 1: splitter = partial(split_tensor_along_last_dim, num_partitions=self.tp_size) q = splitter(q)[self.tp_rank] k = splitter(k)[self.tp_rank] return q, k def forward( self, positions: torch.Tensor, hidden_states: torch.Tensor, ) -> torch.Tensor: qkv, _ = self.qkv_proj(hidden_states) q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1) q, k = self._apply_qk_norm(q, k) if self._use_rope: assert self.rotary_emb is not None q, k = self.rotary_emb(positions, q, k) attn_output = self.attn(q, k, v) output, _ = self.o_proj(attn_output) return output class OlmoHybridMLP(nn.Module): def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""): super().__init__() config = vllm_config.model_config.hf_config hidden_size = config.hidden_size intermediate_size = config.intermediate_size self.gate_up_proj = MergedColumnParallelLinear( hidden_size, [intermediate_size] * 2, bias=False, quant_config=vllm_config.quant_config, prefix=f"{prefix}.gate_up_proj", ) self.act_fn = SiluAndMul() self.down_proj = RowParallelLinear( intermediate_size, hidden_size, bias=False, quant_config=vllm_config.quant_config, prefix=f"{prefix}.down_proj", ) def forward(self, x: torch.Tensor) -> torch.Tensor: gate_up, _ = self.gate_up_proj(x) x = self.act_fn(gate_up) x, _ = self.down_proj(x) return x class OlmoHybridDecoderLayer(nn.Module): def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None: super().__init__() config = vllm_config.model_config.hf_config layer_idx = extract_layer_index(prefix) self.layer_type = config.layer_types[layer_idx] self.layer_idx = layer_idx if self.layer_type == "linear_attention": self.linear_attn = OlmoHybridGatedDeltaNetAttention( config, vllm_config, prefix=f"{prefix}.linear_attn", ) self.input_layernorm = RMSNorm( config.hidden_size, eps=config.rms_norm_eps, ) self.post_attention_layernorm = RMSNorm( config.hidden_size, eps=config.rms_norm_eps, ) else: self.self_attn = OlmoHybridAttention( vllm_config=vllm_config, prefix=f"{prefix}.self_attn", ) # Attention layers use these norm names self.post_attention_layernorm = RMSNorm( config.hidden_size, eps=config.rms_norm_eps, ) self.post_feedforward_layernorm = RMSNorm( config.hidden_size, eps=config.rms_norm_eps, ) self.mlp = OlmoHybridMLP( vllm_config=vllm_config, prefix=f"{prefix}.mlp", ) def forward( self, positions: torch.Tensor, hidden_states: torch.Tensor, ) -> torch.Tensor: if self.layer_type == "linear_attention": residual = hidden_states hidden_states = self.input_layernorm(hidden_states) attn_output = torch.empty_like(hidden_states) self.linear_attn( hidden_states=hidden_states, output=attn_output, ) hidden_states = residual + attn_output residual = hidden_states hidden_states = self.post_attention_layernorm(hidden_states) hidden_states = self.mlp(hidden_states) hidden_states = residual + hidden_states else: residual = hidden_states hidden_states = self.self_attn(positions, hidden_states) hidden_states = self.post_attention_layernorm(hidden_states) hidden_states = residual + hidden_states residual = hidden_states hidden_states = self.mlp(hidden_states) hidden_states = self.post_feedforward_layernorm(hidden_states) hidden_states = residual + hidden_states return hidden_states @support_torch_compile class OlmoHybridModel(nn.Module): hf_to_vllm_mapper = WeightsMapper( orig_to_new_stacked={ ".self_attn.q_proj": (".self_attn.qkv_proj", "q"), ".self_attn.k_proj": (".self_attn.qkv_proj", "k"), ".self_attn.v_proj": (".self_attn.qkv_proj", "v"), ".linear_attn.q_proj": (".linear_attn.in_proj_qkvg", 0), ".linear_attn.k_proj": (".linear_attn.in_proj_qkvg", 1), ".linear_attn.v_proj": (".linear_attn.in_proj_qkvg", 2), ".linear_attn.g_proj": (".linear_attn.in_proj_qkvg", 3), ".q_conv1d": (".conv1d", 0), ".k_conv1d": (".conv1d", 1), ".v_conv1d": (".conv1d", 2), ".gate_proj": (".gate_up_proj", 0), ".up_proj": (".gate_up_proj", 1), } ) def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""): super().__init__() self.config = vllm_config.model_config.hf_config self.embed_tokens = VocabParallelEmbedding( self.config.vocab_size, self.config.hidden_size, prefix=f"{prefix}.embed_tokens", ) self.start_layer, self.end_layer, self.layers = make_layers( self.config.num_hidden_layers, lambda prefix: OlmoHybridDecoderLayer( vllm_config=vllm_config, prefix=prefix ), prefix=f"{prefix}.layers", ) self.norm = RMSNorm( self.config.hidden_size, eps=self.config.rms_norm_eps, ) self.make_empty_intermediate_tensors = make_empty_intermediate_tensors_factory( ["hidden_states"], self.config.hidden_size ) def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor: return self.embed_tokens(input_ids) def forward( self, input_ids: torch.Tensor, positions: torch.Tensor, intermediate_tensors: IntermediateTensors | None, inputs_embeds: torch.Tensor | None = None, ) -> torch.Tensor | IntermediateTensors: if get_pp_group().is_first_rank: if inputs_embeds is not None: hidden_states = inputs_embeds else: hidden_states = self.embed_tokens(input_ids) else: assert intermediate_tensors is not None hidden_states = intermediate_tensors["hidden_states"] assert isinstance(hidden_states, torch.Tensor) for layer in islice(self.layers, self.start_layer, self.end_layer): hidden_states = layer(positions, hidden_states) if not get_pp_group().is_last_rank: return IntermediateTensors({"hidden_states": hidden_states}) hidden_states = self.norm(hidden_states) return hidden_states def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: loader = AutoWeightsLoader(self) return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper) class OlmoHybridForCausalLM( nn.Module, HasInnerState, SupportsPP, SupportsLoRA, IsHybrid ): packed_modules_mapping = { "qkv_proj": ["q_proj", "k_proj", "v_proj"], "gate_up_proj": ["gate_proj", "up_proj"], "in_proj_qkvg": ["q_proj", "k_proj", "v_proj", "g_proj"], } def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""): super().__init__() config = vllm_config.model_config.hf_config self.config = config self.vllm_config = vllm_config self.model_config = vllm_config.model_config self.model = OlmoHybridModel( vllm_config=vllm_config, prefix=maybe_prefix(prefix, "model") ) self.lm_head = ParallelLMHead( config.vocab_size, config.hidden_size, quant_config=vllm_config.quant_config, prefix=maybe_prefix(prefix, "lm_head"), ) if config.tie_word_embeddings: self.lm_head = self.lm_head.tie_weights(self.model.embed_tokens) self.logits_processor = LogitsProcessor(config.vocab_size) self.make_empty_intermediate_tensors = ( self.model.make_empty_intermediate_tensors ) def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor: return self.model.embed_input_ids(input_ids) def forward( self, input_ids: torch.Tensor, positions: torch.Tensor, intermediate_tensors: IntermediateTensors | None = None, inputs_embeds: torch.Tensor | None = None, ) -> torch.Tensor | IntermediateTensors: hidden_states = self.model( input_ids=input_ids, positions=positions, intermediate_tensors=intermediate_tensors, inputs_embeds=inputs_embeds, ) return hidden_states def compute_logits( self, hidden_states: torch.Tensor, ) -> torch.Tensor | None: logits = self.logits_processor(self.lm_head, hidden_states) return logits @classmethod def get_mamba_state_dtype_from_config( cls, vllm_config: "VllmConfig", ) -> tuple[torch.dtype, torch.dtype]: return MambaStateDtypeCalculator.gated_delta_net_state_dtype( vllm_config.model_config.dtype, vllm_config.cache_config.mamba_cache_dtype, vllm_config.cache_config.mamba_ssm_cache_dtype, ) @classmethod def get_mamba_state_shape_from_config( cls, vllm_config: "VllmConfig" ) -> tuple[tuple[int, int], tuple[int, int]]: parallel_config = vllm_config.parallel_config hf_config = vllm_config.model_config.hf_config tp_size = parallel_config.tensor_parallel_size num_spec = ( vllm_config.speculative_config.num_speculative_tokens if vllm_config.speculative_config else 0 ) return MambaStateShapeCalculator.gated_delta_net_state_shape( tp_size, hf_config.linear_num_key_heads, hf_config.linear_num_value_heads, hf_config.linear_key_head_dim, hf_config.linear_value_head_dim, hf_config.linear_conv_kernel_dim, num_spec, ) @classmethod def get_mamba_state_copy_func(cls) -> tuple[MambaStateCopyFunc, MambaStateCopyFunc]: return MambaStateCopyFuncCalculator.gated_delta_net_state_copy_func() def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]): loader = AutoWeightsLoader(self) return loader.load_weights(weights)