[Model] Remove ten deprecated model architectures (#53608)

Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
Co-authored-by: Xianbao QIAN <xianbao.qian@gmail.com>
Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Harry Mellor
2026-08-24 17:06:00 -07:00
committed by GitHub
co-authored by Xianbao QIAN Claude Opus 5
parent a0f1b9ad05
commit 7ca336929c
46 changed files with 49 additions and 6620 deletions
+1 -2
View File
@@ -255,8 +255,7 @@ class ApertusDecoderLayer(nn.Module):
# Apertus defaults to causal attention as it is a decoder-only model.
# You can override the HF config with `is_causal=False` to enable
# bidirectional attention, which is used in some embedding models
# (e.g. parasail-ai/GritLM-7B-vllm)
# bidirectional attention, which is used in some embedding models.
if getattr(config, "is_causal", True):
attn_type = AttentionType.DECODER
else:
-593
View File
@@ -1,593 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Inference-only Snowflake Arctic model."""
from collections.abc import Iterable
from itertools import islice
import torch
from torch import nn
from vllm.compilation.decorators import support_torch_compile
from vllm.config import CacheConfig, VllmConfig
from vllm.distributed import (
get_pp_group,
get_tensor_model_parallel_rank,
get_tensor_model_parallel_world_size,
tensor_model_parallel_all_reduce,
)
from vllm.model_executor.layers.activation import SiluAndMul
from vllm.model_executor.layers.attention import Attention
from vllm.model_executor.layers.fused_moe import (
fused_experts,
fused_topk,
)
from vllm.model_executor.layers.layernorm import RMSNorm
from vllm.model_executor.layers.linear import (
MergedColumnParallelLinear,
QKVParallelLinear,
ReplicatedLinear,
RowParallelLinear,
)
from vllm.model_executor.layers.logits_processor import LogitsProcessor
from vllm.model_executor.layers.quantization import QuantizationConfig
from vllm.model_executor.layers.rotary_embedding import get_rope
from vllm.model_executor.layers.vocab_parallel_embedding import (
ParallelLMHead,
VocabParallelEmbedding,
)
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
from vllm.model_executor.utils import set_weight_attrs
from vllm.platforms import current_platform
from vllm.sequence import IntermediateTensors
from vllm.transformers_utils.configs.arctic import ArcticConfig
from .interfaces import SupportsPP, SupportsQuant
from .utils import (
AutoWeightsLoader,
extract_layer_index,
is_pp_missing_parameter,
make_empty_intermediate_tensors_factory,
make_layers,
maybe_prefix,
)
class ArcticMLP(nn.Module):
def __init__(
self,
config: ArcticConfig,
expert_id: int = -1,
is_residual_mlp: bool = False,
quant_config: QuantizationConfig | None = None,
reduce_results: bool = True,
prefix: str = "",
):
super().__init__()
self.hidden_size = config.hidden_size
self.expert_id = expert_id
self.ffn_dim = (
config.intermediate_size if not is_residual_mlp else self.hidden_size
)
self.w13 = MergedColumnParallelLinear(
self.hidden_size,
[self.ffn_dim] * 2,
bias=False,
quant_config=quant_config,
prefix=f"{prefix}.w13",
)
self.w2 = RowParallelLinear(
self.ffn_dim,
self.hidden_size,
bias=False,
reduce_results=reduce_results,
quant_config=quant_config,
prefix=f"{prefix}.w2",
)
if config.hidden_act != "silu":
raise ValueError(
f"Unsupported activation: {config.hidden_act}. "
"Only silu is supported for now."
)
self.act_fn = SiluAndMul()
def forward(self, hidden_states):
gate_up, _ = self.w13(hidden_states)
hidden_states = self.act_fn(gate_up)
hidden_states, _ = self.w2(hidden_states)
return hidden_states
class ArcticMoE(nn.Module):
"""
Model-parallel implementation of Arctic MoE Layer.
"""
def __init__(
self,
config: ArcticConfig,
tp_size: int | None = None,
params_dtype: torch.dtype | None = None,
quant_config: QuantizationConfig | None = None,
reduce_results: bool = True,
prefix: str = "",
):
super().__init__()
layer_id = extract_layer_index(prefix)
self.tp_size = tp_size or get_tensor_model_parallel_world_size()
self.hidden_size = config.hidden_size
self.num_experts = config.num_local_experts
self.layer_id = layer_id
self.top_k = config.num_experts_per_tok
self.intermediate_size = config.intermediate_size // self.tp_size
self.is_moe_layer = (layer_id + 1) % config.moe_layer_frequency == 0
self.reduce_results = reduce_results
# Some other parameters
if params_dtype is None:
params_dtype = torch.get_default_dtype()
self.params_dtype = params_dtype
if not self.is_moe_layer:
self.mlp = ArcticMLP(
config,
quant_config=quant_config,
reduce_results=reduce_results,
prefix=f"{prefix}.mlp",
)
else:
self.gate = ReplicatedLinear(
self.hidden_size,
self.num_experts,
bias=False,
params_dtype=self.params_dtype,
quant_config=quant_config,
prefix=f"{prefix}.gate",
)
self.ws = nn.Parameter(
torch.empty(
self.num_experts,
2 * self.intermediate_size,
self.hidden_size,
device=current_platform.device_type,
dtype=self.params_dtype,
)
)
self.w2s = nn.Parameter(
torch.empty(
self.num_experts,
self.hidden_size,
self.intermediate_size,
device=current_platform.device_type,
dtype=self.params_dtype,
)
)
set_weight_attrs(
self.ws,
{
"weight_loader": self.weight_loader,
},
)
set_weight_attrs(
self.w2s,
{
"weight_loader": self.weight_loader,
},
)
def weight_loader(
self,
param: nn.Parameter,
loaded_weight: torch.Tensor,
weight_name: str,
expert_id: int,
):
tp_rank = get_tensor_model_parallel_rank()
param_data = param.data
shard_size = self.intermediate_size
shard = slice(tp_rank * shard_size, (tp_rank + 1) * shard_size)
if weight_name.endswith("w1.weight"):
param_data[expert_id, 0:shard_size, :] = loaded_weight[shard, :]
if weight_name.endswith("w3.weight"):
param_data[expert_id, shard_size : 2 * shard_size, :] = loaded_weight[
shard, :
]
if weight_name.endswith("w2.weight"):
param_data[expert_id, :, :] = loaded_weight[:, shard]
def local_moe_fused(self, hidden_states: torch.Tensor) -> torch.Tensor:
num_tokens, hidden_size = hidden_states.shape
hidden_states = hidden_states.view(-1, self.hidden_size)
# router_logits: (num_tokens, n_experts)
router_logits, _ = self.gate(hidden_states)
do_normalize = self.top_k > 1
topk_weights, topk_ids, token_expert_indices = fused_topk(
hidden_states, router_logits, self.top_k, renormalize=do_normalize
)
final_hidden_states = fused_experts(
hidden_states,
self.ws,
self.w2s,
topk_weights,
topk_ids,
)
if self.reduce_results and self.tp_size > 1:
final_hidden_states = tensor_model_parallel_all_reduce(final_hidden_states)
return final_hidden_states.view(num_tokens, hidden_size)
def forward(self, hidden_states: torch.Tensor):
if self.is_moe_layer:
final_hidden_states = self.local_moe_fused(hidden_states)
else:
final_hidden_states = self.mlp(hidden_states)
return final_hidden_states
class ArcticAttention(nn.Module):
def __init__(
self,
config: ArcticConfig,
cache_config: CacheConfig | None = None,
quant_config: QuantizationConfig | None = None,
prefix: str = "",
):
super().__init__()
self.config = config
self.hidden_size = config.hidden_size
tp_size = get_tensor_model_parallel_world_size()
self.total_num_heads = config.num_attention_heads
assert self.total_num_heads % tp_size == 0
self.num_heads = self.total_num_heads // tp_size
self.total_num_kv_heads = config.num_key_value_heads
if self.total_num_kv_heads >= tp_size:
assert self.total_num_kv_heads % tp_size == 0
else:
assert tp_size % self.total_num_kv_heads == 0
self.num_kv_heads = max(1, self.total_num_kv_heads // tp_size)
self.head_dim = self.hidden_size // self.total_num_heads
self.q_size = self.num_heads * self.head_dim
self.kv_size = self.num_kv_heads * self.head_dim
self.max_position_embeddings = config.max_position_embeddings
self.scaling = self.head_dim**-0.5
self.qkv_proj = QKVParallelLinear(
self.hidden_size,
self.head_dim,
self.total_num_heads,
self.total_num_kv_heads,
bias=False,
quant_config=quant_config,
prefix=f"{prefix}.qkv_proj",
)
self.o_proj = RowParallelLinear(
self.total_num_heads * self.head_dim,
self.hidden_size,
bias=False,
reduce_results=True,
quant_config=quant_config,
prefix=f"{prefix}.o_proj",
)
self.rotary_emb = get_rope(
self.head_dim,
max_position=self.max_position_embeddings,
rope_parameters=config.rope_parameters,
is_neox_style=True,
)
self.attn = Attention(
self.num_heads,
self.head_dim,
self.scaling,
num_kv_heads=self.num_kv_heads,
cache_config=cache_config,
quant_config=quant_config,
prefix=f"{prefix}.attn",
)
def forward(
self,
positions: torch.Tensor,
hidden_states: torch.Tensor,
) -> torch.Tensor:
qkv, _ = self.qkv_proj(hidden_states)
q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1)
q, k = self.rotary_emb(positions, q, k)
attn_output = self.attn(q, k, v)
output, _ = self.o_proj(attn_output)
return output
class ArcticDecoderLayer(nn.Module):
def __init__(
self,
config: ArcticConfig,
cache_config: CacheConfig | None = None,
quant_config: QuantizationConfig | None = None,
prefix: str = "",
) -> None:
super().__init__()
self.hidden_size = config.hidden_size
layer_idx = extract_layer_index(prefix)
is_moe_layer = (layer_idx + 1) % config.moe_layer_frequency == 0
self.use_residual = config.use_residual and is_moe_layer
self.self_attn = ArcticAttention(
config,
cache_config,
quant_config=quant_config,
prefix=f"{prefix}.self_attn",
)
self.block_sparse_moe = ArcticMoE(
config,
quant_config=quant_config,
reduce_results=(not self.use_residual),
prefix=f"{prefix}.block_sparse_moe",
)
self.input_layernorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
self.post_attention_layernorm = RMSNorm(
config.hidden_size, eps=config.rms_norm_eps
)
if self.use_residual:
self.residual_layernorm = RMSNorm(
config.hidden_size, eps=config.rms_norm_eps
)
self.residual_mlp = ArcticMLP(
config,
is_residual_mlp=True,
reduce_results=False,
prefix=f"{prefix}.residual_mlp",
)
def forward(
self,
positions: torch.Tensor,
hidden_states: torch.Tensor,
) -> torch.Tensor:
residual_input = hidden_states
hidden_states = self.input_layernorm(hidden_states)
hidden_states = self.self_attn(
positions=positions,
hidden_states=hidden_states,
)
hidden_states = residual_input + hidden_states
residual_attn = hidden_states
if self.use_residual:
hidden_states = self.residual_layernorm(hidden_states)
hidden_states = self.residual_mlp(hidden_states)
residual_mlp = hidden_states
hidden_states = self.post_attention_layernorm(residual_input)
hidden_states = self.block_sparse_moe(hidden_states)
hidden_states = residual_mlp + hidden_states
hidden_states = tensor_model_parallel_all_reduce(hidden_states)
hidden_states = residual_attn + hidden_states
else:
hidden_states = self.post_attention_layernorm(hidden_states)
hidden_states = self.block_sparse_moe(hidden_states)
hidden_states = residual_attn + hidden_states
return hidden_states
@support_torch_compile
class ArcticModel(nn.Module):
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
super().__init__()
config = vllm_config.model_config.hf_config
cache_config = vllm_config.cache_config
quant_config = vllm_config.quant_config
self.config = config
self.vocab_size = config.vocab_size
self.embed_tokens = VocabParallelEmbedding(
self.vocab_size, config.hidden_size, org_num_embeddings=self.vocab_size
)
self.start_layer, self.end_layer, self.layers = make_layers(
config.num_hidden_layers,
lambda prefix: ArcticDecoderLayer(
config, cache_config, quant_config, prefix=prefix
),
prefix=f"{prefix}.layers",
)
self._attn_implementation = config._attn_implementation
self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
self.make_empty_intermediate_tensors = make_empty_intermediate_tensors_factory(
["hidden_states"], config.hidden_size
)
def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
return self.embed_tokens(input_ids)
def forward(
self,
input_ids: torch.Tensor | None,
positions: torch.Tensor,
intermediate_tensors: IntermediateTensors | None,
inputs_embeds: torch.Tensor | None = None,
) -> torch.Tensor | IntermediateTensors:
if get_pp_group().is_first_rank:
if inputs_embeds is not None:
hidden_states = inputs_embeds
else:
hidden_states = self.embed_input_ids(input_ids)
else:
assert intermediate_tensors is not None
hidden_states = intermediate_tensors["hidden_states"]
for layer in islice(self.layers, self.start_layer, self.end_layer):
hidden_states = layer(positions, hidden_states)
if not get_pp_group().is_last_rank:
return IntermediateTensors({"hidden_states": hidden_states})
hidden_states = self.norm(hidden_states)
return hidden_states
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
stacked_params_mapping = [
# (param_name, shard_name, shard_id)
("qkv_proj", "q_proj", "q"),
("qkv_proj", "k_proj", "k"),
("qkv_proj", "v_proj", "v"),
]
mlp_params_mapping: list[tuple[str, str, int]] = []
expert_params_mapping: list[tuple[str, str, int]] = []
for layer in range(self.config.num_hidden_layers):
is_moe_layer = (layer + 1) % self.config.moe_layer_frequency == 0
if is_moe_layer and self.config.use_residual:
mlp_params_mapping.append(
(
f"layers.{layer}.residual_mlp.w13.weight",
f"layers.{layer}.residual_mlp.w1.weight",
0,
)
)
mlp_params_mapping.append(
(
f"layers.{layer}.residual_mlp.w13.weight",
f"layers.{layer}.residual_mlp.w3.weight",
1,
)
)
if is_moe_layer:
for expert_id in range(self.config.num_local_experts):
expert_params_mapping.append(
("ws", f"experts.{expert_id}.w1.weight", expert_id)
)
expert_params_mapping.append(
("w2s", f"experts.{expert_id}.w2.weight", expert_id)
)
expert_params_mapping.append(
("ws", f"experts.{expert_id}.w3.weight", expert_id)
)
else:
mlp_params_mapping.append(
(
f"layers.{layer}.block_sparse_moe.mlp.w13.weight",
f"layers.{layer}.block_sparse_moe.mlp.w1.weight",
0,
)
)
mlp_params_mapping.append(
(
f"layers.{layer}.block_sparse_moe.mlp.w13.weight",
f"layers.{layer}.block_sparse_moe.mlp.w3.weight",
1,
)
)
params_dict = dict(self.named_parameters())
loaded_params: set[str] = set()
for name, loaded_weight in weights:
for param_name, weight_name, shard_id in stacked_params_mapping:
if weight_name not in name:
continue
name = name.replace(weight_name, param_name)
# Skip loading extra bias for GPTQ models.
if name.endswith(".bias") and name not in params_dict:
continue
if is_pp_missing_parameter(name, self):
continue
param = params_dict[name]
weight_loader = param.weight_loader
weight_loader(param, loaded_weight, shard_id)
break
else:
for param_name, weight_name, mlp_shard_id in mlp_params_mapping:
if weight_name not in name:
continue
name = name.replace(weight_name, param_name)
if is_pp_missing_parameter(name, self):
continue
param = params_dict[name]
weight_loader = param.weight_loader
weight_loader(param, loaded_weight, mlp_shard_id)
break
else:
for param_name, weight_name, expert_id in expert_params_mapping:
if weight_name not in name:
continue
name = name.replace(weight_name, param_name)
if is_pp_missing_parameter(name, self):
continue
param = params_dict[name]
weight_loader = param.weight_loader
weight_loader(
param, loaded_weight, weight_name, expert_id=expert_id
)
break
else:
if name.endswith(".bias") and name not in params_dict:
continue
if is_pp_missing_parameter(name, self):
continue
param = params_dict[name]
weight_loader = getattr(
param, "weight_loader", default_weight_loader
)
weight_loader(param, loaded_weight)
loaded_params.add(name)
return loaded_params
class ArcticForCausalLM(nn.Module, SupportsPP, SupportsQuant):
packed_modules_mapping = {"qkv_proj": ["q_proj", "k_proj", "v_proj"]}
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
super().__init__()
config = vllm_config.model_config.hf_config
quant_config = vllm_config.quant_config
self.config = config
self.model = ArcticModel(
vllm_config=vllm_config, prefix=maybe_prefix(prefix, "model")
)
self.vocab_size = config.vocab_size
self.lm_head = ParallelLMHead(
self.vocab_size,
config.hidden_size,
quant_config=quant_config,
prefix=maybe_prefix(prefix, "lm_head"),
)
if self.config.tie_word_embeddings:
self.lm_head = self.lm_head.tie_weights(self.model.embed_tokens)
self.num_experts = config.num_local_experts
self.num_experts_per_tok = config.num_experts_per_tok
self.logits_processor = LogitsProcessor(config.vocab_size)
self.make_empty_intermediate_tensors = (
self.model.make_empty_intermediate_tensors
)
def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
return self.model.embed_input_ids(input_ids)
def forward(
self,
input_ids: torch.Tensor | None,
positions: torch.Tensor,
intermediate_tensors: IntermediateTensors | None = None,
inputs_embeds: torch.Tensor | None = None,
) -> torch.Tensor | IntermediateTensors:
hidden_states = self.model(
input_ids, positions, intermediate_tensors, inputs_embeds
)
return hidden_states
def compute_logits(
self,
hidden_states: torch.Tensor,
) -> torch.Tensor | None:
logits = self.logits_processor(self.lm_head, hidden_states)
return logits
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
loader = AutoWeightsLoader(self)
return loader.load_weights(weights)
File diff suppressed because it is too large Load Diff
-734
View File
@@ -1,734 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Inference-only Cheers (UMM) model compatible with HuggingFace weights.
Cheers is a unified multimodal model for image understanding and generation.
For vLLM, we focus on the image understanding (vision-to-text) capabilities.
The image generation part (gen_projector, hi_gate, etc.) is not supported,
but the VAE encoder + decoder projector are required for image understanding.
"""
import math
from collections.abc import Iterable, Mapping, Sequence
from typing import Any, Literal, TypeAlias
import torch
import torch.nn as nn
import torch.nn.functional as F
from einops import rearrange
from vllm.config import VllmConfig
from vllm.config.multimodal import BaseDummyOptions
from vllm.inputs import MultiModalDataDict
from vllm.logger import init_logger
from vllm.model_executor.layers.quantization import QuantizationConfig
from vllm.multimodal import MULTIMODAL_REGISTRY
from vllm.multimodal.inputs import (
MultiModalFieldConfig,
MultiModalKwargsItems,
)
from vllm.multimodal.parse import MultiModalDataItems
from vllm.multimodal.processing import (
BaseDummyInputsBuilder,
BaseMultiModalProcessor,
BaseProcessingInfo,
PromptReplacement,
)
from vllm.sequence import IntermediateTensors
from vllm.transformers_utils.processors.cheers import CheersProcessor
from vllm.utils.tensor_schema import TensorSchema
from .interfaces import (
MultiModalEmbeddings,
SupportsLoRA,
SupportsMultiModal,
SupportsPP,
)
from .siglip import SiglipVisionModel
from .utils import (
AutoWeightsLoader,
WeightsMapper,
init_vllm_registered_model,
maybe_prefix,
)
logger = init_logger(__name__)
# ── VAE components (needed for image understanding pipeline) ────────
def _swish(x: torch.Tensor) -> torch.Tensor:
return x * torch.sigmoid(x)
class _AttnBlock(nn.Module):
def __init__(self, in_channels: int):
super().__init__()
self.norm = nn.GroupNorm(32, in_channels, eps=1e-6, affine=True)
self.q = nn.Conv2d(in_channels, in_channels, 1)
self.k = nn.Conv2d(in_channels, in_channels, 1)
self.v = nn.Conv2d(in_channels, in_channels, 1)
self.proj_out = nn.Conv2d(in_channels, in_channels, 1)
def forward(self, x: torch.Tensor) -> torch.Tensor:
h_ = self.norm(x)
q = self.q(h_)
k = self.k(h_)
v = self.v(h_)
b, c, h, w = q.shape
q = rearrange(q, "b c h w -> b 1 (h w) c").contiguous()
k = rearrange(k, "b c h w -> b 1 (h w) c").contiguous()
v = rearrange(v, "b c h w -> b 1 (h w) c").contiguous()
h_ = F.scaled_dot_product_attention(q, k, v)
h_ = rearrange(h_, "b 1 (h w) c -> b c h w", h=h, w=w, c=c, b=b)
return x + self.proj_out(h_)
class _ResnetBlock(nn.Module):
def __init__(self, in_channels: int, out_channels: int):
super().__init__()
self.in_channels = in_channels
self.out_channels = out_channels
self.norm1 = nn.GroupNorm(32, in_channels, eps=1e-6, affine=True)
self.conv1 = nn.Conv2d(in_channels, out_channels, 3, 1, 1)
self.norm2 = nn.GroupNorm(32, out_channels, eps=1e-6, affine=True)
self.conv2 = nn.Conv2d(out_channels, out_channels, 3, 1, 1)
if in_channels != out_channels:
self.nin_shortcut = nn.Conv2d(in_channels, out_channels, 1)
def forward(self, x: torch.Tensor) -> torch.Tensor:
h = _swish(self.norm1(x))
h = self.conv1(h)
h = _swish(self.norm2(h))
h = self.conv2(h)
if self.in_channels != self.out_channels:
x = self.nin_shortcut(x)
return x + h
class _Downsample(nn.Module):
def __init__(self, in_channels: int):
super().__init__()
self.conv = nn.Conv2d(in_channels, in_channels, 3, stride=2, padding=0)
def forward(self, x: torch.Tensor) -> torch.Tensor:
x = F.pad(x, (0, 1, 0, 1), mode="constant", value=0)
return self.conv(x)
class _Upsample(nn.Module):
def __init__(self, in_channels: int):
super().__init__()
self.conv = nn.Conv2d(in_channels, in_channels, 3, 1, 1)
def forward(self, x: torch.Tensor) -> torch.Tensor:
x = F.interpolate(x, scale_factor=2.0, mode="nearest")
return self.conv(x)
_VAE_ENCODER_DEFAULTS = {
"in_channels": 3,
"ch": 128,
"ch_mult": [1, 2, 4, 4],
"num_res_blocks": 2,
"z_channels": 32,
}
_VAE_DECODER_DEFAULTS = {
"in_channels": 3,
"out_ch": 3,
"ch": 128,
"ch_mult": [1, 2, 4, 4],
"num_res_blocks": 2,
"z_channels": 32,
}
def _cfg(config, key, defaults=None):
"""Access config attribute whether it's a dict or namespace object."""
if isinstance(config, dict):
if key in config:
return config[key]
if defaults and key in defaults:
return defaults[key]
raise KeyError(f"Key '{key}' not found in config dict: {list(config.keys())}")
return getattr(config, key)
class CheersVAEEncoder(nn.Module):
"""VAE encoder from the Cheers/UMM model."""
def __init__(self, config):
super().__init__()
d = _VAE_ENCODER_DEFAULTS
ch = _cfg(config, "ch", d)
ch_mult = _cfg(config, "ch_mult", d)
num_res_blocks = _cfg(config, "num_res_blocks", d)
z_channels = _cfg(config, "z_channels", d)
in_channels = _cfg(config, "in_channels", d)
num_resolutions = len(ch_mult)
self.quant_conv = nn.Conv2d(2 * z_channels, 2 * z_channels, 1)
self.conv_in = nn.Conv2d(in_channels, ch, 3, 1, 1)
in_ch_mult = (1,) + tuple(ch_mult)
self.down = nn.ModuleList()
block_in = ch
for i_level in range(num_resolutions):
block = nn.ModuleList()
attn = nn.ModuleList()
block_in = ch * in_ch_mult[i_level]
block_out = ch * ch_mult[i_level]
for _ in range(num_res_blocks):
block.append(_ResnetBlock(block_in, block_out))
block_in = block_out
down = nn.Module()
down.block = block
down.attn = attn
if i_level != num_resolutions - 1:
down.downsample = _Downsample(block_in)
self.down.append(down)
self.mid = nn.Module()
self.mid.block_1 = _ResnetBlock(block_in, block_in)
self.mid.attn_1 = _AttnBlock(block_in)
self.mid.block_2 = _ResnetBlock(block_in, block_in)
self.norm_out = nn.GroupNorm(32, block_in, eps=1e-6, affine=True)
self.conv_out = nn.Conv2d(block_in, 2 * z_channels, 3, 1, 1)
self._num_resolutions = num_resolutions
self._num_res_blocks = num_res_blocks
def forward(self, x: torch.Tensor) -> torch.Tensor:
hs = [self.conv_in(x)]
for i_level in range(self._num_resolutions):
for i_block in range(self._num_res_blocks):
h = self.down[i_level].block[i_block](hs[-1])
if len(self.down[i_level].attn) > 0:
h = self.down[i_level].attn[i_block](h)
hs.append(h)
if hasattr(self.down[i_level], "downsample"):
hs.append(self.down[i_level].downsample(hs[-1]))
h = hs[-1]
h = self.mid.block_1(h)
h = self.mid.attn_1(h)
h = self.mid.block_2(h)
h = _swish(self.norm_out(h))
h = self.conv_out(h)
h = self.quant_conv(h)
return h
class CheersVAEDecoder(nn.Module):
"""VAE decoder (used inside VAEDecoderProjector)."""
def __init__(self, config):
super().__init__()
d = _VAE_DECODER_DEFAULTS
ch = _cfg(config, "ch", d)
ch_mult = _cfg(config, "ch_mult", d)
num_res_blocks = _cfg(config, "num_res_blocks", d)
z_channels = _cfg(config, "z_channels", d)
out_ch = _cfg(config, "out_ch", d)
num_resolutions = len(ch_mult)
self.post_quant_conv = nn.Conv2d(z_channels, z_channels, 1)
block_in = ch * ch_mult[num_resolutions - 1]
self.conv_in = nn.Conv2d(z_channels, block_in, 3, 1, 1)
self.mid = nn.Module()
self.mid.block_1 = _ResnetBlock(block_in, block_in)
self.mid.attn_1 = _AttnBlock(block_in)
self.mid.block_2 = _ResnetBlock(block_in, block_in)
self.up = nn.ModuleList()
for i_level in reversed(range(num_resolutions)):
block = nn.ModuleList()
attn = nn.ModuleList()
block_out = ch * ch_mult[i_level]
for _ in range(num_res_blocks + 1):
block.append(_ResnetBlock(block_in, block_out))
block_in = block_out
up = nn.Module()
up.block = block
up.attn = attn
if i_level != 0:
up.upsample = _Upsample(block_in)
self.up.insert(0, up)
self.norm_out = nn.GroupNorm(32, block_in, eps=1e-6, affine=True)
self.conv_out = nn.Conv2d(block_in, out_ch, 3, 1, 1)
self._num_resolutions = num_resolutions
self._num_res_blocks = num_res_blocks
def forward(self, z: torch.Tensor) -> torch.Tensor:
z = self.post_quant_conv(z)
upscale_dtype = next(self.up.parameters()).dtype
h = self.conv_in(z)
h = self.mid.block_1(h)
h = self.mid.attn_1(h)
h = self.mid.block_2(h)
h = h.to(upscale_dtype)
for i_level in reversed(range(self._num_resolutions)):
for i_block in range(self._num_res_blocks + 1):
h = self.up[i_level].block[i_block](h)
if len(self.up[i_level].attn) > 0:
h = self.up[i_level].attn[i_block](h)
if i_level != 0:
h = self.up[i_level].upsample(h)
h = _swish(self.norm_out(h))
return self.conv_out(h)
class CheersVAEModel(nn.Module):
"""VAE model with encoder only (for image understanding)."""
def __init__(self, config):
super().__init__()
enc_cfg = _cfg(config, "vae_encoder_config")
self.encoder = CheersVAEEncoder(enc_cfg)
self.ps = [2, 2]
z_ch = _cfg(enc_cfg, "z_channels", _VAE_ENCODER_DEFAULTS)
self.bn = nn.BatchNorm2d(
math.prod(self.ps) * z_ch,
eps=1e-4,
momentum=0.1,
affine=False,
track_running_stats=True,
)
def encode(self, x: torch.Tensor) -> torch.Tensor:
self.bn.eval()
moments = self.encoder(x)
mean = torch.chunk(moments, 2, dim=1)[0]
z = rearrange(
mean,
"... c (i pi) (j pj) -> ... (c pi pj) i j",
pi=self.ps[0],
pj=self.ps[1],
)
return self.bn(z)
class CheersVAEDecoderProjector(nn.Module):
"""VAE decoder projector that converts latent back to pixel-like space."""
def __init__(self, config):
super().__init__()
dec_cfg = _cfg(config, "vae_decoder_config")
enc_cfg = _cfg(config, "vae_encoder_config")
self.decoder = CheersVAEDecoder(dec_cfg)
self.ps = [2, 2]
z_ch = _cfg(enc_cfg, "z_channels", _VAE_ENCODER_DEFAULTS)
self.bn = nn.BatchNorm2d(
math.prod(self.ps) * z_ch,
eps=1e-4,
momentum=0.1,
affine=False,
track_running_stats=True,
)
def forward(self, z: torch.Tensor) -> torch.Tensor:
self.bn.eval()
s = torch.sqrt(self.bn.running_var.view(1, -1, 1, 1) + 1e-4)
m = self.bn.running_mean.view(1, -1, 1, 1)
z = z * s + m
z = rearrange(
z,
"... (c pi pj) i j -> ... c (i pi) (j pj)",
pi=self.ps[0],
pj=self.ps[1],
)
return self.decoder(z)
class CheersImagePixelInputs(TensorSchema):
"""
Dimensions:
- bn: Batch size * number of images
- c: Number of channels (3)
- h: Height of each image
- w: Width of each image
"""
type: Literal["pixel_values"]
pixel_values: torch.Tensor # Shape: (bn, 3, h, w)
CheersImageInputs: TypeAlias = CheersImagePixelInputs
class CheersUndProjector(nn.Module):
"""Understanding projector that maps vision features to LLM dimension
with 2x2 spatial compression (4x token reduction)."""
def __init__(
self,
image_embed_dim: int,
text_embed_dim: int,
compression_factor: tuple[int, int] = (2, 2),
quant_config: QuantizationConfig | None = None,
prefix: str = "",
):
super().__init__()
self.image_embed_dim = image_embed_dim
self.text_embed_dim = text_embed_dim
self.compression_factor = compression_factor
self.layernorm = nn.LayerNorm(image_embed_dim)
hidden_size = image_embed_dim * (compression_factor[0] * compression_factor[1])
self.mlp = nn.Sequential(
nn.Linear(hidden_size, hidden_size),
nn.GELU(),
nn.Linear(hidden_size, text_embed_dim),
)
def forward(self, x: torch.Tensor) -> torch.Tensor:
x = self.layernorm(x)
height = width = int(x.size(1) ** 0.5)
x = x.permute(0, 2, 1).unflatten(-1, (height, width))
batch_size, dim, height, width = x.shape
unfolded = x.unfold(
2, self.compression_factor[0], self.compression_factor[0]
).unfold(3, self.compression_factor[1], self.compression_factor[1])
unfolded = unfolded.contiguous().view(
batch_size,
dim,
-1,
self.compression_factor[0] * self.compression_factor[1],
)
unfolded = (
unfolded.permute(0, 2, 3, 1)
.contiguous()
.view(
batch_size,
-1,
dim * self.compression_factor[0] * self.compression_factor[1],
)
)
return self.mlp(unfolded)
class CheersProcessingInfo(BaseProcessingInfo):
"""Processing information for Cheers model."""
def get_hf_processor(self, **kwargs: object) -> CheersProcessor:
from vllm.transformers_utils.processor import cached_get_image_processor
image_processor = cached_get_image_processor(
self.ctx.model_config.model,
revision=self.ctx.model_config.revision,
trust_remote_code=self.ctx.model_config.trust_remote_code,
)
tokenizer = self.get_tokenizer()
return CheersProcessor(
image_processor=image_processor,
tokenizer=tokenizer,
**kwargs,
)
def get_supported_mm_limits(self) -> Mapping[str, int | None]:
return {"image": None}
def get_mm_max_tokens_per_item(
self,
seq_len: int,
mm_counts: Mapping[str, int],
) -> Mapping[str, int]:
hf_config = self.get_hf_config()
vit_config = hf_config.vision_representation_config
patch_size = vit_config.patch_size
image_size = vit_config.image_size
num_patches = (image_size // patch_size) ** 2
# After 2x2 compression, tokens reduce by 4x
num_tokens = num_patches // 4
return {"image": num_tokens}
def get_num_image_tokens(
self,
*,
image_width: int,
image_height: int,
) -> int:
hf_config = self.get_hf_config()
vit_config = hf_config.vision_representation_config
patch_size = vit_config.patch_size
image_size = vit_config.image_size
num_patches = (image_size // patch_size) ** 2
return num_patches // 4
class CheersDummyInputsBuilder(BaseDummyInputsBuilder[CheersProcessingInfo]):
"""Build dummy inputs for Cheers model profiling."""
def get_dummy_text(self, mm_counts: Mapping[str, int]) -> str:
num_images = mm_counts.get("image", 0)
return "<|image_pad|>" * num_images
def get_dummy_mm_data(
self,
seq_len: int,
mm_counts: Mapping[str, int],
mm_options: Mapping[str, BaseDummyOptions] | None = None,
) -> MultiModalDataDict:
num_images = mm_counts.get("image", 0)
hf_config = self.info.get_hf_config()
vit_config = hf_config.vision_representation_config
image_size = vit_config.image_size
image_overrides = mm_options.get("image") if mm_options else None
return {
"image": self._get_dummy_images(
width=image_size,
height=image_size,
num_images=num_images,
overrides=image_overrides,
),
}
class CheersMultiModalProcessor(BaseMultiModalProcessor[CheersProcessingInfo]):
"""Multimodal processor for Cheers model."""
def _get_prompt_updates(
self,
mm_items: MultiModalDataItems,
hf_processor_mm_kwargs: Mapping[str, Any],
out_mm_kwargs: MultiModalKwargsItems,
) -> Sequence[PromptReplacement]:
hf_config = self.info.get_hf_config()
vit_config = hf_config.vision_representation_config
patch_size = vit_config.patch_size
image_size = vit_config.image_size
tokenizer = self.info.get_tokenizer()
image_token_id = tokenizer.get_vocab().get("<|image_pad|>")
if image_token_id is None:
raise ValueError(
"Image token '<|image_pad|>' not found in tokenizer vocabulary"
)
def get_replacement_cheers(item_idx: int):
num_patches = (image_size // patch_size) ** 2
num_tokens = num_patches // 4
return [image_token_id] * num_tokens
return [
PromptReplacement(
modality="image",
target=[image_token_id],
replacement=get_replacement_cheers,
)
]
def _get_mm_fields_config(
self,
hf_inputs: Any,
hf_processor_mm_kwargs: Mapping[str, object],
) -> Mapping[str, MultiModalFieldConfig]:
return {
"pixel_values": MultiModalFieldConfig.batched("image"),
}
@MULTIMODAL_REGISTRY.register_processor(
CheersMultiModalProcessor,
info=CheersProcessingInfo,
dummy_inputs=CheersDummyInputsBuilder,
)
class CheersForConditionalGeneration(
nn.Module, SupportsMultiModal, SupportsLoRA, SupportsPP
):
"""
Cheers: A unified multimodal model for image understanding and generation.
For vLLM, we focus on the image understanding (vision-to-text) capabilities.
The image generation part is not supported in vLLM.
"""
requires_raw_input_tokens = True
hf_to_vllm_mapper = WeightsMapper(
orig_to_new_prefix={
"model.language_model.": "language_model.model.",
"model.vision_representation.": "vision_representation.vision_model.",
"model.und_projector.": "und_projector.",
"model.vae_model.": "vae_model.",
"model.vae_decoder_projector.": "vae_decoder_projector.",
"lm_head.": "language_model.lm_head.",
}
)
@classmethod
def get_placeholder_str(cls, modality: str, i: int) -> str | None:
if modality.startswith("image"):
return "<|image_pad|>"
raise ValueError("Only image modality is supported")
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
super().__init__()
config = vllm_config.model_config.hf_config
quant_config = vllm_config.quant_config
multimodal_config = vllm_config.model_config.multimodal_config
if type(config).__name__ not in ("CheersConfig", "UMMConfig"):
raise ValueError(
f"Expected CheersConfig or UMMConfig, got {type(config).__name__}."
)
self.config = config
self.multimodal_config = multimodal_config
# The Cheers model's custom Qwen2Config defaults rope_theta to
# 1_000_000, but this isn't stored in the JSON. vLLM's standard
# Qwen2Config defaults to 10_000, causing a 100× mismatch.
# We must patch BOTH the attribute AND rope_parameters (which
# patch_rope_parameters may have already populated from the wrong
# default before __init__ runs).
_CHEERS_ROPE_THETA = 1_000_000.0
tc = config.text_config
old_theta = getattr(tc, "rope_theta", None)
if old_theta != _CHEERS_ROPE_THETA:
logger.info(
"Overriding text_config.rope_theta from %s to %s",
old_theta,
_CHEERS_ROPE_THETA,
)
tc.rope_theta = _CHEERS_ROPE_THETA
rp = getattr(tc, "rope_parameters", None)
if rp is not None and rp.get("rope_theta") != _CHEERS_ROPE_THETA:
logger.info(
"Overriding rope_parameters.rope_theta from %s to %s",
rp.get("rope_theta"),
_CHEERS_ROPE_THETA,
)
rp["rope_theta"] = _CHEERS_ROPE_THETA
with self._mark_language_model(vllm_config):
self.language_model = init_vllm_registered_model(
vllm_config=vllm_config,
hf_config=config.text_config,
prefix=maybe_prefix(prefix, "language_model"),
architectures=["Qwen2ForCausalLM"],
)
vit_config = config.vision_representation_config
with self._mark_tower_model(vllm_config, "image"):
self.vae_model = CheersVAEModel(config)
self.vae_decoder_projector = CheersVAEDecoderProjector(config)
self.vision_representation = SiglipVisionModel(
config=vit_config,
quant_config=quant_config,
prefix=maybe_prefix(prefix, "vision_representation"),
)
vit_hidden_size = vit_config.hidden_size
llm_hidden_size = config.text_config.hidden_size
self.und_projector = CheersUndProjector(
image_embed_dim=vit_hidden_size,
text_embed_dim=llm_hidden_size,
compression_factor=(2, 2),
quant_config=quant_config,
prefix=maybe_prefix(prefix, "und_projector"),
)
self.make_empty_intermediate_tensors = (
self.language_model.make_empty_intermediate_tensors
)
def _parse_and_validate_image_input(
self, **kwargs: object
) -> CheersImageInputs | None:
pixel_values = kwargs.pop("pixel_values", None)
if pixel_values is None:
return None
return CheersImagePixelInputs(
type="pixel_values",
pixel_values=pixel_values,
)
def _process_image_input(
self, image_input: CheersImageInputs
) -> tuple[torch.Tensor, ...]:
"""Process image inputs through VAE → SigLIP → projector pipeline.
HF native path: pixel_values → VAE.encode(t=1.0) → vae_decoder_projector
→ SigLIP → und_projector → text-space embeddings
"""
pixel_values = image_input["pixel_values"]
if pixel_values.ndim == 5:
batch_size, num_images, channels, height, width = pixel_values.shape
pixel_values = pixel_values.reshape(
batch_size * num_images, channels, height, width
)
with torch.no_grad():
vae_dtype = next(self.vae_model.parameters()).dtype
image_latent = self.vae_model.encode(pixel_values.to(dtype=vae_dtype))
image_pixel_hat = self.vae_decoder_projector(image_latent)
vision_features = self.vision_representation(image_pixel_hat)
vision_embeds = self.und_projector(vision_features)
return tuple(vision_embeds)
def embed_multimodal(self, **kwargs: object) -> MultiModalEmbeddings:
image_input = self._parse_and_validate_image_input(**kwargs)
if image_input is None:
return []
return self._process_image_input(image_input)
def forward(
self,
input_ids: torch.Tensor | None,
positions: torch.Tensor,
intermediate_tensors: IntermediateTensors | None = None,
inputs_embeds: torch.Tensor | None = None,
**kwargs: object,
) -> torch.Tensor | IntermediateTensors:
if intermediate_tensors is not None:
inputs_embeds = None
hidden_states = self.language_model.model(
input_ids=input_ids,
positions=positions,
intermediate_tensors=intermediate_tensors,
inputs_embeds=inputs_embeds,
)
return hidden_states
def compute_logits(
self,
hidden_states: torch.Tensor,
) -> torch.Tensor | None:
return self.language_model.compute_logits(hidden_states)
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
"""Load weights, keeping VAE encoder/decoder projector for understanding."""
skip_prefixes = [
"model.time_embed.",
"model.gen_projector.",
"model.hi_gate.",
"model.hi_projector.",
"model.vae_model.decoder.",
]
skip_keywords = [
"text_loss_fc",
]
filtered_weights = []
for name, tensor in weights:
if any(name.startswith(p) for p in skip_prefixes):
continue
if any(kw in name for kw in skip_keywords):
continue
filtered_weights.append((name, tensor))
loader = AutoWeightsLoader(self)
return loader.load_weights(filtered_weights, mapper=self.hf_to_vllm_mapper)
@@ -1,11 +1,10 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
Shared Conformer encoder components for FireRedASR2 and FireRedLID.
Conformer encoder components for FireRedASR2.
Both models use the same Conformer-based audio encoder architecture
The audio encoder architecture is
(Conv2dSubsampling → RelPositionalEncoding → N × RelPosEmbConformerBlock).
This module factors out the common building blocks to avoid duplication.
"""
import torch
@@ -288,7 +287,7 @@ class RelPosEmbConformerBlock(nn.Module):
class ConformerEncoder(nn.Module):
"""
Conformer encoder shared by FireRedASR2 and FireRedLID.
Conformer encoder used by FireRedASR2.
"""
def __init__(
@@ -1,149 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
# Copyright 2024 The vLLM team.
# Copyright 2024 Meta Platforms, Inc. and affiliates. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Llama model for fairseq2 weights."""
from collections.abc import Iterable
import torch
from torch.nn import Parameter
from vllm.config import VllmConfig
from vllm.distributed import (
get_tensor_model_parallel_rank,
get_tensor_model_parallel_world_size,
)
from vllm.model_executor.layers.linear import set_weight_attrs
from vllm.model_executor.models.llama import LlamaForCausalLM
from .utils import AutoWeightsLoader, WeightsMapper
class Fairseq2LlamaForCausalLM(LlamaForCausalLM):
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
super().__init__(vllm_config=vllm_config, prefix=prefix)
self.tp_rank = get_tensor_model_parallel_rank()
self.tp_size = get_tensor_model_parallel_world_size()
# For the model loader to read only the relevant checkpoint files
self.allow_patterns_overrides = [
# either the full checkpoint
"model.pt",
# or the tp-sharded checkpoint of the current rank
f"model.{self.tp_rank}.pt",
]
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
# fairseq2's serialization adds a wrapper to usual .pt state_dict's:
# { "model_key": my_model_name, "my_model_name": state_dict }
# which we first need to unpack
weights_wrapped = dict(weights)
weights = weights_wrapped[weights_wrapped["model_key"]].items() # type: ignore
# remap keys
fs2_to_vllm_mapper = WeightsMapper(
orig_to_new_prefix={
"decoder_frontend.embed.": "model.embed_tokens.",
"decoder.": "model.",
"final_proj.": "lm_head.",
},
orig_to_new_substr={
".self_attn_layer_norm.": ".input_layernorm.",
".ffn_layer_norm.": ".post_attention_layernorm.",
".self_attn.output_proj.": ".self_attn.o_proj.",
".ffn.gate_proj.": ".mlp.gate_proj.",
".ffn.inner_proj.": ".mlp.up_proj.",
".ffn.output_proj.": ".mlp.down_proj.",
".layer_norm.": ".norm.",
},
)
weights = fs2_to_vllm_mapper.apply(weights)
params = dict(self.named_parameters())
loader = AutoWeightsLoader(self)
return loader.load_weights(
self.reshape_fairseq2_weights(name, loaded_weight, params)
for name, loaded_weight in weights
)
def flag_sharded_weights(self, params: dict[str, Parameter]):
"""Sets the `is_sharded_weight` flag to True for all sharded weights"""
for name, param in params.items():
modules = name.split(".")
if "norm" in name and len(param.size()) < 2:
# layer norms are not sharded
continue
elif any(emb in modules for emb in ["embed_tokens", "lm_head"]):
# for now we repeat embedding layers for compatibility
continue
else:
# all other layers are sharded
set_weight_attrs(param, {"is_sharded_weight": True})
def reshape_fairseq2_weights(
self,
name: str,
loaded_weight: torch.Tensor,
params: dict[str, Parameter],
) -> tuple[str, torch.Tensor]:
"""Reshape fairseq2's weights."""
def permute(w: torch.Tensor, n_heads: int) -> torch.Tensor:
attn_in = self.config.head_dim * n_heads
# check for a sharded weight on dim 0
if attn_in // self.tp_size == w.size()[0]:
attn_in //= self.tp_size
n_heads //= self.tp_size
attn_out = self.config.hidden_size
return (
w.view(n_heads, attn_in // n_heads // 2, 2, attn_out)
.transpose(1, 2)
.reshape(attn_in, attn_out)
)
modules = name.split(".")
# rotary embeds should be sliced
if "k_proj" in modules:
loaded_weight = permute(loaded_weight, self.config.num_key_value_heads)
elif "q_proj" in modules:
loaded_weight = permute(loaded_weight, self.config.num_attention_heads)
# We make the loaded weights compatible with both
# full checkpoints and tp sharded checkpoints.
# Embeddings are repeated to fit the vocab size.
# Other weights are flagged for the weight_loader calls.
if any(emb in modules for emb in ["embed_tokens", "lm_head"]):
# Embeddings are sharded on dim 0
dim = 0
# In fairseq2, vocab size has to be divisible by tp_size
# so we don't worry about padding
if self.tp_size > 1 and loaded_weight.shape[dim] < self.config.vocab_size:
assert (
loaded_weight.shape[dim] * self.tp_size == self.config.vocab_size
), "vocab_size should be divisible by tp_size."
repeats = [1] * len(loaded_weight.size())
repeats[dim] = self.tp_size
# repeat to match vocab size and to be easily 'narrow'able
loaded_weight = loaded_weight.repeat(repeats)
set_weight_attrs(params[name], {"is_sharded_weight": False})
# if embeddings are sharded, the rest is too
if "embed_tokens" in modules:
self.flag_sharded_weights(params)
return name, loaded_weight
-794
View File
@@ -1,794 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
FireRedLID – Language Identification model adapted for vLLM.
Architecture: ConformerEncoder + TransformerDecoder (6-layer cross-attn)
Vocabulary: 120 LID tokens (dict.txt)
Output: Up to 2 tokens (e.g. "en", "zh mandarin")
This implementation follows the Whisper-style encoder-decoder pattern:
• Encoder processes audio features (Fbank + CMVN via FeatureExtractor)
• Decoder performs single-step autoregressive forward
• vLLM's generation loop handles beam search / sampling
"""
from __future__ import annotations
from collections.abc import Iterable, Mapping, Sequence
from typing import Annotated, Literal
import torch
from torch import nn
from transformers import BatchFeature
from vllm.config import ModelConfig, VllmConfig
from vllm.config.multimodal import AudioDummyOptions, BaseDummyOptions
from vllm.config.speech_to_text import SpeechToTextConfig, SpeechToTextParams
from vllm.distributed import get_tensor_model_parallel_world_size
from vllm.inputs import MultiModalDataDict, PromptType
from vllm.logger import init_logger
from vllm.model_executor.layers.attention import Attention, CrossAttention
from vllm.model_executor.layers.linear import (
ColumnParallelLinear,
ReplicatedLinear,
RowParallelLinear,
)
from vllm.model_executor.layers.logits_processor import LogitsProcessor
from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
from vllm.multimodal import MULTIMODAL_REGISTRY
from vllm.multimodal.inputs import (
MultiModalFieldConfig,
MultiModalKwargsItems,
)
from vllm.multimodal.parse import MultiModalDataItems, MultiModalDataParser
from vllm.multimodal.processing import (
BaseDummyInputsBuilder,
BaseProcessingInfo,
EncDecMultiModalProcessor,
PromptReplacement,
PromptUpdate,
)
from vllm.transformers_utils.processor import cached_processor_from_config
from vllm.utils.tensor_schema import TensorSchema, TensorShape
from .conformer_encoder import ConformerEncoder
from .interfaces import (
MultiModalEmbeddings,
SupportsMultiModal,
SupportsTranscription,
)
from .utils import (
AutoWeightsLoader,
WeightsMapper,
maybe_prefix,
)
from .whisper_utils import ISO639_1_SUPPORTED_LANGS
logger = init_logger(__name__)
class FireRedLIDAudioInputs(TensorSchema):
"""
Dimensions:
- b: Batch size
- t: Time frames (variable across utterances)
- nmb: Number of mel bins (80)
"""
input_features: Annotated[
list[torch.Tensor] | None,
TensorShape("b", "t", "nmb", dynamic_dims={"t"}),
]
speech_lengths: Annotated[
list[torch.Tensor] | None,
TensorShape("b"),
]
fake_token_lengths: Annotated[
list[torch.Tensor] | None,
TensorShape("b"),
]
FireRedLIDEncoder = ConformerEncoder
class FireRedLIDPositionalEmbedding(nn.Module):
"""Absolute sinusoidal positional embedding indexed by `positions`."""
def __init__(self, d_model: int, max_len: int = 5000):
super().__init__()
assert d_model % 2 == 0
pe = torch.zeros(max_len, d_model, requires_grad=False)
position = torch.arange(0, max_len).unsqueeze(1).float()
div_term = torch.exp(
torch.arange(0, d_model, 2).float()
* -(torch.log(torch.tensor(10000.0)).item() / d_model)
)
pe[:, 0::2] = torch.sin(position * div_term)
pe[:, 1::2] = torch.cos(position * div_term)
self.register_buffer("pe", pe, persistent=False)
def forward(self, position_ids: torch.Tensor) -> torch.Tensor:
return self.pe[position_ids]
class FireRedLIDAttention(nn.Module):
"""Base attention with shared QKV/FC projections for the LID decoder."""
def __init__(
self,
d_model: int,
n_head: int,
*,
vllm_config: VllmConfig,
prefix: str = "",
):
super().__init__()
tp_size = get_tensor_model_parallel_world_size()
assert n_head % tp_size == 0
self.total_num_heads = n_head
self.num_heads = n_head // tp_size
self.num_kv_heads = max(1, n_head // tp_size)
self.head_dim = d_model // n_head
self.scaling = self.head_dim**-0.5
cache_config = vllm_config.cache_config
quant_config = vllm_config.quant_config
self.w_qs = ColumnParallelLinear(
d_model,
d_model,
bias=True,
quant_config=quant_config,
prefix=f"{prefix}.w_qs",
)
self.w_ks = ColumnParallelLinear(
d_model,
d_model,
bias=False,
quant_config=quant_config,
prefix=f"{prefix}.w_ks",
)
self.w_vs = ColumnParallelLinear(
d_model,
d_model,
bias=True,
quant_config=quant_config,
prefix=f"{prefix}.w_vs",
)
self.fc = RowParallelLinear(
d_model,
d_model,
bias=True,
quant_config=quant_config,
prefix=f"{prefix}.fc",
)
self._init_attn(cache_config, quant_config, prefix)
def _init_attn(self, cache_config, quant_config, prefix: str) -> None:
raise NotImplementedError
class FireRedLIDSelfAttention(FireRedLIDAttention):
def _init_attn(self, cache_config, quant_config, prefix: str) -> None:
self.attn = Attention(
self.num_heads,
self.head_dim,
self.scaling,
num_kv_heads=self.num_kv_heads,
cache_config=cache_config,
quant_config=quant_config,
prefix=f"{prefix}.attn",
)
def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
q, _ = self.w_qs(hidden_states)
k, _ = self.w_ks(hidden_states)
v, _ = self.w_vs(hidden_states)
attn_output = self.attn(q, k, v)
output, _ = self.fc(attn_output)
return output
class FireRedLIDCrossAttention(FireRedLIDAttention):
def _init_attn(self, cache_config, quant_config, prefix: str) -> None:
self.attn = CrossAttention(
self.num_heads,
self.head_dim,
self.scaling,
num_kv_heads=self.num_kv_heads,
cache_config=cache_config,
quant_config=quant_config,
prefix=f"{prefix}.attn",
)
def forward(
self,
hidden_states: torch.Tensor,
encoder_hidden_states: torch.Tensor | None,
) -> torch.Tensor:
q, _ = self.w_qs(hidden_states)
if encoder_hidden_states is not None:
k, _ = self.w_ks(encoder_hidden_states)
v, _ = self.w_vs(encoder_hidden_states)
else:
k = v = None
attn_output = self.attn(q, k, v)
output, _ = self.fc(attn_output)
return output
class FireRedLIDFFN(nn.Module):
def __init__(self, d_model: int, d_ff: int):
super().__init__()
self.w_1 = ReplicatedLinear(d_model, d_ff, bias=True)
self.act = nn.GELU()
self.w_2 = ReplicatedLinear(d_ff, d_model, bias=True)
def forward(self, x: torch.Tensor) -> torch.Tensor:
x, _ = self.w_1(x)
x = self.act(x)
x, _ = self.w_2(x)
return x
class FireRedLIDDecoderLayer(nn.Module):
"""vLLM-native decoder layer while preserving FireRedLID parameter names."""
def __init__(
self,
d_model: int,
n_head: int,
*,
vllm_config: VllmConfig,
prefix: str = "",
):
super().__init__()
self.self_attn_norm = nn.LayerNorm(d_model)
self.self_attn = FireRedLIDSelfAttention(
d_model,
n_head,
vllm_config=vllm_config,
prefix=f"{prefix}.self_attn",
)
self.cross_attn_norm = nn.LayerNorm(d_model)
self.cross_attn = FireRedLIDCrossAttention(
d_model,
n_head,
vllm_config=vllm_config,
prefix=f"{prefix}.cross_attn",
)
self.mlp_norm = nn.LayerNorm(d_model)
self.mlp = FireRedLIDFFN(d_model, d_model * 4)
def forward(
self,
hidden_states: torch.Tensor,
encoder_hidden_states: torch.Tensor | None,
) -> torch.Tensor:
residual = hidden_states
hidden_states = self.self_attn_norm(hidden_states)
hidden_states = self.self_attn(hidden_states)
hidden_states = residual + hidden_states
residual = hidden_states
hidden_states = self.cross_attn_norm(hidden_states)
hidden_states = self.cross_attn(hidden_states, encoder_hidden_states)
hidden_states = residual + hidden_states
residual = hidden_states
hidden_states = self.mlp_norm(hidden_states)
hidden_states = residual + self.mlp(hidden_states)
return hidden_states
class FireRedLIDDecoder(nn.Module):
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
super().__init__()
config = vllm_config.model_config.hf_config
self.pad_id = getattr(config, "pad_token_id", 2)
self.n_layers = getattr(config, "n_layers_lid_dec", 6)
self.d_model = getattr(config, "d_model", 1280)
self.scale = self.d_model**0.5
self.tgt_word_emb = nn.Embedding(
getattr(config, "vocab_size", 120),
self.d_model,
padding_idx=self.pad_id,
)
self.positional_encoding = FireRedLIDPositionalEmbedding(
self.d_model,
max_len=getattr(config, "pe_maxlen", 5000),
)
self.layer_stack = nn.ModuleList(
[
FireRedLIDDecoderLayer(
self.d_model,
getattr(config, "n_head", 20),
vllm_config=vllm_config,
prefix=f"{prefix}.layer_stack.{idx}",
)
for idx in range(self.n_layers)
]
)
self.layer_norm_out = nn.LayerNorm(self.d_model)
def forward(
self,
input_ids: torch.Tensor,
positions: torch.Tensor,
encoder_hidden_states: torch.Tensor | None,
) -> torch.Tensor:
hidden_states = self.tgt_word_emb(input_ids) * self.scale
hidden_states = hidden_states + self.positional_encoding(positions)
for layer in self.layer_stack:
hidden_states = layer(hidden_states, encoder_hidden_states)
hidden_states = self.layer_norm_out(hidden_states)
return hidden_states
def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
return self.tgt_word_emb(input_ids)
class FireRedLIDModel(nn.Module):
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
super().__init__()
config = vllm_config.model_config.hf_config
self.encoder = FireRedLIDEncoder(
idim=getattr(config, "idim", 80),
n_layers_enc=getattr(config, "n_layers_enc", 16),
n_head=getattr(config, "n_head", 20),
d_model=getattr(config, "d_model", 1280),
kernel_size=getattr(config, "kernel_size", 33),
pe_maxlen=getattr(config, "pe_maxlen", 5000),
)
self.decoder = FireRedLIDDecoder(
vllm_config=vllm_config,
prefix=maybe_prefix(prefix, "decoder"),
)
def forward(
self,
input_ids: torch.Tensor,
positions: torch.Tensor,
encoder_outputs: list[torch.Tensor] | None = None,
) -> torch.Tensor:
enc_states = (
torch.cat(encoder_outputs, dim=0)
if encoder_outputs and len(encoder_outputs) > 0
else None
)
decoder_outputs = self.decoder(
input_ids=input_ids,
positions=positions,
encoder_hidden_states=enc_states,
)
return decoder_outputs
def get_encoder_outputs(
self,
speech: torch.Tensor | list[torch.Tensor],
speech_lengths: torch.Tensor | list[torch.Tensor],
) -> tuple[torch.Tensor, torch.Tensor]:
"""Run the encoder and return padded outputs plus true sequence lengths."""
enc_output, enc_lengths, _ = self.encoder(speech, speech_lengths)
return enc_output, enc_lengths
class FireRedLIDProcessingInfo(BaseProcessingInfo):
def get_hf_config(self):
return self.ctx.get_hf_config()
def get_supported_mm_limits(self) -> Mapping[str, int | None]:
return {"audio": 1}
def get_feature_extractor(self, **kwargs):
hf_processor = self.get_hf_processor(**kwargs)
feature_extractor = hf_processor.feature_extractor
return feature_extractor
def get_data_parser(self) -> MultiModalDataParser:
feature_extractor = self.get_feature_extractor()
return MultiModalDataParser(
target_sr=feature_extractor.sampling_rate,
target_channels=1,
)
@property
def skip_prompt_length_check(self) -> bool:
return True
def get_num_audio_tokens(self) -> int:
# For encoder profiling – return a reasonable dummy length.
# This doesn't affect actual inference since encoder processes
# variable-length features.
return 1
class FireRedLIDDummyInputsBuilder(BaseDummyInputsBuilder[FireRedLIDProcessingInfo]):
def get_dummy_text(self, mm_counts: Mapping[str, int]) -> str:
return "<sos>"
def get_dummy_mm_data(
self,
seq_len: int,
mm_counts: Mapping[str, int],
mm_options: Mapping[str, BaseDummyOptions],
) -> MultiModalDataDict:
feature_extractor = self.info.get_feature_extractor()
sampling_rate = feature_extractor.sampling_rate
audio_len = feature_extractor.chunk_length * sampling_rate
num_audios = mm_counts.get("audio", 0)
audio_overrides = mm_options.get("audio")
assert audio_overrides is None or isinstance(audio_overrides, AudioDummyOptions)
return {
"audio": self._get_dummy_audios(
length=audio_len,
num_audios=num_audios,
overrides=audio_overrides,
)
}
class FireRedLIDMultiModalProcessor(
EncDecMultiModalProcessor[FireRedLIDProcessingInfo]
):
def create_encoder_prompt(
self,
prompt: list[int],
mm_items: MultiModalDataItems,
) -> list[int]:
# Dummy encoder prompt for profiling (encoder only processes audio).
return [0]
def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:
return self.dummy_inputs.get_dummy_text(mm_counts)
def _preprocess_hf_mm_data(
self,
mm_data: Mapping[str, object],
hf_processor_mm_kwargs: Mapping[str, object],
) -> tuple[Mapping[str, object], Mapping[str, object]]:
feature_extractor = self.info.get_feature_extractor(**hf_processor_mm_kwargs)
mm_data = dict(mm_data)
mm_data["audio"] = mm_data.pop("audios")
hf_processor_mm_kwargs = dict(
**hf_processor_mm_kwargs,
sampling_rate=feature_extractor.sampling_rate,
)
return mm_data, hf_processor_mm_kwargs
def _postprocess_hf_mm_data(
self,
mm_data: Mapping[str, object],
hf_processor_mm_kwargs: Mapping[str, object],
processed_data: BatchFeature,
) -> BatchFeature:
if "labels" in processed_data:
processed_data["input_ids"] = processed_data.pop("labels")
return processed_data
def _get_mm_fields_config(
self,
hf_inputs: BatchFeature,
hf_processor_mm_kwargs: Mapping[str, object],
) -> Mapping[str, MultiModalFieldConfig]:
return dict(
input_features=MultiModalFieldConfig.batched("audio"),
speech_lengths=MultiModalFieldConfig.batched("audio"),
fake_token_lengths=MultiModalFieldConfig.batched("audio", keep_on_cpu=True),
)
def _get_prompt_updates(
self,
mm_items: MultiModalDataItems,
hf_processor_mm_kwargs: Mapping[str, object],
out_mm_kwargs: MultiModalKwargsItems,
) -> Sequence[PromptUpdate]:
out_mm_data = out_mm_kwargs.get_data()
fake_token_lengths = out_mm_data.get("fake_token_lengths")
if fake_token_lengths is None:
# Fallback to max encoder output length if not available
audio_output_lengths = []
else:
assert isinstance(fake_token_lengths, torch.Tensor)
audio_output_lengths = fake_token_lengths.tolist()
def get_replacement(item_idx: int):
if audio_output_lengths:
num_tokens = int(audio_output_lengths[item_idx])
else:
num_tokens = self.info.get_num_audio_tokens()
return [0] * num_tokens
return [
PromptReplacement(
modality="audio",
target=[0],
replacement=get_replacement,
)
]
# FireRedLID supports a wider set of languages than Whisper's shared list.
# Only ISO 639-1 codes are listed; FireRedLID's dialect tokens (mandarin,
# xinan, wu, …) are output tokens but not valid language *request* codes.
_FIREREDLID_SUPPORTED_LANGUAGES: Mapping[str, str] = {
**ISO639_1_SUPPORTED_LANGS,
"am": "Amharic",
"as": "Assamese",
"ba": "Bashkir",
"bn": "Bengali",
"bo": "Tibetan",
"br": "Breton",
"eu": "Basque",
"fo": "Faroese",
"gu": "Gujarati",
"ha": "Hausa",
"haw": "Hawaiian",
"ht": "Haitian Creole",
"jw": "Javanese",
"ka": "Georgian",
"km": "Khmer",
"la": "Latin",
"lb": "Luxembourgish",
"ln": "Lingala",
"lo": "Lao",
"mg": "Malagasy",
"ml": "Malayalam",
"mn": "Mongolian",
"mt": "Maltese",
"my": "Myanmar",
"nn": "Nynorsk",
"oc": "Occitan",
"pa": "Panjabi",
"ps": "Pashto",
"sa": "Sanskrit",
"sd": "Sindhi",
"si": "Sinhala",
"sn": "Shona",
"so": "Somali",
"sq": "Albanian",
"su": "Sundanese",
"te": "Telugu",
"tg": "Tajik",
"tk": "Turkmen",
"tt": "Tatar",
"uz": "Uzbek",
"yi": "Yiddish",
"yo": "Yoruba",
"yue": "Cantonese",
}
@MULTIMODAL_REGISTRY.register_processor(
FireRedLIDMultiModalProcessor,
info=FireRedLIDProcessingInfo,
dummy_inputs=FireRedLIDDummyInputsBuilder,
)
class FireRedLIDForConditionalGeneration(
nn.Module, SupportsTranscription, SupportsMultiModal
):
# -- SupportsTranscription protocol attributes --
supports_transcription_only = True
supported_languages = _FIREREDLID_SUPPORTED_LANGUAGES
hf_to_vllm_mapper = WeightsMapper(
orig_to_new_substr={
"encoder.": "model.encoder.",
"lid_decoder.": "model.decoder.",
# Encoder FFN: nn.Sequential indices → named children
"net.0": "pre_layer_norm",
"net.1": "linear_expand",
"net.4": "linear_project",
},
orig_to_new_prefix={
# Position encoding buffers are rebuilt at init, and the output
# projection is tied to the embedding.
"model.encoder.positional_encoding.pe": None,
"model.decoder.positional_encoding.pe": None,
"model.decoder.tgt_word_prj.weight": None,
"proj_out.": None,
},
)
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
super().__init__()
config = vllm_config.model_config.hf_config
self.config = config
self.dtype = vllm_config.model_config.dtype
with self._mark_composite_model(
vllm_config,
language_targets=FireRedLIDDecoder,
tower_targets={"audio": FireRedLIDEncoder},
):
self.model = FireRedLIDModel(
vllm_config=vllm_config,
prefix=maybe_prefix(prefix, "model"),
)
self.proj_out = ParallelLMHead(
getattr(config, "vocab_size", 120),
getattr(config, "d_model", 1280),
quant_config=vllm_config.quant_config,
prefix=maybe_prefix(prefix, "proj_out"),
)
self.proj_out = self.proj_out.tie_weights(self.model.decoder.tgt_word_emb)
logit_scale = getattr(config, "logit_scale", 1.0)
self.logits_processor = LogitsProcessor(
getattr(config, "vocab_size", 120),
scale=logit_scale,
)
def forward(
self,
input_ids: torch.Tensor,
positions: torch.Tensor,
encoder_outputs: list[torch.Tensor] | None = None,
**kwargs,
) -> torch.Tensor:
if encoder_outputs is None:
encoder_outputs = []
decoder_outputs = self.model(
input_ids=input_ids,
positions=positions,
encoder_outputs=encoder_outputs,
)
return decoder_outputs
def embed_multimodal(self, **kwargs: object) -> MultiModalEmbeddings:
"""Run encoder on audio features and return per-item embeddings."""
audio_input = self._parse_and_validate_audio_input(**kwargs)
speech = audio_input["input_features"]
speech_lengths = audio_input["speech_lengths"]
if speech is None or speech_lengths is None:
return []
# When audio items have different time lengths, vLLM's
# MultiModalBatchedField._reduce_data returns a plain
# list[Tensor] instead of a stacked Tensor. The encoder
# expects a padded [B, Tmax, feat_dim] Tensor, so we
# normalise both speech and speech_lengths here.
if isinstance(speech, (list, tuple)):
# Each element: [Ti, feat_dim] (or [1, Ti, feat_dim])
tensors = [
s.squeeze(0) if s.dim() == 3 and s.size(0) == 1 else s for s in speech
]
device = tensors[0].device
dtype = tensors[0].dtype
feat_dim = tensors[0].shape[-1]
lengths = torch.tensor(
[t.size(0) for t in tensors],
device=device,
dtype=torch.int32,
)
t_max = int(lengths.max().item())
# Pre-allocate zero-padded batch tensor
speech = torch.zeros(
(len(tensors), t_max, feat_dim),
device=device,
dtype=dtype,
)
for i, t in enumerate(tensors):
speech[i, : t.size(0)] = t
speech_lengths = lengths
else:
# Already a batched Tensor [B, T, feat_dim]
if speech.dim() == 2:
speech = speech.unsqueeze(0)
speech_lengths = torch.as_tensor(
speech_lengths, dtype=torch.int32, device=speech.device
)
enc_output, enc_lengths = self.model.get_encoder_outputs(
speech=speech,
speech_lengths=speech_lengths,
)
# vLLM expects one 2D tensor per multimodal item. Slice each batch entry
# by the true encoder length so cross-attention never sees padded frames.
return tuple(
enc_output[i, : max(0, int(enc_lengths[i].item()))]
for i in range(enc_output.size(0))
)
def embed_input_ids(
self,
input_ids: torch.Tensor,
multimodal_embeddings: MultiModalEmbeddings | None = None,
*,
is_multimodal: torch.Tensor | None = None,
) -> torch.Tensor:
return self.model.decoder.embed_input_ids(input_ids)
def _parse_and_validate_audio_input(
self, **kwargs: object
) -> FireRedLIDAudioInputs:
input_features = kwargs.pop("input_features", None)
speech_lengths = kwargs.pop("speech_lengths", None)
fake_token_lengths = kwargs.pop("fake_token_lengths", None)
return FireRedLIDAudioInputs(
input_features=input_features,
speech_lengths=speech_lengths,
fake_token_lengths=fake_token_lengths,
)
def compute_logits(self, hidden_states: torch.Tensor) -> torch.Tensor:
logits = self.logits_processor(self.proj_out, hidden_states)
return logits
@classmethod
def validate_language(cls, language: str | None) -> str | None:
# FireRedLID is a language *identification* model – the caller does
# not need to specify a language up-front. Accept None silently.
if language is None:
return None
return super().validate_language(language)
@classmethod
def get_generation_prompt(
cls,
stt_params: SpeechToTextParams,
) -> PromptType:
"""Build the prompt for the FireRedLID encoder-decoder model.
The decoder receives a single <sos> token; the encoder processes
the raw audio waveform via the multimodal pipeline.
"""
prompt: PromptType = {
"encoder_prompt": {
"prompt": "",
"multi_modal_data": {
"audio": (
stt_params.audio,
int(stt_params.stt_config.sample_rate),
),
},
},
"decoder_prompt": {
"prompt": "<sos>",
},
}
return prompt
@classmethod
def get_speech_to_text_config(
cls,
model_config: ModelConfig,
task_type: Literal["transcribe", "translate"],
) -> SpeechToTextConfig:
processor = cached_processor_from_config(model_config)
return SpeechToTextConfig(
max_audio_clip_s=processor.feature_extractor.chunk_length,
sample_rate=processor.feature_extractor.sampling_rate,
# LID output is at most 2 tokens – no chunking needed.
min_energy_split_window_size=None,
)
@classmethod
def post_process_output(cls, text: str) -> str:
# Strip any leading/trailing whitespace from the raw LID output.
return text.strip()
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
loader = AutoWeightsLoader(self)
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
-243
View File
@@ -1,243 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from collections.abc import Set
import numpy as np
import torch
from vllm.config import ModelConfig, VllmConfig
from vllm.logger import init_logger
from vllm.model_executor.layers.pooler import (
DispatchPooler,
PoolingParamsUpdate,
)
from vllm.model_executor.layers.pooler.activations import PoolerNormalize
from vllm.model_executor.layers.pooler.seqwise import (
EmbeddingPoolerHead,
SequencePooler,
SequencePoolingMethod,
SequencePoolingMethodOutput,
get_seq_pooling_method,
)
from vllm.model_executor.layers.pooler.tokwise import pooler_for_token_embed
from vllm.model_executor.models.llama import LlamaForCausalLM
from vllm.tasks import PoolingTask
from vllm.tokenizers import cached_tokenizer_from_config
from vllm.v1.pool.metadata import PoolingMetadata
from .interfaces_base import default_pooling_type
logger = init_logger(__name__)
class GritLMMeanPool(SequencePoolingMethod):
"""As `MeanPool`, but only includes non-instruction tokens."""
def __init__(self, model_config: ModelConfig):
super().__init__()
self.model_config = model_config
tokenizer = cached_tokenizer_from_config(self.model_config)
# Collect the tokens needed for pattern matching.
# "▁<" is different from "_<". The former uses "▁" to indicate that
# the next token is the start of a word.
# "<0x0A>" is the newline token (i.e. "\n")."
self.token_ids = {
tok: tokenizer.convert_tokens_to_ids([tok])[0]
for tok in ["<s>", "▁<", "<", "|", "embed", ">", "<0x0A>", "user"]
}
def tokens_to_ids(tokens: list[str]) -> np.ndarray:
return np.array([self.token_ids[token] for token in tokens])
self.user_pattern_ids = tokens_to_ids(["▁<", "|", "user", "|", ">", "<0x0A>"])
self.embed_newline_pattern_ids = tokens_to_ids(
["<0x0A>", "<", "|", "embed", "|", ">", "<0x0A>"]
)
self.embed_pattern_ids = tokens_to_ids(["▁<", "|", "embed", "|", ">", "<0x0A>"])
def _find_array(
self,
arr: np.ndarray,
target: np.ndarray,
start_idx: int = 0,
end_idx: int | None = None,
) -> int:
"""
Find the first occurrence of `target` in `arr` starting from
`start_idx`.
Args:
arr: The array to search within.
target: The consecutive subsequence to find.
start_idx: The starting index to search from (inclusive).
end_idx: The ending index to search from (exclusive).
Returns:
The index of the first occurrence of `target` in `arr`.
"""
if start_idx < 0:
raise ValueError("`start_idx` must be non-negative")
if len(arr) == 0 or len(target) == 0:
raise ValueError("Empty `arr` or `target` not allowed")
arr_len = len(arr)
target_len = len(target)
if end_idx is None:
end_idx = arr_len
for i in range(start_idx, min(end_idx, arr_len - target_len + 1)):
if (arr[i : i + target_len] == target).all():
return i
return -1
def _get_instruction_len(self, prompt_token_ids: np.ndarray) -> int:
"""
Get the length of the instruction in the prompt.
We do a pattern matching to find the instruction in the prompt,
and then return the length of the instruction.
The pattern matching is done using integers instead of strings
because the prompt is given as a list of token IDs.
"""
instruction_len = 0
# Return no instruction in case of missing BOS token.
if prompt_token_ids[0] != self.token_ids["<s>"]:
logger.warning(
"BOS token not found in prompt, "
"thus using empty string for instruction. "
"GritLM requires BOS token in prompt."
)
return instruction_len
# If user pattern is found in the prompt, that means there should be
# a newline token before the embed pattern.
embed_pattern_ids = self.embed_pattern_ids
if (
self._find_array(
prompt_token_ids, self.user_pattern_ids, start_idx=1, end_idx=2
)
== 1
):
embed_pattern_ids = self.embed_newline_pattern_ids
# Find the embed pattern in the prompt.
found_embed_pattern_idx = self._find_array(
prompt_token_ids, embed_pattern_ids, start_idx=1
)
if found_embed_pattern_idx != -1:
instruction_len = found_embed_pattern_idx + len(embed_pattern_ids)
else:
logger.warning(
"Query instruction not found in prompt, "
"thus using BOS token as instruction instead. "
"GritLM requires query instruction in prompt."
)
instruction_len = 1
return instruction_len
def get_supported_tasks(self) -> Set[PoolingTask]:
return {"embed"}
def get_pooling_updates(self, task: PoolingTask) -> PoolingParamsUpdate:
return PoolingParamsUpdate(requires_token_ids=True)
def forward(
self,
hidden_states: torch.Tensor,
pooling_metadata: PoolingMetadata,
) -> SequencePoolingMethodOutput:
prompt_lens = pooling_metadata.prompt_lens
prompt_token_ids = pooling_metadata.get_prompt_token_ids_cpu()
instr_lens = torch.tensor(
[
self._get_instruction_len(token_ids.numpy())
for token_ids in prompt_token_ids
],
device="cpu",
)
offset = 0
pooled_data = list[torch.Tensor]()
for prompt_len, instr_len in zip(prompt_lens, instr_lens):
pooled_data.append(
hidden_states[offset + instr_len : offset + prompt_len].mean(
dim=0, dtype=torch.float32
)
)
offset += prompt_len
return pooled_data
class GritLMPooler(SequencePooler):
def __init__(self, model_config: ModelConfig):
pooler_config = model_config.pooler_config
assert pooler_config is not None
super().__init__(
pooling=(
GritLMMeanPool(model_config)
if pooler_config.seq_pooling_type == "MEAN"
else get_seq_pooling_method(pooler_config.seq_pooling_type)
),
head=EmbeddingPoolerHead(
head_dtype=model_config.head_dtype,
activation=PoolerNormalize(),
),
)
@default_pooling_type(seq_pooling_type="MEAN")
class GritLM(LlamaForCausalLM):
"""This class implements the embedding model for parasail-ai/GritLM-7B-vllm.
The class inherits from LlamaForCausalLM and provides a custom pooling
layer.
The main difference between the pooling layer in GritLM and the one in
LlamaForCausalLM is that GritLM ignores the query instruction in the prompt
when pooling the hidden states.
Embedding prompts should be in the following format:
- With instruction: "<|user|>\nINSTRUCTION\n<|embed|>\nPROMPT".
- Without instruction: "<|embed|>\nPROMPT".
Generation prompts should be in the following format:
- "<|user|>\nPROMPT\n<|assistant|>\n"
"""
is_pooling_model = True
def __init__(
self,
vllm_config: VllmConfig,
prefix: str = "",
**kwargs,
) -> None:
if vllm_config.model_config.runner_type == "pooling":
hf_config = vllm_config.model_config.hf_config
hf_config.is_causal = False
vllm_config.cache_config.sliding_window = None
hf_config.sliding_window = None
super().__init__(vllm_config=vllm_config, prefix=prefix, **kwargs)
pooler_config = vllm_config.model_config.pooler_config
if pooler_config is not None:
self.pooler = DispatchPooler(
{
"token_embed": pooler_for_token_embed(pooler_config),
"embed": GritLMPooler(vllm_config.model_config),
}
)
File diff suppressed because it is too large Load Diff
+1 -1
View File
@@ -273,7 +273,7 @@ class LlamaDecoderLayer(nn.Module):
# By default, Llama uses causal attention as it is a decoder-only model.
# You can override the HF config with `is_causal=False` to enable
# bidirectional attention, which is used in some embedding models
# (e.g. parasail-ai/GritLM-7B-vllm)
# (e.g. nvidia/llama-nemotron-embed-1b-v2)
if getattr(config, "is_causal", True):
attn_type = AttentionType.DECODER
else:
-317
View File
@@ -1,317 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
# Adapted from https://huggingface.co/mosaicml/mpt-7b/tree/main
import math
from collections.abc import Iterable
from itertools import islice
import torch
import torch.nn as nn
from transformers import MptConfig
from vllm.compilation.decorators import support_torch_compile
from vllm.config import CacheConfig, VllmConfig
from vllm.distributed import (
get_pp_group,
get_tensor_model_parallel_rank,
get_tensor_model_parallel_world_size,
)
from vllm.model_executor.layers.activation import get_act_fn
from vllm.model_executor.layers.attention import Attention
from vllm.model_executor.layers.linear import (
ColumnParallelLinear,
QKVParallelLinear,
RowParallelLinear,
)
from vllm.model_executor.layers.logits_processor import LogitsProcessor
from vllm.model_executor.layers.quantization import QuantizationConfig
from vllm.model_executor.layers.vocab_parallel_embedding import VocabParallelEmbedding
from vllm.sequence import IntermediateTensors
from .interfaces import SupportsPP
from .utils import (
AutoWeightsLoader,
make_empty_intermediate_tensors_factory,
make_layers,
maybe_prefix,
)
def _get_alibi_slopes(
total_num_heads: int,
alibi_bias_max: int,
) -> torch.Tensor:
next_power_of_2 = 2 ** math.ceil(math.log2(total_num_heads))
m = torch.arange(1, next_power_of_2 + 1, dtype=torch.float32)
m = m.mul(alibi_bias_max / next_power_of_2)
slopes = 1.0 / torch.pow(2, m)
if next_power_of_2 != total_num_heads:
slopes = torch.concat([slopes[1::2], slopes[::2]])[:total_num_heads]
return slopes
class MPTAttention(nn.Module):
def __init__(
self,
config: MptConfig,
cache_config: CacheConfig | None = None,
quant_config: QuantizationConfig | None = None,
prefix: str = "",
):
super().__init__()
self.d_model = config.d_model
self.total_num_heads = config.n_heads
self.head_dim = self.d_model // self.total_num_heads
self.clip_qkv = config.attn_config.clip_qkv
self.qk_ln = config.attn_config.qk_ln
self.alibi_bias_max = config.attn_config.alibi_bias_max
if "kv_n_heads" in config.attn_config:
self.total_num_kv_heads = config.attn_config.kv_n_heads
else:
self.total_num_kv_heads = self.total_num_heads
assert not config.attn_config.prefix_lm
assert config.attn_config.alibi
# pylint: disable=invalid-name
self.Wqkv = QKVParallelLinear(
self.d_model,
self.d_model // self.total_num_heads,
self.total_num_heads,
self.total_num_kv_heads,
bias=not config.no_bias,
quant_config=quant_config,
prefix=f"{prefix}.Wqkv",
)
if self.qk_ln:
self.q_ln = nn.LayerNorm(self.d_model)
self.k_ln = nn.LayerNorm(self.d_model)
self.out_proj = RowParallelLinear(
self.d_model,
self.d_model,
bias=not config.no_bias,
quant_config=quant_config,
prefix=f"{prefix}.out_proj",
)
tp_world_size = get_tensor_model_parallel_world_size()
assert self.total_num_heads % tp_world_size == 0
self.num_heads = self.total_num_heads // tp_world_size
if self.total_num_kv_heads >= tp_world_size:
# Number of KV heads is greater than TP size, so we partition
# the KV heads across multiple tensor parallel GPUs.
assert self.total_num_kv_heads % tp_world_size == 0
else:
# Number of KV heads is less than TP size, so we replicate
# the KV heads across multiple tensor parallel GPUs.
assert tp_world_size % self.total_num_kv_heads == 0
self.num_kv_heads = max(1, self.total_num_kv_heads // tp_world_size)
self.q_size = self.num_heads * self.head_dim
self.kv_size = self.num_kv_heads * self.head_dim
# Create the alibi slopes and slice them.
tp_rank = get_tensor_model_parallel_rank()
head_start = tp_rank * self.num_heads
head_end = (tp_rank + 1) * self.num_heads
alibi_slopes = _get_alibi_slopes(self.total_num_heads, self.alibi_bias_max)
alibi_slopes = alibi_slopes[head_start:head_end].tolist()
self.head_dim = self.d_model // self.total_num_heads
scaling = self.head_dim**-0.5
self.attn = Attention(
self.num_heads,
self.head_dim,
scaling,
alibi_slopes=alibi_slopes,
num_kv_heads=self.num_kv_heads,
cache_config=cache_config,
quant_config=quant_config,
prefix=f"{prefix}.attn",
)
def forward(
self,
position_ids: torch.Tensor,
hidden_states: torch.Tensor,
) -> torch.Tensor:
del position_ids # unused.
qkv, _ = self.Wqkv(hidden_states)
if self.clip_qkv is not None:
qkv.clamp_(min=-self.clip_qkv, max=self.clip_qkv)
q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1)
if self.qk_ln:
q = self.q_ln(q)
k = self.k_ln(k)
attn_output = self.attn(q, k, v)
output, _ = self.out_proj(attn_output)
return output
class MPTMLP(nn.Module):
def __init__(
self,
config: MptConfig,
quant_config: QuantizationConfig | None = None,
prefix: str = "",
):
super().__init__()
hidden_size = config.d_model
expansion_ratio = config.expansion_ratio
intermediate_size = expansion_ratio * hidden_size
self.up_proj = ColumnParallelLinear(
hidden_size,
intermediate_size,
bias=not config.no_bias,
quant_config=quant_config,
prefix=f"{prefix}.up_proj",
)
self.act = get_act_fn("gelu")
self.down_proj = RowParallelLinear(
intermediate_size,
hidden_size,
bias=not config.no_bias,
quant_config=quant_config,
prefix=f"{prefix}.down_proj",
)
def forward(self, x: torch.Tensor) -> torch.Tensor:
x, _ = self.up_proj(x)
x = self.act(x)
x, _ = self.down_proj(x)
return x
class MPTBlock(nn.Module):
def __init__(
self,
config: MptConfig,
cache_config: CacheConfig | None = None,
quant_config: QuantizationConfig | None = None,
prefix: str = "",
):
super().__init__()
hidden_size = config.d_model
self.norm_1 = nn.LayerNorm(hidden_size)
self.attn = MPTAttention(
config, cache_config, quant_config, prefix=f"{prefix}.attn"
)
self.norm_2 = nn.LayerNorm(hidden_size)
self.ffn = MPTMLP(config, quant_config, prefix=f"{prefix}.ffn")
def forward(
self,
position_ids: torch.Tensor,
hidden_states: torch.Tensor,
) -> torch.Tensor:
x = self.norm_1(hidden_states)
x = self.attn(
position_ids=position_ids,
hidden_states=x,
)
hidden_states = hidden_states + x
x = self.norm_2(hidden_states)
x = self.ffn(x)
hidden_states = hidden_states + x
return hidden_states
@support_torch_compile
class MPTModel(nn.Module):
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
super().__init__()
config = vllm_config.model_config.hf_config
cache_config = vllm_config.cache_config
quant_config = vllm_config.quant_config
assert config.embedding_fraction == 1.0
assert config.norm_type == "low_precision_layernorm"
self.wte = VocabParallelEmbedding(
config.vocab_size,
config.d_model,
)
self.start_layer, self.end_layer, self.blocks = make_layers(
config.n_layers,
lambda prefix: MPTBlock(config, cache_config, quant_config, prefix=prefix),
prefix=f"{prefix}.blocks",
)
self.norm_f = nn.LayerNorm(config.d_model)
if config.no_bias:
for module in self.modules():
if hasattr(module, "bias") and isinstance(module.bias, nn.Parameter):
# Remove the bias term in Linear and LayerNorm.
module.register_parameter("bias", None)
self.make_empty_intermediate_tensors = make_empty_intermediate_tensors_factory(
["hidden_states"], config.d_model
)
def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
return self.wte(input_ids)
def forward(
self,
input_ids: torch.Tensor | None,
position_ids: torch.Tensor,
intermediate_tensors: IntermediateTensors | None,
inputs_embeds: torch.Tensor | None = None,
) -> torch.Tensor | IntermediateTensors:
if get_pp_group().is_first_rank:
if inputs_embeds is not None:
hidden_states = inputs_embeds
else:
hidden_states = self.embed_input_ids(input_ids)
else:
assert intermediate_tensors is not None
hidden_states = intermediate_tensors["hidden_states"]
for block in islice(self.blocks, self.start_layer, self.end_layer):
hidden_states = block(position_ids, hidden_states)
if not get_pp_group().is_last_rank:
return IntermediateTensors({"hidden_states": hidden_states})
hidden_states = self.norm_f(hidden_states)
return hidden_states
class MPTForCausalLM(nn.Module, SupportsPP):
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
super().__init__()
config = vllm_config.model_config.hf_config
quant_config = vllm_config.quant_config
self.config = config
self.quant_config = quant_config
self.transformer = MPTModel(
vllm_config=vllm_config, prefix=maybe_prefix(prefix, "transformer")
)
self.lm_head = self.transformer.wte
self.logits_processor = LogitsProcessor(config.vocab_size)
self.make_empty_intermediate_tensors = (
self.transformer.make_empty_intermediate_tensors
)
def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
return self.transformer.embed_input_ids(input_ids)
def forward(
self,
input_ids: torch.Tensor | None,
positions: torch.Tensor,
intermediate_tensors: IntermediateTensors | None = None,
inputs_embeds: torch.Tensor | None = None,
) -> torch.Tensor | IntermediateTensors:
hidden_states = self.transformer(
input_ids, positions, intermediate_tensors, inputs_embeds
)
return hidden_states
def compute_logits(
self,
hidden_states: torch.Tensor,
) -> torch.Tensor | None:
logits = self.logits_processor(self.lm_head, hidden_states)
return logits
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
loader = AutoWeightsLoader(self)
return loader.load_weights(weights)
+14 -21
View File
@@ -74,7 +74,6 @@ _TEXT_GENERATION_MODELS = {
"AfmoeForCausalLM": ("afmoe", "AfmoeForCausalLM"),
"ApertusForCausalLM": ("apertus", "ApertusForCausalLM"),
"ArceeForCausalLM": ("arcee", "ArceeForCausalLM"),
"ArcticForCausalLM": ("arctic", "ArcticForCausalLM"),
"AXK1ForCausalLM": ("AXK1", "AXK1ForCausalLM"),
"BailingMoeForCausalLM": ("bailing_moe", "BailingMoeForCausalLM"),
"BailingMoeV2ForCausalLM": ("bailing_moe", "BailingMoeV2ForCausalLM"),
@@ -102,7 +101,6 @@ _TEXT_GENERATION_MODELS = {
"ExaoneForCausalLM": ("exaone", "ExaoneForCausalLM"),
"Exaone4ForCausalLM": ("exaone4", "Exaone4ForCausalLM"),
"ExaoneMoeForCausalLM": ("exaone_moe", "ExaoneMoeForCausalLM"),
"Fairseq2LlamaForCausalLM": ("fairseq2_llama", "Fairseq2LlamaForCausalLM"),
"FalconForCausalLM": ("falcon", "FalconForCausalLM"),
"FalconMambaForCausalLM": ("mamba", "MambaForCausalLM"),
"FalconH1ForCausalLM": ("falcon_h1", "FalconH1ForCausalLM"),
@@ -129,12 +127,10 @@ _TEXT_GENERATION_MODELS = {
"GraniteMoeSharedForCausalLM": ("granitemoeshared", "GraniteMoeSharedForCausalLM"),
"GraniteMoeSWAForCausalLM": ("granitemoeshared", "GraniteMoeSharedForCausalLM"),
"GraniteSWAForCausalLM": ("granite", "GraniteForCausalLM"),
"GritLM": ("gritlm", "GritLM"),
"HrmTextForCausalLM": ("hrm_text", "HrmTextForCausalLM"),
"HunYuanMoEV1ForCausalLM": ("hunyuan_v1", "HunYuanMoEV1ForCausalLM"),
"HunYuanDenseV1ForCausalLM": ("hunyuan_v1", "HunYuanDenseV1ForCausalLM"),
"HYV3ForCausalLM": ("hy_v3", "HYV3ForCausalLM"),
"HCXVisionForCausalLM": ("hyperclovax_vision", "HCXVisionForCausalLM"),
"HCXVisionV2ForCausalLM": ("hyperclovax_vision_v2", "HCXVisionV2ForCausalLM"),
"HyperCLOVAXForCausalLM": ("hyperclovax", "HyperCLOVAXForCausalLM"),
"InternLM2ForCausalLM": ("internlm2", "InternLM2ForCausalLM"),
@@ -178,9 +174,6 @@ _TEXT_GENERATION_MODELS = {
"MistralForCausalLM": ("mistral", "MistralForCausalLM"),
"MistralLarge3ForCausalLM": ("mistral_large_3", "MistralLarge3ForCausalLM"),
"MixtralForCausalLM": ("mixtral", "MixtralForCausalLM"),
# transformers's mpt class has lower case
"MptForCausalLM": ("mpt", "MPTForCausalLM"),
"MPTForCausalLM": ("mpt", "MPTForCausalLM"),
"MiMoForCausalLM": ("mimo", "MiMoForCausalLM"),
"MiMoV2FlashForCausalLM": ("mimo_v2", "MiMoV2FlashForCausalLM"),
"MiMoV2ForCausalLM": ("mimo_v2", "MiMoV2ForCausalLM"),
@@ -207,14 +200,12 @@ _TEXT_GENERATION_MODELS = {
"Qwen3MoeForCausalLM": ("qwen3_moe", "Qwen3MoeForCausalLM"),
"Qwen3_5ForCausalLM": ("qwen3_5", "Qwen3_5ForCausalLM"),
"Qwen3_5MoeForCausalLM": ("qwen3_5", "Qwen3_5MoeForCausalLM"),
"RWForCausalLM": ("falcon", "FalconForCausalLM"),
"SarvamMoEForCausalLM": ("sarvam", "SarvamMoEForCausalLM"),
"SarvamMLAForCausalLM": ("sarvam", "SarvamMLAForCausalLM"),
"SeedOssForCausalLM": ("seed_oss", "SeedOssForCausalLM"),
"Step1ForCausalLM": ("step1", "Step1ForCausalLM"),
"Step3TextForCausalLM": ("step3_text", "Step3TextForCausalLM"),
"Step3p5ForCausalLM": ("step3p5", "Step3p5ForCausalLM"),
"StableLMEpochForCausalLM": ("stablelm", "StablelmForCausalLM"),
"StableLmForCausalLM": ("stablelm", "StablelmForCausalLM"),
"SolarForCausalLM": ("solar", "SolarForCausalLM"),
"TeleChat2ForCausalLM": ("telechat2", "TeleChat2ForCausalLM"),
@@ -234,7 +225,6 @@ _EMBEDDING_MODELS = {
"Gemma2Model": ("gemma2", "Gemma2ForCausalLM"),
"Gemma3TextModel": ("gemma3", "Gemma3Model"),
"GlmForCausalLM": ("glm", "GlmForCausalLM"),
"GritLM": ("gritlm", "GritLM"),
"GteModel": ("bert_with_rope", "SnowflakeGteNewModel"),
"GteNewModel": ("bert_with_rope", "GteNewModel"),
"JinaEmbeddingsV5Model": ("jina", "JinaEmbeddingsV5Model"),
@@ -274,7 +264,6 @@ _EMBEDDING_MODELS = {
# Technically Terratorch models work on images, both in
# input and output. I am adding it here because it piggy-backs on embedding
# models for the time being.
"PrithviGeoSpatialMAE": ("terratorch", "Terratorch"),
"Terratorch": ("terratorch", "Terratorch"),
}
@@ -360,12 +349,6 @@ _MULTIMODAL_MODELS = {
"BagelForConditionalGeneration": ("bagel", "BagelForConditionalGeneration"),
"BeeForConditionalGeneration": ("bee", "BeeForConditionalGeneration"),
"Blip2ForConditionalGeneration": ("blip2", "Blip2ForConditionalGeneration"),
"ChameleonForConditionalGeneration": (
"chameleon",
"ChameleonForConditionalGeneration",
),
"Cheers": ("cheers", "CheersForConditionalGeneration"),
"CheersForConditionalGeneration": ("cheers", "CheersForConditionalGeneration"),
"Cohere2VisionForConditionalGeneration": (
"cohere2_vision",
"Cohere2VisionForConditionalGeneration",
@@ -401,10 +384,6 @@ _MULTIMODAL_MODELS = {
"FireRedASR2ForConditionalGeneration",
),
"FunASRForConditionalGeneration": ("funasr", "FunASRForConditionalGeneration"),
"FireRedLIDForConditionalGeneration": (
"fireredlid",
"FireRedLIDForConditionalGeneration",
),
"FunAudioChatForConditionalGeneration": (
"funaudiochat",
"FunAudioChatForConditionalGeneration",
@@ -811,6 +790,20 @@ _PREVIOUSLY_SUPPORTED_MODELS = {
"FuyuForCausalLM": "0.25.0",
"Plamo2ForCausalLM": "0.26.0",
"OuroForCausalLM": "0.26.0",
"ArcticForCausalLM": "0.28.0",
"ChameleonForConditionalGeneration": "0.28.0",
"Cheers": "0.28.0",
"CheersForConditionalGeneration": "0.28.0",
"Fairseq2LlamaForCausalLM": "0.28.0",
"FireRedLIDForConditionalGeneration": "0.28.0",
"GritLM": "0.28.0",
"HCXVisionForCausalLM": "0.28.0",
"MPTForCausalLM": "0.28.0",
"MptForCausalLM": "0.28.0",
"RWForCausalLM": "0.28.0",
"StableLMEpochForCausalLM": "0.28.0",
# Superseded by the generic "Terratorch" architecture
"PrithviGeoSpatialMAE": "0.28.0",
}
_OOT_SUPPORTED_MODELS = {