[Model] Remove ten deprecated model architectures (#53608)
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com> Co-authored-by: Xianbao QIAN <xianbao.qian@gmail.com> Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Xianbao QIAN
Claude Opus 5
parent
a0f1b9ad05
commit
7ca336929c
@@ -255,8 +255,7 @@ class ApertusDecoderLayer(nn.Module):
|
||||
|
||||
# Apertus defaults to causal attention as it is a decoder-only model.
|
||||
# You can override the HF config with `is_causal=False` to enable
|
||||
# bidirectional attention, which is used in some embedding models
|
||||
# (e.g. parasail-ai/GritLM-7B-vllm)
|
||||
# bidirectional attention, which is used in some embedding models.
|
||||
if getattr(config, "is_causal", True):
|
||||
attn_type = AttentionType.DECODER
|
||||
else:
|
||||
|
||||
@@ -1,593 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Inference-only Snowflake Arctic model."""
|
||||
|
||||
from collections.abc import Iterable
|
||||
from itertools import islice
|
||||
|
||||
import torch
|
||||
from torch import nn
|
||||
|
||||
from vllm.compilation.decorators import support_torch_compile
|
||||
from vllm.config import CacheConfig, VllmConfig
|
||||
from vllm.distributed import (
|
||||
get_pp_group,
|
||||
get_tensor_model_parallel_rank,
|
||||
get_tensor_model_parallel_world_size,
|
||||
tensor_model_parallel_all_reduce,
|
||||
)
|
||||
from vllm.model_executor.layers.activation import SiluAndMul
|
||||
from vllm.model_executor.layers.attention import Attention
|
||||
from vllm.model_executor.layers.fused_moe import (
|
||||
fused_experts,
|
||||
fused_topk,
|
||||
)
|
||||
from vllm.model_executor.layers.layernorm import RMSNorm
|
||||
from vllm.model_executor.layers.linear import (
|
||||
MergedColumnParallelLinear,
|
||||
QKVParallelLinear,
|
||||
ReplicatedLinear,
|
||||
RowParallelLinear,
|
||||
)
|
||||
from vllm.model_executor.layers.logits_processor import LogitsProcessor
|
||||
from vllm.model_executor.layers.quantization import QuantizationConfig
|
||||
from vllm.model_executor.layers.rotary_embedding import get_rope
|
||||
from vllm.model_executor.layers.vocab_parallel_embedding import (
|
||||
ParallelLMHead,
|
||||
VocabParallelEmbedding,
|
||||
)
|
||||
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
|
||||
from vllm.model_executor.utils import set_weight_attrs
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.sequence import IntermediateTensors
|
||||
from vllm.transformers_utils.configs.arctic import ArcticConfig
|
||||
|
||||
from .interfaces import SupportsPP, SupportsQuant
|
||||
from .utils import (
|
||||
AutoWeightsLoader,
|
||||
extract_layer_index,
|
||||
is_pp_missing_parameter,
|
||||
make_empty_intermediate_tensors_factory,
|
||||
make_layers,
|
||||
maybe_prefix,
|
||||
)
|
||||
|
||||
|
||||
class ArcticMLP(nn.Module):
|
||||
def __init__(
|
||||
self,
|
||||
config: ArcticConfig,
|
||||
expert_id: int = -1,
|
||||
is_residual_mlp: bool = False,
|
||||
quant_config: QuantizationConfig | None = None,
|
||||
reduce_results: bool = True,
|
||||
prefix: str = "",
|
||||
):
|
||||
super().__init__()
|
||||
self.hidden_size = config.hidden_size
|
||||
self.expert_id = expert_id
|
||||
|
||||
self.ffn_dim = (
|
||||
config.intermediate_size if not is_residual_mlp else self.hidden_size
|
||||
)
|
||||
|
||||
self.w13 = MergedColumnParallelLinear(
|
||||
self.hidden_size,
|
||||
[self.ffn_dim] * 2,
|
||||
bias=False,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.w13",
|
||||
)
|
||||
self.w2 = RowParallelLinear(
|
||||
self.ffn_dim,
|
||||
self.hidden_size,
|
||||
bias=False,
|
||||
reduce_results=reduce_results,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.w2",
|
||||
)
|
||||
if config.hidden_act != "silu":
|
||||
raise ValueError(
|
||||
f"Unsupported activation: {config.hidden_act}. "
|
||||
"Only silu is supported for now."
|
||||
)
|
||||
self.act_fn = SiluAndMul()
|
||||
|
||||
def forward(self, hidden_states):
|
||||
gate_up, _ = self.w13(hidden_states)
|
||||
hidden_states = self.act_fn(gate_up)
|
||||
hidden_states, _ = self.w2(hidden_states)
|
||||
return hidden_states
|
||||
|
||||
|
||||
class ArcticMoE(nn.Module):
|
||||
"""
|
||||
Model-parallel implementation of Arctic MoE Layer.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
config: ArcticConfig,
|
||||
tp_size: int | None = None,
|
||||
params_dtype: torch.dtype | None = None,
|
||||
quant_config: QuantizationConfig | None = None,
|
||||
reduce_results: bool = True,
|
||||
prefix: str = "",
|
||||
):
|
||||
super().__init__()
|
||||
|
||||
layer_id = extract_layer_index(prefix)
|
||||
self.tp_size = tp_size or get_tensor_model_parallel_world_size()
|
||||
self.hidden_size = config.hidden_size
|
||||
self.num_experts = config.num_local_experts
|
||||
self.layer_id = layer_id
|
||||
self.top_k = config.num_experts_per_tok
|
||||
self.intermediate_size = config.intermediate_size // self.tp_size
|
||||
|
||||
self.is_moe_layer = (layer_id + 1) % config.moe_layer_frequency == 0
|
||||
self.reduce_results = reduce_results
|
||||
# Some other parameters
|
||||
if params_dtype is None:
|
||||
params_dtype = torch.get_default_dtype()
|
||||
self.params_dtype = params_dtype
|
||||
|
||||
if not self.is_moe_layer:
|
||||
self.mlp = ArcticMLP(
|
||||
config,
|
||||
quant_config=quant_config,
|
||||
reduce_results=reduce_results,
|
||||
prefix=f"{prefix}.mlp",
|
||||
)
|
||||
else:
|
||||
self.gate = ReplicatedLinear(
|
||||
self.hidden_size,
|
||||
self.num_experts,
|
||||
bias=False,
|
||||
params_dtype=self.params_dtype,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.gate",
|
||||
)
|
||||
self.ws = nn.Parameter(
|
||||
torch.empty(
|
||||
self.num_experts,
|
||||
2 * self.intermediate_size,
|
||||
self.hidden_size,
|
||||
device=current_platform.device_type,
|
||||
dtype=self.params_dtype,
|
||||
)
|
||||
)
|
||||
self.w2s = nn.Parameter(
|
||||
torch.empty(
|
||||
self.num_experts,
|
||||
self.hidden_size,
|
||||
self.intermediate_size,
|
||||
device=current_platform.device_type,
|
||||
dtype=self.params_dtype,
|
||||
)
|
||||
)
|
||||
set_weight_attrs(
|
||||
self.ws,
|
||||
{
|
||||
"weight_loader": self.weight_loader,
|
||||
},
|
||||
)
|
||||
set_weight_attrs(
|
||||
self.w2s,
|
||||
{
|
||||
"weight_loader": self.weight_loader,
|
||||
},
|
||||
)
|
||||
|
||||
def weight_loader(
|
||||
self,
|
||||
param: nn.Parameter,
|
||||
loaded_weight: torch.Tensor,
|
||||
weight_name: str,
|
||||
expert_id: int,
|
||||
):
|
||||
tp_rank = get_tensor_model_parallel_rank()
|
||||
param_data = param.data
|
||||
shard_size = self.intermediate_size
|
||||
shard = slice(tp_rank * shard_size, (tp_rank + 1) * shard_size)
|
||||
if weight_name.endswith("w1.weight"):
|
||||
param_data[expert_id, 0:shard_size, :] = loaded_weight[shard, :]
|
||||
if weight_name.endswith("w3.weight"):
|
||||
param_data[expert_id, shard_size : 2 * shard_size, :] = loaded_weight[
|
||||
shard, :
|
||||
]
|
||||
if weight_name.endswith("w2.weight"):
|
||||
param_data[expert_id, :, :] = loaded_weight[:, shard]
|
||||
|
||||
def local_moe_fused(self, hidden_states: torch.Tensor) -> torch.Tensor:
|
||||
num_tokens, hidden_size = hidden_states.shape
|
||||
hidden_states = hidden_states.view(-1, self.hidden_size)
|
||||
# router_logits: (num_tokens, n_experts)
|
||||
router_logits, _ = self.gate(hidden_states)
|
||||
do_normalize = self.top_k > 1
|
||||
topk_weights, topk_ids, token_expert_indices = fused_topk(
|
||||
hidden_states, router_logits, self.top_k, renormalize=do_normalize
|
||||
)
|
||||
final_hidden_states = fused_experts(
|
||||
hidden_states,
|
||||
self.ws,
|
||||
self.w2s,
|
||||
topk_weights,
|
||||
topk_ids,
|
||||
)
|
||||
if self.reduce_results and self.tp_size > 1:
|
||||
final_hidden_states = tensor_model_parallel_all_reduce(final_hidden_states)
|
||||
return final_hidden_states.view(num_tokens, hidden_size)
|
||||
|
||||
def forward(self, hidden_states: torch.Tensor):
|
||||
if self.is_moe_layer:
|
||||
final_hidden_states = self.local_moe_fused(hidden_states)
|
||||
else:
|
||||
final_hidden_states = self.mlp(hidden_states)
|
||||
return final_hidden_states
|
||||
|
||||
|
||||
class ArcticAttention(nn.Module):
|
||||
def __init__(
|
||||
self,
|
||||
config: ArcticConfig,
|
||||
cache_config: CacheConfig | None = None,
|
||||
quant_config: QuantizationConfig | None = None,
|
||||
prefix: str = "",
|
||||
):
|
||||
super().__init__()
|
||||
self.config = config
|
||||
self.hidden_size = config.hidden_size
|
||||
|
||||
tp_size = get_tensor_model_parallel_world_size()
|
||||
self.total_num_heads = config.num_attention_heads
|
||||
assert self.total_num_heads % tp_size == 0
|
||||
self.num_heads = self.total_num_heads // tp_size
|
||||
self.total_num_kv_heads = config.num_key_value_heads
|
||||
if self.total_num_kv_heads >= tp_size:
|
||||
assert self.total_num_kv_heads % tp_size == 0
|
||||
else:
|
||||
assert tp_size % self.total_num_kv_heads == 0
|
||||
self.num_kv_heads = max(1, self.total_num_kv_heads // tp_size)
|
||||
self.head_dim = self.hidden_size // self.total_num_heads
|
||||
self.q_size = self.num_heads * self.head_dim
|
||||
self.kv_size = self.num_kv_heads * self.head_dim
|
||||
|
||||
self.max_position_embeddings = config.max_position_embeddings
|
||||
self.scaling = self.head_dim**-0.5
|
||||
|
||||
self.qkv_proj = QKVParallelLinear(
|
||||
self.hidden_size,
|
||||
self.head_dim,
|
||||
self.total_num_heads,
|
||||
self.total_num_kv_heads,
|
||||
bias=False,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.qkv_proj",
|
||||
)
|
||||
self.o_proj = RowParallelLinear(
|
||||
self.total_num_heads * self.head_dim,
|
||||
self.hidden_size,
|
||||
bias=False,
|
||||
reduce_results=True,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.o_proj",
|
||||
)
|
||||
|
||||
self.rotary_emb = get_rope(
|
||||
self.head_dim,
|
||||
max_position=self.max_position_embeddings,
|
||||
rope_parameters=config.rope_parameters,
|
||||
is_neox_style=True,
|
||||
)
|
||||
|
||||
self.attn = Attention(
|
||||
self.num_heads,
|
||||
self.head_dim,
|
||||
self.scaling,
|
||||
num_kv_heads=self.num_kv_heads,
|
||||
cache_config=cache_config,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.attn",
|
||||
)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
positions: torch.Tensor,
|
||||
hidden_states: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
qkv, _ = self.qkv_proj(hidden_states)
|
||||
q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1)
|
||||
q, k = self.rotary_emb(positions, q, k)
|
||||
attn_output = self.attn(q, k, v)
|
||||
output, _ = self.o_proj(attn_output)
|
||||
return output
|
||||
|
||||
|
||||
class ArcticDecoderLayer(nn.Module):
|
||||
def __init__(
|
||||
self,
|
||||
config: ArcticConfig,
|
||||
cache_config: CacheConfig | None = None,
|
||||
quant_config: QuantizationConfig | None = None,
|
||||
prefix: str = "",
|
||||
) -> None:
|
||||
super().__init__()
|
||||
self.hidden_size = config.hidden_size
|
||||
layer_idx = extract_layer_index(prefix)
|
||||
is_moe_layer = (layer_idx + 1) % config.moe_layer_frequency == 0
|
||||
self.use_residual = config.use_residual and is_moe_layer
|
||||
self.self_attn = ArcticAttention(
|
||||
config,
|
||||
cache_config,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.self_attn",
|
||||
)
|
||||
self.block_sparse_moe = ArcticMoE(
|
||||
config,
|
||||
quant_config=quant_config,
|
||||
reduce_results=(not self.use_residual),
|
||||
prefix=f"{prefix}.block_sparse_moe",
|
||||
)
|
||||
|
||||
self.input_layernorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
|
||||
self.post_attention_layernorm = RMSNorm(
|
||||
config.hidden_size, eps=config.rms_norm_eps
|
||||
)
|
||||
|
||||
if self.use_residual:
|
||||
self.residual_layernorm = RMSNorm(
|
||||
config.hidden_size, eps=config.rms_norm_eps
|
||||
)
|
||||
self.residual_mlp = ArcticMLP(
|
||||
config,
|
||||
is_residual_mlp=True,
|
||||
reduce_results=False,
|
||||
prefix=f"{prefix}.residual_mlp",
|
||||
)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
positions: torch.Tensor,
|
||||
hidden_states: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
residual_input = hidden_states
|
||||
hidden_states = self.input_layernorm(hidden_states)
|
||||
hidden_states = self.self_attn(
|
||||
positions=positions,
|
||||
hidden_states=hidden_states,
|
||||
)
|
||||
hidden_states = residual_input + hidden_states
|
||||
|
||||
residual_attn = hidden_states
|
||||
if self.use_residual:
|
||||
hidden_states = self.residual_layernorm(hidden_states)
|
||||
hidden_states = self.residual_mlp(hidden_states)
|
||||
residual_mlp = hidden_states
|
||||
hidden_states = self.post_attention_layernorm(residual_input)
|
||||
hidden_states = self.block_sparse_moe(hidden_states)
|
||||
hidden_states = residual_mlp + hidden_states
|
||||
hidden_states = tensor_model_parallel_all_reduce(hidden_states)
|
||||
hidden_states = residual_attn + hidden_states
|
||||
else:
|
||||
hidden_states = self.post_attention_layernorm(hidden_states)
|
||||
hidden_states = self.block_sparse_moe(hidden_states)
|
||||
hidden_states = residual_attn + hidden_states
|
||||
return hidden_states
|
||||
|
||||
|
||||
@support_torch_compile
|
||||
class ArcticModel(nn.Module):
|
||||
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
|
||||
super().__init__()
|
||||
|
||||
config = vllm_config.model_config.hf_config
|
||||
cache_config = vllm_config.cache_config
|
||||
quant_config = vllm_config.quant_config
|
||||
|
||||
self.config = config
|
||||
self.vocab_size = config.vocab_size
|
||||
self.embed_tokens = VocabParallelEmbedding(
|
||||
self.vocab_size, config.hidden_size, org_num_embeddings=self.vocab_size
|
||||
)
|
||||
self.start_layer, self.end_layer, self.layers = make_layers(
|
||||
config.num_hidden_layers,
|
||||
lambda prefix: ArcticDecoderLayer(
|
||||
config, cache_config, quant_config, prefix=prefix
|
||||
),
|
||||
prefix=f"{prefix}.layers",
|
||||
)
|
||||
self._attn_implementation = config._attn_implementation
|
||||
self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
|
||||
self.make_empty_intermediate_tensors = make_empty_intermediate_tensors_factory(
|
||||
["hidden_states"], config.hidden_size
|
||||
)
|
||||
|
||||
def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
|
||||
return self.embed_tokens(input_ids)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
input_ids: torch.Tensor | None,
|
||||
positions: torch.Tensor,
|
||||
intermediate_tensors: IntermediateTensors | None,
|
||||
inputs_embeds: torch.Tensor | None = None,
|
||||
) -> torch.Tensor | IntermediateTensors:
|
||||
if get_pp_group().is_first_rank:
|
||||
if inputs_embeds is not None:
|
||||
hidden_states = inputs_embeds
|
||||
else:
|
||||
hidden_states = self.embed_input_ids(input_ids)
|
||||
else:
|
||||
assert intermediate_tensors is not None
|
||||
hidden_states = intermediate_tensors["hidden_states"]
|
||||
for layer in islice(self.layers, self.start_layer, self.end_layer):
|
||||
hidden_states = layer(positions, hidden_states)
|
||||
if not get_pp_group().is_last_rank:
|
||||
return IntermediateTensors({"hidden_states": hidden_states})
|
||||
hidden_states = self.norm(hidden_states)
|
||||
return hidden_states
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
stacked_params_mapping = [
|
||||
# (param_name, shard_name, shard_id)
|
||||
("qkv_proj", "q_proj", "q"),
|
||||
("qkv_proj", "k_proj", "k"),
|
||||
("qkv_proj", "v_proj", "v"),
|
||||
]
|
||||
|
||||
mlp_params_mapping: list[tuple[str, str, int]] = []
|
||||
expert_params_mapping: list[tuple[str, str, int]] = []
|
||||
|
||||
for layer in range(self.config.num_hidden_layers):
|
||||
is_moe_layer = (layer + 1) % self.config.moe_layer_frequency == 0
|
||||
if is_moe_layer and self.config.use_residual:
|
||||
mlp_params_mapping.append(
|
||||
(
|
||||
f"layers.{layer}.residual_mlp.w13.weight",
|
||||
f"layers.{layer}.residual_mlp.w1.weight",
|
||||
0,
|
||||
)
|
||||
)
|
||||
mlp_params_mapping.append(
|
||||
(
|
||||
f"layers.{layer}.residual_mlp.w13.weight",
|
||||
f"layers.{layer}.residual_mlp.w3.weight",
|
||||
1,
|
||||
)
|
||||
)
|
||||
|
||||
if is_moe_layer:
|
||||
for expert_id in range(self.config.num_local_experts):
|
||||
expert_params_mapping.append(
|
||||
("ws", f"experts.{expert_id}.w1.weight", expert_id)
|
||||
)
|
||||
expert_params_mapping.append(
|
||||
("w2s", f"experts.{expert_id}.w2.weight", expert_id)
|
||||
)
|
||||
expert_params_mapping.append(
|
||||
("ws", f"experts.{expert_id}.w3.weight", expert_id)
|
||||
)
|
||||
else:
|
||||
mlp_params_mapping.append(
|
||||
(
|
||||
f"layers.{layer}.block_sparse_moe.mlp.w13.weight",
|
||||
f"layers.{layer}.block_sparse_moe.mlp.w1.weight",
|
||||
0,
|
||||
)
|
||||
)
|
||||
mlp_params_mapping.append(
|
||||
(
|
||||
f"layers.{layer}.block_sparse_moe.mlp.w13.weight",
|
||||
f"layers.{layer}.block_sparse_moe.mlp.w3.weight",
|
||||
1,
|
||||
)
|
||||
)
|
||||
|
||||
params_dict = dict(self.named_parameters())
|
||||
loaded_params: set[str] = set()
|
||||
|
||||
for name, loaded_weight in weights:
|
||||
for param_name, weight_name, shard_id in stacked_params_mapping:
|
||||
if weight_name not in name:
|
||||
continue
|
||||
name = name.replace(weight_name, param_name)
|
||||
# Skip loading extra bias for GPTQ models.
|
||||
if name.endswith(".bias") and name not in params_dict:
|
||||
continue
|
||||
if is_pp_missing_parameter(name, self):
|
||||
continue
|
||||
param = params_dict[name]
|
||||
weight_loader = param.weight_loader
|
||||
weight_loader(param, loaded_weight, shard_id)
|
||||
break
|
||||
else:
|
||||
for param_name, weight_name, mlp_shard_id in mlp_params_mapping:
|
||||
if weight_name not in name:
|
||||
continue
|
||||
name = name.replace(weight_name, param_name)
|
||||
if is_pp_missing_parameter(name, self):
|
||||
continue
|
||||
param = params_dict[name]
|
||||
weight_loader = param.weight_loader
|
||||
weight_loader(param, loaded_weight, mlp_shard_id)
|
||||
break
|
||||
else:
|
||||
for param_name, weight_name, expert_id in expert_params_mapping:
|
||||
if weight_name not in name:
|
||||
continue
|
||||
name = name.replace(weight_name, param_name)
|
||||
if is_pp_missing_parameter(name, self):
|
||||
continue
|
||||
param = params_dict[name]
|
||||
weight_loader = param.weight_loader
|
||||
weight_loader(
|
||||
param, loaded_weight, weight_name, expert_id=expert_id
|
||||
)
|
||||
break
|
||||
else:
|
||||
if name.endswith(".bias") and name not in params_dict:
|
||||
continue
|
||||
if is_pp_missing_parameter(name, self):
|
||||
continue
|
||||
param = params_dict[name]
|
||||
weight_loader = getattr(
|
||||
param, "weight_loader", default_weight_loader
|
||||
)
|
||||
weight_loader(param, loaded_weight)
|
||||
loaded_params.add(name)
|
||||
return loaded_params
|
||||
|
||||
|
||||
class ArcticForCausalLM(nn.Module, SupportsPP, SupportsQuant):
|
||||
packed_modules_mapping = {"qkv_proj": ["q_proj", "k_proj", "v_proj"]}
|
||||
|
||||
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
|
||||
super().__init__()
|
||||
config = vllm_config.model_config.hf_config
|
||||
quant_config = vllm_config.quant_config
|
||||
self.config = config
|
||||
self.model = ArcticModel(
|
||||
vllm_config=vllm_config, prefix=maybe_prefix(prefix, "model")
|
||||
)
|
||||
self.vocab_size = config.vocab_size
|
||||
self.lm_head = ParallelLMHead(
|
||||
self.vocab_size,
|
||||
config.hidden_size,
|
||||
quant_config=quant_config,
|
||||
prefix=maybe_prefix(prefix, "lm_head"),
|
||||
)
|
||||
if self.config.tie_word_embeddings:
|
||||
self.lm_head = self.lm_head.tie_weights(self.model.embed_tokens)
|
||||
self.num_experts = config.num_local_experts
|
||||
self.num_experts_per_tok = config.num_experts_per_tok
|
||||
|
||||
self.logits_processor = LogitsProcessor(config.vocab_size)
|
||||
self.make_empty_intermediate_tensors = (
|
||||
self.model.make_empty_intermediate_tensors
|
||||
)
|
||||
|
||||
def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
|
||||
return self.model.embed_input_ids(input_ids)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
input_ids: torch.Tensor | None,
|
||||
positions: torch.Tensor,
|
||||
intermediate_tensors: IntermediateTensors | None = None,
|
||||
inputs_embeds: torch.Tensor | None = None,
|
||||
) -> torch.Tensor | IntermediateTensors:
|
||||
hidden_states = self.model(
|
||||
input_ids, positions, intermediate_tensors, inputs_embeds
|
||||
)
|
||||
return hidden_states
|
||||
|
||||
def compute_logits(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
) -> torch.Tensor | None:
|
||||
logits = self.logits_processor(self.lm_head, hidden_states)
|
||||
return logits
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
loader = AutoWeightsLoader(self)
|
||||
return loader.load_weights(weights)
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,734 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Inference-only Cheers (UMM) model compatible with HuggingFace weights.
|
||||
|
||||
Cheers is a unified multimodal model for image understanding and generation.
|
||||
For vLLM, we focus on the image understanding (vision-to-text) capabilities.
|
||||
The image generation part (gen_projector, hi_gate, etc.) is not supported,
|
||||
but the VAE encoder + decoder projector are required for image understanding.
|
||||
"""
|
||||
|
||||
import math
|
||||
from collections.abc import Iterable, Mapping, Sequence
|
||||
from typing import Any, Literal, TypeAlias
|
||||
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
import torch.nn.functional as F
|
||||
from einops import rearrange
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.config.multimodal import BaseDummyOptions
|
||||
from vllm.inputs import MultiModalDataDict
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.quantization import QuantizationConfig
|
||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
from vllm.multimodal.inputs import (
|
||||
MultiModalFieldConfig,
|
||||
MultiModalKwargsItems,
|
||||
)
|
||||
from vllm.multimodal.parse import MultiModalDataItems
|
||||
from vllm.multimodal.processing import (
|
||||
BaseDummyInputsBuilder,
|
||||
BaseMultiModalProcessor,
|
||||
BaseProcessingInfo,
|
||||
PromptReplacement,
|
||||
)
|
||||
from vllm.sequence import IntermediateTensors
|
||||
from vllm.transformers_utils.processors.cheers import CheersProcessor
|
||||
from vllm.utils.tensor_schema import TensorSchema
|
||||
|
||||
from .interfaces import (
|
||||
MultiModalEmbeddings,
|
||||
SupportsLoRA,
|
||||
SupportsMultiModal,
|
||||
SupportsPP,
|
||||
)
|
||||
from .siglip import SiglipVisionModel
|
||||
from .utils import (
|
||||
AutoWeightsLoader,
|
||||
WeightsMapper,
|
||||
init_vllm_registered_model,
|
||||
maybe_prefix,
|
||||
)
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
# ── VAE components (needed for image understanding pipeline) ────────
|
||||
|
||||
|
||||
def _swish(x: torch.Tensor) -> torch.Tensor:
|
||||
return x * torch.sigmoid(x)
|
||||
|
||||
|
||||
class _AttnBlock(nn.Module):
|
||||
def __init__(self, in_channels: int):
|
||||
super().__init__()
|
||||
self.norm = nn.GroupNorm(32, in_channels, eps=1e-6, affine=True)
|
||||
self.q = nn.Conv2d(in_channels, in_channels, 1)
|
||||
self.k = nn.Conv2d(in_channels, in_channels, 1)
|
||||
self.v = nn.Conv2d(in_channels, in_channels, 1)
|
||||
self.proj_out = nn.Conv2d(in_channels, in_channels, 1)
|
||||
|
||||
def forward(self, x: torch.Tensor) -> torch.Tensor:
|
||||
h_ = self.norm(x)
|
||||
q = self.q(h_)
|
||||
k = self.k(h_)
|
||||
v = self.v(h_)
|
||||
b, c, h, w = q.shape
|
||||
q = rearrange(q, "b c h w -> b 1 (h w) c").contiguous()
|
||||
k = rearrange(k, "b c h w -> b 1 (h w) c").contiguous()
|
||||
v = rearrange(v, "b c h w -> b 1 (h w) c").contiguous()
|
||||
h_ = F.scaled_dot_product_attention(q, k, v)
|
||||
h_ = rearrange(h_, "b 1 (h w) c -> b c h w", h=h, w=w, c=c, b=b)
|
||||
return x + self.proj_out(h_)
|
||||
|
||||
|
||||
class _ResnetBlock(nn.Module):
|
||||
def __init__(self, in_channels: int, out_channels: int):
|
||||
super().__init__()
|
||||
self.in_channels = in_channels
|
||||
self.out_channels = out_channels
|
||||
self.norm1 = nn.GroupNorm(32, in_channels, eps=1e-6, affine=True)
|
||||
self.conv1 = nn.Conv2d(in_channels, out_channels, 3, 1, 1)
|
||||
self.norm2 = nn.GroupNorm(32, out_channels, eps=1e-6, affine=True)
|
||||
self.conv2 = nn.Conv2d(out_channels, out_channels, 3, 1, 1)
|
||||
if in_channels != out_channels:
|
||||
self.nin_shortcut = nn.Conv2d(in_channels, out_channels, 1)
|
||||
|
||||
def forward(self, x: torch.Tensor) -> torch.Tensor:
|
||||
h = _swish(self.norm1(x))
|
||||
h = self.conv1(h)
|
||||
h = _swish(self.norm2(h))
|
||||
h = self.conv2(h)
|
||||
if self.in_channels != self.out_channels:
|
||||
x = self.nin_shortcut(x)
|
||||
return x + h
|
||||
|
||||
|
||||
class _Downsample(nn.Module):
|
||||
def __init__(self, in_channels: int):
|
||||
super().__init__()
|
||||
self.conv = nn.Conv2d(in_channels, in_channels, 3, stride=2, padding=0)
|
||||
|
||||
def forward(self, x: torch.Tensor) -> torch.Tensor:
|
||||
x = F.pad(x, (0, 1, 0, 1), mode="constant", value=0)
|
||||
return self.conv(x)
|
||||
|
||||
|
||||
class _Upsample(nn.Module):
|
||||
def __init__(self, in_channels: int):
|
||||
super().__init__()
|
||||
self.conv = nn.Conv2d(in_channels, in_channels, 3, 1, 1)
|
||||
|
||||
def forward(self, x: torch.Tensor) -> torch.Tensor:
|
||||
x = F.interpolate(x, scale_factor=2.0, mode="nearest")
|
||||
return self.conv(x)
|
||||
|
||||
|
||||
_VAE_ENCODER_DEFAULTS = {
|
||||
"in_channels": 3,
|
||||
"ch": 128,
|
||||
"ch_mult": [1, 2, 4, 4],
|
||||
"num_res_blocks": 2,
|
||||
"z_channels": 32,
|
||||
}
|
||||
_VAE_DECODER_DEFAULTS = {
|
||||
"in_channels": 3,
|
||||
"out_ch": 3,
|
||||
"ch": 128,
|
||||
"ch_mult": [1, 2, 4, 4],
|
||||
"num_res_blocks": 2,
|
||||
"z_channels": 32,
|
||||
}
|
||||
|
||||
|
||||
def _cfg(config, key, defaults=None):
|
||||
"""Access config attribute whether it's a dict or namespace object."""
|
||||
if isinstance(config, dict):
|
||||
if key in config:
|
||||
return config[key]
|
||||
if defaults and key in defaults:
|
||||
return defaults[key]
|
||||
raise KeyError(f"Key '{key}' not found in config dict: {list(config.keys())}")
|
||||
return getattr(config, key)
|
||||
|
||||
|
||||
class CheersVAEEncoder(nn.Module):
|
||||
"""VAE encoder from the Cheers/UMM model."""
|
||||
|
||||
def __init__(self, config):
|
||||
super().__init__()
|
||||
d = _VAE_ENCODER_DEFAULTS
|
||||
ch = _cfg(config, "ch", d)
|
||||
ch_mult = _cfg(config, "ch_mult", d)
|
||||
num_res_blocks = _cfg(config, "num_res_blocks", d)
|
||||
z_channels = _cfg(config, "z_channels", d)
|
||||
in_channels = _cfg(config, "in_channels", d)
|
||||
num_resolutions = len(ch_mult)
|
||||
|
||||
self.quant_conv = nn.Conv2d(2 * z_channels, 2 * z_channels, 1)
|
||||
self.conv_in = nn.Conv2d(in_channels, ch, 3, 1, 1)
|
||||
|
||||
in_ch_mult = (1,) + tuple(ch_mult)
|
||||
self.down = nn.ModuleList()
|
||||
block_in = ch
|
||||
for i_level in range(num_resolutions):
|
||||
block = nn.ModuleList()
|
||||
attn = nn.ModuleList()
|
||||
block_in = ch * in_ch_mult[i_level]
|
||||
block_out = ch * ch_mult[i_level]
|
||||
for _ in range(num_res_blocks):
|
||||
block.append(_ResnetBlock(block_in, block_out))
|
||||
block_in = block_out
|
||||
down = nn.Module()
|
||||
down.block = block
|
||||
down.attn = attn
|
||||
if i_level != num_resolutions - 1:
|
||||
down.downsample = _Downsample(block_in)
|
||||
self.down.append(down)
|
||||
|
||||
self.mid = nn.Module()
|
||||
self.mid.block_1 = _ResnetBlock(block_in, block_in)
|
||||
self.mid.attn_1 = _AttnBlock(block_in)
|
||||
self.mid.block_2 = _ResnetBlock(block_in, block_in)
|
||||
|
||||
self.norm_out = nn.GroupNorm(32, block_in, eps=1e-6, affine=True)
|
||||
self.conv_out = nn.Conv2d(block_in, 2 * z_channels, 3, 1, 1)
|
||||
self._num_resolutions = num_resolutions
|
||||
self._num_res_blocks = num_res_blocks
|
||||
|
||||
def forward(self, x: torch.Tensor) -> torch.Tensor:
|
||||
hs = [self.conv_in(x)]
|
||||
for i_level in range(self._num_resolutions):
|
||||
for i_block in range(self._num_res_blocks):
|
||||
h = self.down[i_level].block[i_block](hs[-1])
|
||||
if len(self.down[i_level].attn) > 0:
|
||||
h = self.down[i_level].attn[i_block](h)
|
||||
hs.append(h)
|
||||
if hasattr(self.down[i_level], "downsample"):
|
||||
hs.append(self.down[i_level].downsample(hs[-1]))
|
||||
h = hs[-1]
|
||||
h = self.mid.block_1(h)
|
||||
h = self.mid.attn_1(h)
|
||||
h = self.mid.block_2(h)
|
||||
h = _swish(self.norm_out(h))
|
||||
h = self.conv_out(h)
|
||||
h = self.quant_conv(h)
|
||||
return h
|
||||
|
||||
|
||||
class CheersVAEDecoder(nn.Module):
|
||||
"""VAE decoder (used inside VAEDecoderProjector)."""
|
||||
|
||||
def __init__(self, config):
|
||||
super().__init__()
|
||||
d = _VAE_DECODER_DEFAULTS
|
||||
ch = _cfg(config, "ch", d)
|
||||
ch_mult = _cfg(config, "ch_mult", d)
|
||||
num_res_blocks = _cfg(config, "num_res_blocks", d)
|
||||
z_channels = _cfg(config, "z_channels", d)
|
||||
out_ch = _cfg(config, "out_ch", d)
|
||||
num_resolutions = len(ch_mult)
|
||||
|
||||
self.post_quant_conv = nn.Conv2d(z_channels, z_channels, 1)
|
||||
block_in = ch * ch_mult[num_resolutions - 1]
|
||||
self.conv_in = nn.Conv2d(z_channels, block_in, 3, 1, 1)
|
||||
|
||||
self.mid = nn.Module()
|
||||
self.mid.block_1 = _ResnetBlock(block_in, block_in)
|
||||
self.mid.attn_1 = _AttnBlock(block_in)
|
||||
self.mid.block_2 = _ResnetBlock(block_in, block_in)
|
||||
|
||||
self.up = nn.ModuleList()
|
||||
for i_level in reversed(range(num_resolutions)):
|
||||
block = nn.ModuleList()
|
||||
attn = nn.ModuleList()
|
||||
block_out = ch * ch_mult[i_level]
|
||||
for _ in range(num_res_blocks + 1):
|
||||
block.append(_ResnetBlock(block_in, block_out))
|
||||
block_in = block_out
|
||||
up = nn.Module()
|
||||
up.block = block
|
||||
up.attn = attn
|
||||
if i_level != 0:
|
||||
up.upsample = _Upsample(block_in)
|
||||
self.up.insert(0, up)
|
||||
|
||||
self.norm_out = nn.GroupNorm(32, block_in, eps=1e-6, affine=True)
|
||||
self.conv_out = nn.Conv2d(block_in, out_ch, 3, 1, 1)
|
||||
self._num_resolutions = num_resolutions
|
||||
self._num_res_blocks = num_res_blocks
|
||||
|
||||
def forward(self, z: torch.Tensor) -> torch.Tensor:
|
||||
z = self.post_quant_conv(z)
|
||||
upscale_dtype = next(self.up.parameters()).dtype
|
||||
h = self.conv_in(z)
|
||||
h = self.mid.block_1(h)
|
||||
h = self.mid.attn_1(h)
|
||||
h = self.mid.block_2(h)
|
||||
h = h.to(upscale_dtype)
|
||||
for i_level in reversed(range(self._num_resolutions)):
|
||||
for i_block in range(self._num_res_blocks + 1):
|
||||
h = self.up[i_level].block[i_block](h)
|
||||
if len(self.up[i_level].attn) > 0:
|
||||
h = self.up[i_level].attn[i_block](h)
|
||||
if i_level != 0:
|
||||
h = self.up[i_level].upsample(h)
|
||||
h = _swish(self.norm_out(h))
|
||||
return self.conv_out(h)
|
||||
|
||||
|
||||
class CheersVAEModel(nn.Module):
|
||||
"""VAE model with encoder only (for image understanding)."""
|
||||
|
||||
def __init__(self, config):
|
||||
super().__init__()
|
||||
enc_cfg = _cfg(config, "vae_encoder_config")
|
||||
self.encoder = CheersVAEEncoder(enc_cfg)
|
||||
self.ps = [2, 2]
|
||||
z_ch = _cfg(enc_cfg, "z_channels", _VAE_ENCODER_DEFAULTS)
|
||||
self.bn = nn.BatchNorm2d(
|
||||
math.prod(self.ps) * z_ch,
|
||||
eps=1e-4,
|
||||
momentum=0.1,
|
||||
affine=False,
|
||||
track_running_stats=True,
|
||||
)
|
||||
|
||||
def encode(self, x: torch.Tensor) -> torch.Tensor:
|
||||
self.bn.eval()
|
||||
moments = self.encoder(x)
|
||||
mean = torch.chunk(moments, 2, dim=1)[0]
|
||||
z = rearrange(
|
||||
mean,
|
||||
"... c (i pi) (j pj) -> ... (c pi pj) i j",
|
||||
pi=self.ps[0],
|
||||
pj=self.ps[1],
|
||||
)
|
||||
return self.bn(z)
|
||||
|
||||
|
||||
class CheersVAEDecoderProjector(nn.Module):
|
||||
"""VAE decoder projector that converts latent back to pixel-like space."""
|
||||
|
||||
def __init__(self, config):
|
||||
super().__init__()
|
||||
dec_cfg = _cfg(config, "vae_decoder_config")
|
||||
enc_cfg = _cfg(config, "vae_encoder_config")
|
||||
self.decoder = CheersVAEDecoder(dec_cfg)
|
||||
self.ps = [2, 2]
|
||||
z_ch = _cfg(enc_cfg, "z_channels", _VAE_ENCODER_DEFAULTS)
|
||||
self.bn = nn.BatchNorm2d(
|
||||
math.prod(self.ps) * z_ch,
|
||||
eps=1e-4,
|
||||
momentum=0.1,
|
||||
affine=False,
|
||||
track_running_stats=True,
|
||||
)
|
||||
|
||||
def forward(self, z: torch.Tensor) -> torch.Tensor:
|
||||
self.bn.eval()
|
||||
s = torch.sqrt(self.bn.running_var.view(1, -1, 1, 1) + 1e-4)
|
||||
m = self.bn.running_mean.view(1, -1, 1, 1)
|
||||
z = z * s + m
|
||||
z = rearrange(
|
||||
z,
|
||||
"... (c pi pj) i j -> ... c (i pi) (j pj)",
|
||||
pi=self.ps[0],
|
||||
pj=self.ps[1],
|
||||
)
|
||||
return self.decoder(z)
|
||||
|
||||
|
||||
class CheersImagePixelInputs(TensorSchema):
|
||||
"""
|
||||
Dimensions:
|
||||
- bn: Batch size * number of images
|
||||
- c: Number of channels (3)
|
||||
- h: Height of each image
|
||||
- w: Width of each image
|
||||
"""
|
||||
|
||||
type: Literal["pixel_values"]
|
||||
pixel_values: torch.Tensor # Shape: (bn, 3, h, w)
|
||||
|
||||
|
||||
CheersImageInputs: TypeAlias = CheersImagePixelInputs
|
||||
|
||||
|
||||
class CheersUndProjector(nn.Module):
|
||||
"""Understanding projector that maps vision features to LLM dimension
|
||||
with 2x2 spatial compression (4x token reduction)."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
image_embed_dim: int,
|
||||
text_embed_dim: int,
|
||||
compression_factor: tuple[int, int] = (2, 2),
|
||||
quant_config: QuantizationConfig | None = None,
|
||||
prefix: str = "",
|
||||
):
|
||||
super().__init__()
|
||||
self.image_embed_dim = image_embed_dim
|
||||
self.text_embed_dim = text_embed_dim
|
||||
self.compression_factor = compression_factor
|
||||
self.layernorm = nn.LayerNorm(image_embed_dim)
|
||||
hidden_size = image_embed_dim * (compression_factor[0] * compression_factor[1])
|
||||
self.mlp = nn.Sequential(
|
||||
nn.Linear(hidden_size, hidden_size),
|
||||
nn.GELU(),
|
||||
nn.Linear(hidden_size, text_embed_dim),
|
||||
)
|
||||
|
||||
def forward(self, x: torch.Tensor) -> torch.Tensor:
|
||||
x = self.layernorm(x)
|
||||
height = width = int(x.size(1) ** 0.5)
|
||||
x = x.permute(0, 2, 1).unflatten(-1, (height, width))
|
||||
batch_size, dim, height, width = x.shape
|
||||
unfolded = x.unfold(
|
||||
2, self.compression_factor[0], self.compression_factor[0]
|
||||
).unfold(3, self.compression_factor[1], self.compression_factor[1])
|
||||
unfolded = unfolded.contiguous().view(
|
||||
batch_size,
|
||||
dim,
|
||||
-1,
|
||||
self.compression_factor[0] * self.compression_factor[1],
|
||||
)
|
||||
unfolded = (
|
||||
unfolded.permute(0, 2, 3, 1)
|
||||
.contiguous()
|
||||
.view(
|
||||
batch_size,
|
||||
-1,
|
||||
dim * self.compression_factor[0] * self.compression_factor[1],
|
||||
)
|
||||
)
|
||||
return self.mlp(unfolded)
|
||||
|
||||
|
||||
class CheersProcessingInfo(BaseProcessingInfo):
|
||||
"""Processing information for Cheers model."""
|
||||
|
||||
def get_hf_processor(self, **kwargs: object) -> CheersProcessor:
|
||||
from vllm.transformers_utils.processor import cached_get_image_processor
|
||||
|
||||
image_processor = cached_get_image_processor(
|
||||
self.ctx.model_config.model,
|
||||
revision=self.ctx.model_config.revision,
|
||||
trust_remote_code=self.ctx.model_config.trust_remote_code,
|
||||
)
|
||||
|
||||
tokenizer = self.get_tokenizer()
|
||||
|
||||
return CheersProcessor(
|
||||
image_processor=image_processor,
|
||||
tokenizer=tokenizer,
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
def get_supported_mm_limits(self) -> Mapping[str, int | None]:
|
||||
return {"image": None}
|
||||
|
||||
def get_mm_max_tokens_per_item(
|
||||
self,
|
||||
seq_len: int,
|
||||
mm_counts: Mapping[str, int],
|
||||
) -> Mapping[str, int]:
|
||||
hf_config = self.get_hf_config()
|
||||
vit_config = hf_config.vision_representation_config
|
||||
patch_size = vit_config.patch_size
|
||||
image_size = vit_config.image_size
|
||||
num_patches = (image_size // patch_size) ** 2
|
||||
# After 2x2 compression, tokens reduce by 4x
|
||||
num_tokens = num_patches // 4
|
||||
return {"image": num_tokens}
|
||||
|
||||
def get_num_image_tokens(
|
||||
self,
|
||||
*,
|
||||
image_width: int,
|
||||
image_height: int,
|
||||
) -> int:
|
||||
hf_config = self.get_hf_config()
|
||||
vit_config = hf_config.vision_representation_config
|
||||
patch_size = vit_config.patch_size
|
||||
image_size = vit_config.image_size
|
||||
num_patches = (image_size // patch_size) ** 2
|
||||
return num_patches // 4
|
||||
|
||||
|
||||
class CheersDummyInputsBuilder(BaseDummyInputsBuilder[CheersProcessingInfo]):
|
||||
"""Build dummy inputs for Cheers model profiling."""
|
||||
|
||||
def get_dummy_text(self, mm_counts: Mapping[str, int]) -> str:
|
||||
num_images = mm_counts.get("image", 0)
|
||||
return "<|image_pad|>" * num_images
|
||||
|
||||
def get_dummy_mm_data(
|
||||
self,
|
||||
seq_len: int,
|
||||
mm_counts: Mapping[str, int],
|
||||
mm_options: Mapping[str, BaseDummyOptions] | None = None,
|
||||
) -> MultiModalDataDict:
|
||||
num_images = mm_counts.get("image", 0)
|
||||
hf_config = self.info.get_hf_config()
|
||||
vit_config = hf_config.vision_representation_config
|
||||
image_size = vit_config.image_size
|
||||
image_overrides = mm_options.get("image") if mm_options else None
|
||||
|
||||
return {
|
||||
"image": self._get_dummy_images(
|
||||
width=image_size,
|
||||
height=image_size,
|
||||
num_images=num_images,
|
||||
overrides=image_overrides,
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
class CheersMultiModalProcessor(BaseMultiModalProcessor[CheersProcessingInfo]):
|
||||
"""Multimodal processor for Cheers model."""
|
||||
|
||||
def _get_prompt_updates(
|
||||
self,
|
||||
mm_items: MultiModalDataItems,
|
||||
hf_processor_mm_kwargs: Mapping[str, Any],
|
||||
out_mm_kwargs: MultiModalKwargsItems,
|
||||
) -> Sequence[PromptReplacement]:
|
||||
hf_config = self.info.get_hf_config()
|
||||
vit_config = hf_config.vision_representation_config
|
||||
patch_size = vit_config.patch_size
|
||||
image_size = vit_config.image_size
|
||||
|
||||
tokenizer = self.info.get_tokenizer()
|
||||
image_token_id = tokenizer.get_vocab().get("<|image_pad|>")
|
||||
if image_token_id is None:
|
||||
raise ValueError(
|
||||
"Image token '<|image_pad|>' not found in tokenizer vocabulary"
|
||||
)
|
||||
|
||||
def get_replacement_cheers(item_idx: int):
|
||||
num_patches = (image_size // patch_size) ** 2
|
||||
num_tokens = num_patches // 4
|
||||
return [image_token_id] * num_tokens
|
||||
|
||||
return [
|
||||
PromptReplacement(
|
||||
modality="image",
|
||||
target=[image_token_id],
|
||||
replacement=get_replacement_cheers,
|
||||
)
|
||||
]
|
||||
|
||||
def _get_mm_fields_config(
|
||||
self,
|
||||
hf_inputs: Any,
|
||||
hf_processor_mm_kwargs: Mapping[str, object],
|
||||
) -> Mapping[str, MultiModalFieldConfig]:
|
||||
return {
|
||||
"pixel_values": MultiModalFieldConfig.batched("image"),
|
||||
}
|
||||
|
||||
|
||||
@MULTIMODAL_REGISTRY.register_processor(
|
||||
CheersMultiModalProcessor,
|
||||
info=CheersProcessingInfo,
|
||||
dummy_inputs=CheersDummyInputsBuilder,
|
||||
)
|
||||
class CheersForConditionalGeneration(
|
||||
nn.Module, SupportsMultiModal, SupportsLoRA, SupportsPP
|
||||
):
|
||||
"""
|
||||
Cheers: A unified multimodal model for image understanding and generation.
|
||||
|
||||
For vLLM, we focus on the image understanding (vision-to-text) capabilities.
|
||||
The image generation part is not supported in vLLM.
|
||||
"""
|
||||
|
||||
requires_raw_input_tokens = True
|
||||
|
||||
hf_to_vllm_mapper = WeightsMapper(
|
||||
orig_to_new_prefix={
|
||||
"model.language_model.": "language_model.model.",
|
||||
"model.vision_representation.": "vision_representation.vision_model.",
|
||||
"model.und_projector.": "und_projector.",
|
||||
"model.vae_model.": "vae_model.",
|
||||
"model.vae_decoder_projector.": "vae_decoder_projector.",
|
||||
"lm_head.": "language_model.lm_head.",
|
||||
}
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def get_placeholder_str(cls, modality: str, i: int) -> str | None:
|
||||
if modality.startswith("image"):
|
||||
return "<|image_pad|>"
|
||||
raise ValueError("Only image modality is supported")
|
||||
|
||||
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
|
||||
super().__init__()
|
||||
|
||||
config = vllm_config.model_config.hf_config
|
||||
quant_config = vllm_config.quant_config
|
||||
multimodal_config = vllm_config.model_config.multimodal_config
|
||||
|
||||
if type(config).__name__ not in ("CheersConfig", "UMMConfig"):
|
||||
raise ValueError(
|
||||
f"Expected CheersConfig or UMMConfig, got {type(config).__name__}."
|
||||
)
|
||||
|
||||
self.config = config
|
||||
self.multimodal_config = multimodal_config
|
||||
|
||||
# The Cheers model's custom Qwen2Config defaults rope_theta to
|
||||
# 1_000_000, but this isn't stored in the JSON. vLLM's standard
|
||||
# Qwen2Config defaults to 10_000, causing a 100× mismatch.
|
||||
# We must patch BOTH the attribute AND rope_parameters (which
|
||||
# patch_rope_parameters may have already populated from the wrong
|
||||
# default before __init__ runs).
|
||||
_CHEERS_ROPE_THETA = 1_000_000.0
|
||||
tc = config.text_config
|
||||
old_theta = getattr(tc, "rope_theta", None)
|
||||
if old_theta != _CHEERS_ROPE_THETA:
|
||||
logger.info(
|
||||
"Overriding text_config.rope_theta from %s to %s",
|
||||
old_theta,
|
||||
_CHEERS_ROPE_THETA,
|
||||
)
|
||||
tc.rope_theta = _CHEERS_ROPE_THETA
|
||||
rp = getattr(tc, "rope_parameters", None)
|
||||
if rp is not None and rp.get("rope_theta") != _CHEERS_ROPE_THETA:
|
||||
logger.info(
|
||||
"Overriding rope_parameters.rope_theta from %s to %s",
|
||||
rp.get("rope_theta"),
|
||||
_CHEERS_ROPE_THETA,
|
||||
)
|
||||
rp["rope_theta"] = _CHEERS_ROPE_THETA
|
||||
|
||||
with self._mark_language_model(vllm_config):
|
||||
self.language_model = init_vllm_registered_model(
|
||||
vllm_config=vllm_config,
|
||||
hf_config=config.text_config,
|
||||
prefix=maybe_prefix(prefix, "language_model"),
|
||||
architectures=["Qwen2ForCausalLM"],
|
||||
)
|
||||
|
||||
vit_config = config.vision_representation_config
|
||||
|
||||
with self._mark_tower_model(vllm_config, "image"):
|
||||
self.vae_model = CheersVAEModel(config)
|
||||
self.vae_decoder_projector = CheersVAEDecoderProjector(config)
|
||||
|
||||
self.vision_representation = SiglipVisionModel(
|
||||
config=vit_config,
|
||||
quant_config=quant_config,
|
||||
prefix=maybe_prefix(prefix, "vision_representation"),
|
||||
)
|
||||
|
||||
vit_hidden_size = vit_config.hidden_size
|
||||
llm_hidden_size = config.text_config.hidden_size
|
||||
|
||||
self.und_projector = CheersUndProjector(
|
||||
image_embed_dim=vit_hidden_size,
|
||||
text_embed_dim=llm_hidden_size,
|
||||
compression_factor=(2, 2),
|
||||
quant_config=quant_config,
|
||||
prefix=maybe_prefix(prefix, "und_projector"),
|
||||
)
|
||||
|
||||
self.make_empty_intermediate_tensors = (
|
||||
self.language_model.make_empty_intermediate_tensors
|
||||
)
|
||||
|
||||
def _parse_and_validate_image_input(
|
||||
self, **kwargs: object
|
||||
) -> CheersImageInputs | None:
|
||||
pixel_values = kwargs.pop("pixel_values", None)
|
||||
if pixel_values is None:
|
||||
return None
|
||||
return CheersImagePixelInputs(
|
||||
type="pixel_values",
|
||||
pixel_values=pixel_values,
|
||||
)
|
||||
|
||||
def _process_image_input(
|
||||
self, image_input: CheersImageInputs
|
||||
) -> tuple[torch.Tensor, ...]:
|
||||
"""Process image inputs through VAE → SigLIP → projector pipeline.
|
||||
|
||||
HF native path: pixel_values → VAE.encode(t=1.0) → vae_decoder_projector
|
||||
→ SigLIP → und_projector → text-space embeddings
|
||||
"""
|
||||
pixel_values = image_input["pixel_values"]
|
||||
|
||||
if pixel_values.ndim == 5:
|
||||
batch_size, num_images, channels, height, width = pixel_values.shape
|
||||
pixel_values = pixel_values.reshape(
|
||||
batch_size * num_images, channels, height, width
|
||||
)
|
||||
|
||||
with torch.no_grad():
|
||||
vae_dtype = next(self.vae_model.parameters()).dtype
|
||||
image_latent = self.vae_model.encode(pixel_values.to(dtype=vae_dtype))
|
||||
image_pixel_hat = self.vae_decoder_projector(image_latent)
|
||||
|
||||
vision_features = self.vision_representation(image_pixel_hat)
|
||||
vision_embeds = self.und_projector(vision_features)
|
||||
|
||||
return tuple(vision_embeds)
|
||||
|
||||
def embed_multimodal(self, **kwargs: object) -> MultiModalEmbeddings:
|
||||
image_input = self._parse_and_validate_image_input(**kwargs)
|
||||
if image_input is None:
|
||||
return []
|
||||
return self._process_image_input(image_input)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
input_ids: torch.Tensor | None,
|
||||
positions: torch.Tensor,
|
||||
intermediate_tensors: IntermediateTensors | None = None,
|
||||
inputs_embeds: torch.Tensor | None = None,
|
||||
**kwargs: object,
|
||||
) -> torch.Tensor | IntermediateTensors:
|
||||
if intermediate_tensors is not None:
|
||||
inputs_embeds = None
|
||||
|
||||
hidden_states = self.language_model.model(
|
||||
input_ids=input_ids,
|
||||
positions=positions,
|
||||
intermediate_tensors=intermediate_tensors,
|
||||
inputs_embeds=inputs_embeds,
|
||||
)
|
||||
return hidden_states
|
||||
|
||||
def compute_logits(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
) -> torch.Tensor | None:
|
||||
return self.language_model.compute_logits(hidden_states)
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
"""Load weights, keeping VAE encoder/decoder projector for understanding."""
|
||||
skip_prefixes = [
|
||||
"model.time_embed.",
|
||||
"model.gen_projector.",
|
||||
"model.hi_gate.",
|
||||
"model.hi_projector.",
|
||||
"model.vae_model.decoder.",
|
||||
]
|
||||
skip_keywords = [
|
||||
"text_loss_fc",
|
||||
]
|
||||
|
||||
filtered_weights = []
|
||||
for name, tensor in weights:
|
||||
if any(name.startswith(p) for p in skip_prefixes):
|
||||
continue
|
||||
if any(kw in name for kw in skip_keywords):
|
||||
continue
|
||||
filtered_weights.append((name, tensor))
|
||||
|
||||
loader = AutoWeightsLoader(self)
|
||||
return loader.load_weights(filtered_weights, mapper=self.hf_to_vllm_mapper)
|
||||
@@ -1,11 +1,10 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
Shared Conformer encoder components for FireRedASR2 and FireRedLID.
|
||||
Conformer encoder components for FireRedASR2.
|
||||
|
||||
Both models use the same Conformer-based audio encoder architecture
|
||||
The audio encoder architecture is
|
||||
(Conv2dSubsampling → RelPositionalEncoding → N × RelPosEmbConformerBlock).
|
||||
This module factors out the common building blocks to avoid duplication.
|
||||
"""
|
||||
|
||||
import torch
|
||||
@@ -288,7 +287,7 @@ class RelPosEmbConformerBlock(nn.Module):
|
||||
|
||||
class ConformerEncoder(nn.Module):
|
||||
"""
|
||||
Conformer encoder shared by FireRedASR2 and FireRedLID.
|
||||
Conformer encoder used by FireRedASR2.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
|
||||
@@ -1,149 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
# Copyright 2024 The vLLM team.
|
||||
# Copyright 2024 Meta Platforms, Inc. and affiliates. All rights reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
"""Llama model for fairseq2 weights."""
|
||||
|
||||
from collections.abc import Iterable
|
||||
|
||||
import torch
|
||||
from torch.nn import Parameter
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.distributed import (
|
||||
get_tensor_model_parallel_rank,
|
||||
get_tensor_model_parallel_world_size,
|
||||
)
|
||||
from vllm.model_executor.layers.linear import set_weight_attrs
|
||||
from vllm.model_executor.models.llama import LlamaForCausalLM
|
||||
|
||||
from .utils import AutoWeightsLoader, WeightsMapper
|
||||
|
||||
|
||||
class Fairseq2LlamaForCausalLM(LlamaForCausalLM):
|
||||
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
|
||||
super().__init__(vllm_config=vllm_config, prefix=prefix)
|
||||
self.tp_rank = get_tensor_model_parallel_rank()
|
||||
self.tp_size = get_tensor_model_parallel_world_size()
|
||||
# For the model loader to read only the relevant checkpoint files
|
||||
self.allow_patterns_overrides = [
|
||||
# either the full checkpoint
|
||||
"model.pt",
|
||||
# or the tp-sharded checkpoint of the current rank
|
||||
f"model.{self.tp_rank}.pt",
|
||||
]
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
# fairseq2's serialization adds a wrapper to usual .pt state_dict's:
|
||||
# { "model_key": my_model_name, "my_model_name": state_dict }
|
||||
# which we first need to unpack
|
||||
weights_wrapped = dict(weights)
|
||||
weights = weights_wrapped[weights_wrapped["model_key"]].items() # type: ignore
|
||||
|
||||
# remap keys
|
||||
fs2_to_vllm_mapper = WeightsMapper(
|
||||
orig_to_new_prefix={
|
||||
"decoder_frontend.embed.": "model.embed_tokens.",
|
||||
"decoder.": "model.",
|
||||
"final_proj.": "lm_head.",
|
||||
},
|
||||
orig_to_new_substr={
|
||||
".self_attn_layer_norm.": ".input_layernorm.",
|
||||
".ffn_layer_norm.": ".post_attention_layernorm.",
|
||||
".self_attn.output_proj.": ".self_attn.o_proj.",
|
||||
".ffn.gate_proj.": ".mlp.gate_proj.",
|
||||
".ffn.inner_proj.": ".mlp.up_proj.",
|
||||
".ffn.output_proj.": ".mlp.down_proj.",
|
||||
".layer_norm.": ".norm.",
|
||||
},
|
||||
)
|
||||
weights = fs2_to_vllm_mapper.apply(weights)
|
||||
|
||||
params = dict(self.named_parameters())
|
||||
|
||||
loader = AutoWeightsLoader(self)
|
||||
return loader.load_weights(
|
||||
self.reshape_fairseq2_weights(name, loaded_weight, params)
|
||||
for name, loaded_weight in weights
|
||||
)
|
||||
|
||||
def flag_sharded_weights(self, params: dict[str, Parameter]):
|
||||
"""Sets the `is_sharded_weight` flag to True for all sharded weights"""
|
||||
for name, param in params.items():
|
||||
modules = name.split(".")
|
||||
if "norm" in name and len(param.size()) < 2:
|
||||
# layer norms are not sharded
|
||||
continue
|
||||
elif any(emb in modules for emb in ["embed_tokens", "lm_head"]):
|
||||
# for now we repeat embedding layers for compatibility
|
||||
continue
|
||||
else:
|
||||
# all other layers are sharded
|
||||
set_weight_attrs(param, {"is_sharded_weight": True})
|
||||
|
||||
def reshape_fairseq2_weights(
|
||||
self,
|
||||
name: str,
|
||||
loaded_weight: torch.Tensor,
|
||||
params: dict[str, Parameter],
|
||||
) -> tuple[str, torch.Tensor]:
|
||||
"""Reshape fairseq2's weights."""
|
||||
|
||||
def permute(w: torch.Tensor, n_heads: int) -> torch.Tensor:
|
||||
attn_in = self.config.head_dim * n_heads
|
||||
# check for a sharded weight on dim 0
|
||||
if attn_in // self.tp_size == w.size()[0]:
|
||||
attn_in //= self.tp_size
|
||||
n_heads //= self.tp_size
|
||||
attn_out = self.config.hidden_size
|
||||
return (
|
||||
w.view(n_heads, attn_in // n_heads // 2, 2, attn_out)
|
||||
.transpose(1, 2)
|
||||
.reshape(attn_in, attn_out)
|
||||
)
|
||||
|
||||
modules = name.split(".")
|
||||
|
||||
# rotary embeds should be sliced
|
||||
if "k_proj" in modules:
|
||||
loaded_weight = permute(loaded_weight, self.config.num_key_value_heads)
|
||||
|
||||
elif "q_proj" in modules:
|
||||
loaded_weight = permute(loaded_weight, self.config.num_attention_heads)
|
||||
|
||||
# We make the loaded weights compatible with both
|
||||
# full checkpoints and tp sharded checkpoints.
|
||||
# Embeddings are repeated to fit the vocab size.
|
||||
# Other weights are flagged for the weight_loader calls.
|
||||
if any(emb in modules for emb in ["embed_tokens", "lm_head"]):
|
||||
# Embeddings are sharded on dim 0
|
||||
dim = 0
|
||||
# In fairseq2, vocab size has to be divisible by tp_size
|
||||
# so we don't worry about padding
|
||||
if self.tp_size > 1 and loaded_weight.shape[dim] < self.config.vocab_size:
|
||||
assert (
|
||||
loaded_weight.shape[dim] * self.tp_size == self.config.vocab_size
|
||||
), "vocab_size should be divisible by tp_size."
|
||||
repeats = [1] * len(loaded_weight.size())
|
||||
repeats[dim] = self.tp_size
|
||||
# repeat to match vocab size and to be easily 'narrow'able
|
||||
loaded_weight = loaded_weight.repeat(repeats)
|
||||
set_weight_attrs(params[name], {"is_sharded_weight": False})
|
||||
# if embeddings are sharded, the rest is too
|
||||
if "embed_tokens" in modules:
|
||||
self.flag_sharded_weights(params)
|
||||
|
||||
return name, loaded_weight
|
||||
@@ -1,794 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
FireRedLID – Language Identification model adapted for vLLM.
|
||||
|
||||
Architecture: ConformerEncoder + TransformerDecoder (6-layer cross-attn)
|
||||
Vocabulary: 120 LID tokens (dict.txt)
|
||||
Output: Up to 2 tokens (e.g. "en", "zh mandarin")
|
||||
|
||||
This implementation follows the Whisper-style encoder-decoder pattern:
|
||||
• Encoder processes audio features (Fbank + CMVN via FeatureExtractor)
|
||||
• Decoder performs single-step autoregressive forward
|
||||
• vLLM's generation loop handles beam search / sampling
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Iterable, Mapping, Sequence
|
||||
from typing import Annotated, Literal
|
||||
|
||||
import torch
|
||||
from torch import nn
|
||||
from transformers import BatchFeature
|
||||
|
||||
from vllm.config import ModelConfig, VllmConfig
|
||||
from vllm.config.multimodal import AudioDummyOptions, BaseDummyOptions
|
||||
from vllm.config.speech_to_text import SpeechToTextConfig, SpeechToTextParams
|
||||
from vllm.distributed import get_tensor_model_parallel_world_size
|
||||
from vllm.inputs import MultiModalDataDict, PromptType
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.attention import Attention, CrossAttention
|
||||
from vllm.model_executor.layers.linear import (
|
||||
ColumnParallelLinear,
|
||||
ReplicatedLinear,
|
||||
RowParallelLinear,
|
||||
)
|
||||
from vllm.model_executor.layers.logits_processor import LogitsProcessor
|
||||
from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
|
||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
from vllm.multimodal.inputs import (
|
||||
MultiModalFieldConfig,
|
||||
MultiModalKwargsItems,
|
||||
)
|
||||
from vllm.multimodal.parse import MultiModalDataItems, MultiModalDataParser
|
||||
from vllm.multimodal.processing import (
|
||||
BaseDummyInputsBuilder,
|
||||
BaseProcessingInfo,
|
||||
EncDecMultiModalProcessor,
|
||||
PromptReplacement,
|
||||
PromptUpdate,
|
||||
)
|
||||
from vllm.transformers_utils.processor import cached_processor_from_config
|
||||
from vllm.utils.tensor_schema import TensorSchema, TensorShape
|
||||
|
||||
from .conformer_encoder import ConformerEncoder
|
||||
from .interfaces import (
|
||||
MultiModalEmbeddings,
|
||||
SupportsMultiModal,
|
||||
SupportsTranscription,
|
||||
)
|
||||
from .utils import (
|
||||
AutoWeightsLoader,
|
||||
WeightsMapper,
|
||||
maybe_prefix,
|
||||
)
|
||||
from .whisper_utils import ISO639_1_SUPPORTED_LANGS
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
class FireRedLIDAudioInputs(TensorSchema):
|
||||
"""
|
||||
Dimensions:
|
||||
- b: Batch size
|
||||
- t: Time frames (variable across utterances)
|
||||
- nmb: Number of mel bins (80)
|
||||
"""
|
||||
|
||||
input_features: Annotated[
|
||||
list[torch.Tensor] | None,
|
||||
TensorShape("b", "t", "nmb", dynamic_dims={"t"}),
|
||||
]
|
||||
speech_lengths: Annotated[
|
||||
list[torch.Tensor] | None,
|
||||
TensorShape("b"),
|
||||
]
|
||||
fake_token_lengths: Annotated[
|
||||
list[torch.Tensor] | None,
|
||||
TensorShape("b"),
|
||||
]
|
||||
|
||||
|
||||
FireRedLIDEncoder = ConformerEncoder
|
||||
|
||||
|
||||
class FireRedLIDPositionalEmbedding(nn.Module):
|
||||
"""Absolute sinusoidal positional embedding indexed by `positions`."""
|
||||
|
||||
def __init__(self, d_model: int, max_len: int = 5000):
|
||||
super().__init__()
|
||||
assert d_model % 2 == 0
|
||||
pe = torch.zeros(max_len, d_model, requires_grad=False)
|
||||
position = torch.arange(0, max_len).unsqueeze(1).float()
|
||||
div_term = torch.exp(
|
||||
torch.arange(0, d_model, 2).float()
|
||||
* -(torch.log(torch.tensor(10000.0)).item() / d_model)
|
||||
)
|
||||
pe[:, 0::2] = torch.sin(position * div_term)
|
||||
pe[:, 1::2] = torch.cos(position * div_term)
|
||||
self.register_buffer("pe", pe, persistent=False)
|
||||
|
||||
def forward(self, position_ids: torch.Tensor) -> torch.Tensor:
|
||||
return self.pe[position_ids]
|
||||
|
||||
|
||||
class FireRedLIDAttention(nn.Module):
|
||||
"""Base attention with shared QKV/FC projections for the LID decoder."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
d_model: int,
|
||||
n_head: int,
|
||||
*,
|
||||
vllm_config: VllmConfig,
|
||||
prefix: str = "",
|
||||
):
|
||||
super().__init__()
|
||||
tp_size = get_tensor_model_parallel_world_size()
|
||||
assert n_head % tp_size == 0
|
||||
self.total_num_heads = n_head
|
||||
self.num_heads = n_head // tp_size
|
||||
self.num_kv_heads = max(1, n_head // tp_size)
|
||||
self.head_dim = d_model // n_head
|
||||
self.scaling = self.head_dim**-0.5
|
||||
|
||||
cache_config = vllm_config.cache_config
|
||||
quant_config = vllm_config.quant_config
|
||||
|
||||
self.w_qs = ColumnParallelLinear(
|
||||
d_model,
|
||||
d_model,
|
||||
bias=True,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.w_qs",
|
||||
)
|
||||
self.w_ks = ColumnParallelLinear(
|
||||
d_model,
|
||||
d_model,
|
||||
bias=False,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.w_ks",
|
||||
)
|
||||
self.w_vs = ColumnParallelLinear(
|
||||
d_model,
|
||||
d_model,
|
||||
bias=True,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.w_vs",
|
||||
)
|
||||
self.fc = RowParallelLinear(
|
||||
d_model,
|
||||
d_model,
|
||||
bias=True,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.fc",
|
||||
)
|
||||
self._init_attn(cache_config, quant_config, prefix)
|
||||
|
||||
def _init_attn(self, cache_config, quant_config, prefix: str) -> None:
|
||||
raise NotImplementedError
|
||||
|
||||
|
||||
class FireRedLIDSelfAttention(FireRedLIDAttention):
|
||||
def _init_attn(self, cache_config, quant_config, prefix: str) -> None:
|
||||
self.attn = Attention(
|
||||
self.num_heads,
|
||||
self.head_dim,
|
||||
self.scaling,
|
||||
num_kv_heads=self.num_kv_heads,
|
||||
cache_config=cache_config,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.attn",
|
||||
)
|
||||
|
||||
def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
|
||||
q, _ = self.w_qs(hidden_states)
|
||||
k, _ = self.w_ks(hidden_states)
|
||||
v, _ = self.w_vs(hidden_states)
|
||||
attn_output = self.attn(q, k, v)
|
||||
output, _ = self.fc(attn_output)
|
||||
return output
|
||||
|
||||
|
||||
class FireRedLIDCrossAttention(FireRedLIDAttention):
|
||||
def _init_attn(self, cache_config, quant_config, prefix: str) -> None:
|
||||
self.attn = CrossAttention(
|
||||
self.num_heads,
|
||||
self.head_dim,
|
||||
self.scaling,
|
||||
num_kv_heads=self.num_kv_heads,
|
||||
cache_config=cache_config,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.attn",
|
||||
)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
encoder_hidden_states: torch.Tensor | None,
|
||||
) -> torch.Tensor:
|
||||
q, _ = self.w_qs(hidden_states)
|
||||
if encoder_hidden_states is not None:
|
||||
k, _ = self.w_ks(encoder_hidden_states)
|
||||
v, _ = self.w_vs(encoder_hidden_states)
|
||||
else:
|
||||
k = v = None
|
||||
|
||||
attn_output = self.attn(q, k, v)
|
||||
output, _ = self.fc(attn_output)
|
||||
return output
|
||||
|
||||
|
||||
class FireRedLIDFFN(nn.Module):
|
||||
def __init__(self, d_model: int, d_ff: int):
|
||||
super().__init__()
|
||||
self.w_1 = ReplicatedLinear(d_model, d_ff, bias=True)
|
||||
self.act = nn.GELU()
|
||||
self.w_2 = ReplicatedLinear(d_ff, d_model, bias=True)
|
||||
|
||||
def forward(self, x: torch.Tensor) -> torch.Tensor:
|
||||
x, _ = self.w_1(x)
|
||||
x = self.act(x)
|
||||
x, _ = self.w_2(x)
|
||||
return x
|
||||
|
||||
|
||||
class FireRedLIDDecoderLayer(nn.Module):
|
||||
"""vLLM-native decoder layer while preserving FireRedLID parameter names."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
d_model: int,
|
||||
n_head: int,
|
||||
*,
|
||||
vllm_config: VllmConfig,
|
||||
prefix: str = "",
|
||||
):
|
||||
super().__init__()
|
||||
self.self_attn_norm = nn.LayerNorm(d_model)
|
||||
self.self_attn = FireRedLIDSelfAttention(
|
||||
d_model,
|
||||
n_head,
|
||||
vllm_config=vllm_config,
|
||||
prefix=f"{prefix}.self_attn",
|
||||
)
|
||||
|
||||
self.cross_attn_norm = nn.LayerNorm(d_model)
|
||||
self.cross_attn = FireRedLIDCrossAttention(
|
||||
d_model,
|
||||
n_head,
|
||||
vllm_config=vllm_config,
|
||||
prefix=f"{prefix}.cross_attn",
|
||||
)
|
||||
|
||||
self.mlp_norm = nn.LayerNorm(d_model)
|
||||
self.mlp = FireRedLIDFFN(d_model, d_model * 4)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
encoder_hidden_states: torch.Tensor | None,
|
||||
) -> torch.Tensor:
|
||||
residual = hidden_states
|
||||
hidden_states = self.self_attn_norm(hidden_states)
|
||||
hidden_states = self.self_attn(hidden_states)
|
||||
hidden_states = residual + hidden_states
|
||||
|
||||
residual = hidden_states
|
||||
hidden_states = self.cross_attn_norm(hidden_states)
|
||||
hidden_states = self.cross_attn(hidden_states, encoder_hidden_states)
|
||||
hidden_states = residual + hidden_states
|
||||
|
||||
residual = hidden_states
|
||||
hidden_states = self.mlp_norm(hidden_states)
|
||||
hidden_states = residual + self.mlp(hidden_states)
|
||||
|
||||
return hidden_states
|
||||
|
||||
|
||||
class FireRedLIDDecoder(nn.Module):
|
||||
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
|
||||
super().__init__()
|
||||
config = vllm_config.model_config.hf_config
|
||||
self.pad_id = getattr(config, "pad_token_id", 2)
|
||||
self.n_layers = getattr(config, "n_layers_lid_dec", 6)
|
||||
self.d_model = getattr(config, "d_model", 1280)
|
||||
self.scale = self.d_model**0.5
|
||||
|
||||
self.tgt_word_emb = nn.Embedding(
|
||||
getattr(config, "vocab_size", 120),
|
||||
self.d_model,
|
||||
padding_idx=self.pad_id,
|
||||
)
|
||||
self.positional_encoding = FireRedLIDPositionalEmbedding(
|
||||
self.d_model,
|
||||
max_len=getattr(config, "pe_maxlen", 5000),
|
||||
)
|
||||
|
||||
self.layer_stack = nn.ModuleList(
|
||||
[
|
||||
FireRedLIDDecoderLayer(
|
||||
self.d_model,
|
||||
getattr(config, "n_head", 20),
|
||||
vllm_config=vllm_config,
|
||||
prefix=f"{prefix}.layer_stack.{idx}",
|
||||
)
|
||||
for idx in range(self.n_layers)
|
||||
]
|
||||
)
|
||||
self.layer_norm_out = nn.LayerNorm(self.d_model)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
input_ids: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
encoder_hidden_states: torch.Tensor | None,
|
||||
) -> torch.Tensor:
|
||||
hidden_states = self.tgt_word_emb(input_ids) * self.scale
|
||||
hidden_states = hidden_states + self.positional_encoding(positions)
|
||||
|
||||
for layer in self.layer_stack:
|
||||
hidden_states = layer(hidden_states, encoder_hidden_states)
|
||||
|
||||
hidden_states = self.layer_norm_out(hidden_states)
|
||||
return hidden_states
|
||||
|
||||
def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
|
||||
return self.tgt_word_emb(input_ids)
|
||||
|
||||
|
||||
class FireRedLIDModel(nn.Module):
|
||||
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
|
||||
super().__init__()
|
||||
config = vllm_config.model_config.hf_config
|
||||
|
||||
self.encoder = FireRedLIDEncoder(
|
||||
idim=getattr(config, "idim", 80),
|
||||
n_layers_enc=getattr(config, "n_layers_enc", 16),
|
||||
n_head=getattr(config, "n_head", 20),
|
||||
d_model=getattr(config, "d_model", 1280),
|
||||
kernel_size=getattr(config, "kernel_size", 33),
|
||||
pe_maxlen=getattr(config, "pe_maxlen", 5000),
|
||||
)
|
||||
|
||||
self.decoder = FireRedLIDDecoder(
|
||||
vllm_config=vllm_config,
|
||||
prefix=maybe_prefix(prefix, "decoder"),
|
||||
)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
input_ids: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
encoder_outputs: list[torch.Tensor] | None = None,
|
||||
) -> torch.Tensor:
|
||||
enc_states = (
|
||||
torch.cat(encoder_outputs, dim=0)
|
||||
if encoder_outputs and len(encoder_outputs) > 0
|
||||
else None
|
||||
)
|
||||
decoder_outputs = self.decoder(
|
||||
input_ids=input_ids,
|
||||
positions=positions,
|
||||
encoder_hidden_states=enc_states,
|
||||
)
|
||||
return decoder_outputs
|
||||
|
||||
def get_encoder_outputs(
|
||||
self,
|
||||
speech: torch.Tensor | list[torch.Tensor],
|
||||
speech_lengths: torch.Tensor | list[torch.Tensor],
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Run the encoder and return padded outputs plus true sequence lengths."""
|
||||
enc_output, enc_lengths, _ = self.encoder(speech, speech_lengths)
|
||||
return enc_output, enc_lengths
|
||||
|
||||
|
||||
class FireRedLIDProcessingInfo(BaseProcessingInfo):
|
||||
def get_hf_config(self):
|
||||
return self.ctx.get_hf_config()
|
||||
|
||||
def get_supported_mm_limits(self) -> Mapping[str, int | None]:
|
||||
return {"audio": 1}
|
||||
|
||||
def get_feature_extractor(self, **kwargs):
|
||||
hf_processor = self.get_hf_processor(**kwargs)
|
||||
feature_extractor = hf_processor.feature_extractor
|
||||
return feature_extractor
|
||||
|
||||
def get_data_parser(self) -> MultiModalDataParser:
|
||||
feature_extractor = self.get_feature_extractor()
|
||||
return MultiModalDataParser(
|
||||
target_sr=feature_extractor.sampling_rate,
|
||||
target_channels=1,
|
||||
)
|
||||
|
||||
@property
|
||||
def skip_prompt_length_check(self) -> bool:
|
||||
return True
|
||||
|
||||
def get_num_audio_tokens(self) -> int:
|
||||
# For encoder profiling – return a reasonable dummy length.
|
||||
# This doesn't affect actual inference since encoder processes
|
||||
# variable-length features.
|
||||
return 1
|
||||
|
||||
|
||||
class FireRedLIDDummyInputsBuilder(BaseDummyInputsBuilder[FireRedLIDProcessingInfo]):
|
||||
def get_dummy_text(self, mm_counts: Mapping[str, int]) -> str:
|
||||
return "<sos>"
|
||||
|
||||
def get_dummy_mm_data(
|
||||
self,
|
||||
seq_len: int,
|
||||
mm_counts: Mapping[str, int],
|
||||
mm_options: Mapping[str, BaseDummyOptions],
|
||||
) -> MultiModalDataDict:
|
||||
feature_extractor = self.info.get_feature_extractor()
|
||||
sampling_rate = feature_extractor.sampling_rate
|
||||
audio_len = feature_extractor.chunk_length * sampling_rate
|
||||
num_audios = mm_counts.get("audio", 0)
|
||||
audio_overrides = mm_options.get("audio")
|
||||
assert audio_overrides is None or isinstance(audio_overrides, AudioDummyOptions)
|
||||
return {
|
||||
"audio": self._get_dummy_audios(
|
||||
length=audio_len,
|
||||
num_audios=num_audios,
|
||||
overrides=audio_overrides,
|
||||
)
|
||||
}
|
||||
|
||||
|
||||
class FireRedLIDMultiModalProcessor(
|
||||
EncDecMultiModalProcessor[FireRedLIDProcessingInfo]
|
||||
):
|
||||
def create_encoder_prompt(
|
||||
self,
|
||||
prompt: list[int],
|
||||
mm_items: MultiModalDataItems,
|
||||
) -> list[int]:
|
||||
# Dummy encoder prompt for profiling (encoder only processes audio).
|
||||
return [0]
|
||||
|
||||
def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:
|
||||
return self.dummy_inputs.get_dummy_text(mm_counts)
|
||||
|
||||
def _preprocess_hf_mm_data(
|
||||
self,
|
||||
mm_data: Mapping[str, object],
|
||||
hf_processor_mm_kwargs: Mapping[str, object],
|
||||
) -> tuple[Mapping[str, object], Mapping[str, object]]:
|
||||
feature_extractor = self.info.get_feature_extractor(**hf_processor_mm_kwargs)
|
||||
|
||||
mm_data = dict(mm_data)
|
||||
mm_data["audio"] = mm_data.pop("audios")
|
||||
|
||||
hf_processor_mm_kwargs = dict(
|
||||
**hf_processor_mm_kwargs,
|
||||
sampling_rate=feature_extractor.sampling_rate,
|
||||
)
|
||||
|
||||
return mm_data, hf_processor_mm_kwargs
|
||||
|
||||
def _postprocess_hf_mm_data(
|
||||
self,
|
||||
mm_data: Mapping[str, object],
|
||||
hf_processor_mm_kwargs: Mapping[str, object],
|
||||
processed_data: BatchFeature,
|
||||
) -> BatchFeature:
|
||||
if "labels" in processed_data:
|
||||
processed_data["input_ids"] = processed_data.pop("labels")
|
||||
|
||||
return processed_data
|
||||
|
||||
def _get_mm_fields_config(
|
||||
self,
|
||||
hf_inputs: BatchFeature,
|
||||
hf_processor_mm_kwargs: Mapping[str, object],
|
||||
) -> Mapping[str, MultiModalFieldConfig]:
|
||||
return dict(
|
||||
input_features=MultiModalFieldConfig.batched("audio"),
|
||||
speech_lengths=MultiModalFieldConfig.batched("audio"),
|
||||
fake_token_lengths=MultiModalFieldConfig.batched("audio", keep_on_cpu=True),
|
||||
)
|
||||
|
||||
def _get_prompt_updates(
|
||||
self,
|
||||
mm_items: MultiModalDataItems,
|
||||
hf_processor_mm_kwargs: Mapping[str, object],
|
||||
out_mm_kwargs: MultiModalKwargsItems,
|
||||
) -> Sequence[PromptUpdate]:
|
||||
out_mm_data = out_mm_kwargs.get_data()
|
||||
fake_token_lengths = out_mm_data.get("fake_token_lengths")
|
||||
|
||||
if fake_token_lengths is None:
|
||||
# Fallback to max encoder output length if not available
|
||||
audio_output_lengths = []
|
||||
else:
|
||||
assert isinstance(fake_token_lengths, torch.Tensor)
|
||||
audio_output_lengths = fake_token_lengths.tolist()
|
||||
|
||||
def get_replacement(item_idx: int):
|
||||
if audio_output_lengths:
|
||||
num_tokens = int(audio_output_lengths[item_idx])
|
||||
else:
|
||||
num_tokens = self.info.get_num_audio_tokens()
|
||||
return [0] * num_tokens
|
||||
|
||||
return [
|
||||
PromptReplacement(
|
||||
modality="audio",
|
||||
target=[0],
|
||||
replacement=get_replacement,
|
||||
)
|
||||
]
|
||||
|
||||
|
||||
# FireRedLID supports a wider set of languages than Whisper's shared list.
|
||||
# Only ISO 639-1 codes are listed; FireRedLID's dialect tokens (mandarin,
|
||||
# xinan, wu, …) are output tokens but not valid language *request* codes.
|
||||
_FIREREDLID_SUPPORTED_LANGUAGES: Mapping[str, str] = {
|
||||
**ISO639_1_SUPPORTED_LANGS,
|
||||
"am": "Amharic",
|
||||
"as": "Assamese",
|
||||
"ba": "Bashkir",
|
||||
"bn": "Bengali",
|
||||
"bo": "Tibetan",
|
||||
"br": "Breton",
|
||||
"eu": "Basque",
|
||||
"fo": "Faroese",
|
||||
"gu": "Gujarati",
|
||||
"ha": "Hausa",
|
||||
"haw": "Hawaiian",
|
||||
"ht": "Haitian Creole",
|
||||
"jw": "Javanese",
|
||||
"ka": "Georgian",
|
||||
"km": "Khmer",
|
||||
"la": "Latin",
|
||||
"lb": "Luxembourgish",
|
||||
"ln": "Lingala",
|
||||
"lo": "Lao",
|
||||
"mg": "Malagasy",
|
||||
"ml": "Malayalam",
|
||||
"mn": "Mongolian",
|
||||
"mt": "Maltese",
|
||||
"my": "Myanmar",
|
||||
"nn": "Nynorsk",
|
||||
"oc": "Occitan",
|
||||
"pa": "Panjabi",
|
||||
"ps": "Pashto",
|
||||
"sa": "Sanskrit",
|
||||
"sd": "Sindhi",
|
||||
"si": "Sinhala",
|
||||
"sn": "Shona",
|
||||
"so": "Somali",
|
||||
"sq": "Albanian",
|
||||
"su": "Sundanese",
|
||||
"te": "Telugu",
|
||||
"tg": "Tajik",
|
||||
"tk": "Turkmen",
|
||||
"tt": "Tatar",
|
||||
"uz": "Uzbek",
|
||||
"yi": "Yiddish",
|
||||
"yo": "Yoruba",
|
||||
"yue": "Cantonese",
|
||||
}
|
||||
|
||||
|
||||
@MULTIMODAL_REGISTRY.register_processor(
|
||||
FireRedLIDMultiModalProcessor,
|
||||
info=FireRedLIDProcessingInfo,
|
||||
dummy_inputs=FireRedLIDDummyInputsBuilder,
|
||||
)
|
||||
class FireRedLIDForConditionalGeneration(
|
||||
nn.Module, SupportsTranscription, SupportsMultiModal
|
||||
):
|
||||
# -- SupportsTranscription protocol attributes --
|
||||
supports_transcription_only = True
|
||||
supported_languages = _FIREREDLID_SUPPORTED_LANGUAGES
|
||||
|
||||
hf_to_vllm_mapper = WeightsMapper(
|
||||
orig_to_new_substr={
|
||||
"encoder.": "model.encoder.",
|
||||
"lid_decoder.": "model.decoder.",
|
||||
# Encoder FFN: nn.Sequential indices → named children
|
||||
"net.0": "pre_layer_norm",
|
||||
"net.1": "linear_expand",
|
||||
"net.4": "linear_project",
|
||||
},
|
||||
orig_to_new_prefix={
|
||||
# Position encoding buffers are rebuilt at init, and the output
|
||||
# projection is tied to the embedding.
|
||||
"model.encoder.positional_encoding.pe": None,
|
||||
"model.decoder.positional_encoding.pe": None,
|
||||
"model.decoder.tgt_word_prj.weight": None,
|
||||
"proj_out.": None,
|
||||
},
|
||||
)
|
||||
|
||||
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
|
||||
super().__init__()
|
||||
config = vllm_config.model_config.hf_config
|
||||
self.config = config
|
||||
self.dtype = vllm_config.model_config.dtype
|
||||
|
||||
with self._mark_composite_model(
|
||||
vllm_config,
|
||||
language_targets=FireRedLIDDecoder,
|
||||
tower_targets={"audio": FireRedLIDEncoder},
|
||||
):
|
||||
self.model = FireRedLIDModel(
|
||||
vllm_config=vllm_config,
|
||||
prefix=maybe_prefix(prefix, "model"),
|
||||
)
|
||||
|
||||
self.proj_out = ParallelLMHead(
|
||||
getattr(config, "vocab_size", 120),
|
||||
getattr(config, "d_model", 1280),
|
||||
quant_config=vllm_config.quant_config,
|
||||
prefix=maybe_prefix(prefix, "proj_out"),
|
||||
)
|
||||
self.proj_out = self.proj_out.tie_weights(self.model.decoder.tgt_word_emb)
|
||||
|
||||
logit_scale = getattr(config, "logit_scale", 1.0)
|
||||
self.logits_processor = LogitsProcessor(
|
||||
getattr(config, "vocab_size", 120),
|
||||
scale=logit_scale,
|
||||
)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
input_ids: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
encoder_outputs: list[torch.Tensor] | None = None,
|
||||
**kwargs,
|
||||
) -> torch.Tensor:
|
||||
if encoder_outputs is None:
|
||||
encoder_outputs = []
|
||||
decoder_outputs = self.model(
|
||||
input_ids=input_ids,
|
||||
positions=positions,
|
||||
encoder_outputs=encoder_outputs,
|
||||
)
|
||||
return decoder_outputs
|
||||
|
||||
def embed_multimodal(self, **kwargs: object) -> MultiModalEmbeddings:
|
||||
"""Run encoder on audio features and return per-item embeddings."""
|
||||
audio_input = self._parse_and_validate_audio_input(**kwargs)
|
||||
|
||||
speech = audio_input["input_features"]
|
||||
speech_lengths = audio_input["speech_lengths"]
|
||||
if speech is None or speech_lengths is None:
|
||||
return []
|
||||
|
||||
# When audio items have different time lengths, vLLM's
|
||||
# MultiModalBatchedField._reduce_data returns a plain
|
||||
# list[Tensor] instead of a stacked Tensor. The encoder
|
||||
# expects a padded [B, Tmax, feat_dim] Tensor, so we
|
||||
# normalise both speech and speech_lengths here.
|
||||
if isinstance(speech, (list, tuple)):
|
||||
# Each element: [Ti, feat_dim] (or [1, Ti, feat_dim])
|
||||
tensors = [
|
||||
s.squeeze(0) if s.dim() == 3 and s.size(0) == 1 else s for s in speech
|
||||
]
|
||||
device = tensors[0].device
|
||||
dtype = tensors[0].dtype
|
||||
feat_dim = tensors[0].shape[-1]
|
||||
lengths = torch.tensor(
|
||||
[t.size(0) for t in tensors],
|
||||
device=device,
|
||||
dtype=torch.int32,
|
||||
)
|
||||
t_max = int(lengths.max().item())
|
||||
# Pre-allocate zero-padded batch tensor
|
||||
speech = torch.zeros(
|
||||
(len(tensors), t_max, feat_dim),
|
||||
device=device,
|
||||
dtype=dtype,
|
||||
)
|
||||
for i, t in enumerate(tensors):
|
||||
speech[i, : t.size(0)] = t
|
||||
speech_lengths = lengths
|
||||
else:
|
||||
# Already a batched Tensor [B, T, feat_dim]
|
||||
if speech.dim() == 2:
|
||||
speech = speech.unsqueeze(0)
|
||||
|
||||
speech_lengths = torch.as_tensor(
|
||||
speech_lengths, dtype=torch.int32, device=speech.device
|
||||
)
|
||||
|
||||
enc_output, enc_lengths = self.model.get_encoder_outputs(
|
||||
speech=speech,
|
||||
speech_lengths=speech_lengths,
|
||||
)
|
||||
|
||||
# vLLM expects one 2D tensor per multimodal item. Slice each batch entry
|
||||
# by the true encoder length so cross-attention never sees padded frames.
|
||||
return tuple(
|
||||
enc_output[i, : max(0, int(enc_lengths[i].item()))]
|
||||
for i in range(enc_output.size(0))
|
||||
)
|
||||
|
||||
def embed_input_ids(
|
||||
self,
|
||||
input_ids: torch.Tensor,
|
||||
multimodal_embeddings: MultiModalEmbeddings | None = None,
|
||||
*,
|
||||
is_multimodal: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
return self.model.decoder.embed_input_ids(input_ids)
|
||||
|
||||
def _parse_and_validate_audio_input(
|
||||
self, **kwargs: object
|
||||
) -> FireRedLIDAudioInputs:
|
||||
input_features = kwargs.pop("input_features", None)
|
||||
speech_lengths = kwargs.pop("speech_lengths", None)
|
||||
fake_token_lengths = kwargs.pop("fake_token_lengths", None)
|
||||
return FireRedLIDAudioInputs(
|
||||
input_features=input_features,
|
||||
speech_lengths=speech_lengths,
|
||||
fake_token_lengths=fake_token_lengths,
|
||||
)
|
||||
|
||||
def compute_logits(self, hidden_states: torch.Tensor) -> torch.Tensor:
|
||||
logits = self.logits_processor(self.proj_out, hidden_states)
|
||||
return logits
|
||||
|
||||
@classmethod
|
||||
def validate_language(cls, language: str | None) -> str | None:
|
||||
# FireRedLID is a language *identification* model – the caller does
|
||||
# not need to specify a language up-front. Accept None silently.
|
||||
if language is None:
|
||||
return None
|
||||
return super().validate_language(language)
|
||||
|
||||
@classmethod
|
||||
def get_generation_prompt(
|
||||
cls,
|
||||
stt_params: SpeechToTextParams,
|
||||
) -> PromptType:
|
||||
"""Build the prompt for the FireRedLID encoder-decoder model.
|
||||
|
||||
The decoder receives a single <sos> token; the encoder processes
|
||||
the raw audio waveform via the multimodal pipeline.
|
||||
"""
|
||||
prompt: PromptType = {
|
||||
"encoder_prompt": {
|
||||
"prompt": "",
|
||||
"multi_modal_data": {
|
||||
"audio": (
|
||||
stt_params.audio,
|
||||
int(stt_params.stt_config.sample_rate),
|
||||
),
|
||||
},
|
||||
},
|
||||
"decoder_prompt": {
|
||||
"prompt": "<sos>",
|
||||
},
|
||||
}
|
||||
return prompt
|
||||
|
||||
@classmethod
|
||||
def get_speech_to_text_config(
|
||||
cls,
|
||||
model_config: ModelConfig,
|
||||
task_type: Literal["transcribe", "translate"],
|
||||
) -> SpeechToTextConfig:
|
||||
processor = cached_processor_from_config(model_config)
|
||||
return SpeechToTextConfig(
|
||||
max_audio_clip_s=processor.feature_extractor.chunk_length,
|
||||
sample_rate=processor.feature_extractor.sampling_rate,
|
||||
# LID output is at most 2 tokens – no chunking needed.
|
||||
min_energy_split_window_size=None,
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def post_process_output(cls, text: str) -> str:
|
||||
# Strip any leading/trailing whitespace from the raw LID output.
|
||||
return text.strip()
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
loader = AutoWeightsLoader(self)
|
||||
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
|
||||
@@ -1,243 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from collections.abc import Set
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
|
||||
from vllm.config import ModelConfig, VllmConfig
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.pooler import (
|
||||
DispatchPooler,
|
||||
PoolingParamsUpdate,
|
||||
)
|
||||
from vllm.model_executor.layers.pooler.activations import PoolerNormalize
|
||||
from vllm.model_executor.layers.pooler.seqwise import (
|
||||
EmbeddingPoolerHead,
|
||||
SequencePooler,
|
||||
SequencePoolingMethod,
|
||||
SequencePoolingMethodOutput,
|
||||
get_seq_pooling_method,
|
||||
)
|
||||
from vllm.model_executor.layers.pooler.tokwise import pooler_for_token_embed
|
||||
from vllm.model_executor.models.llama import LlamaForCausalLM
|
||||
from vllm.tasks import PoolingTask
|
||||
from vllm.tokenizers import cached_tokenizer_from_config
|
||||
from vllm.v1.pool.metadata import PoolingMetadata
|
||||
|
||||
from .interfaces_base import default_pooling_type
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
class GritLMMeanPool(SequencePoolingMethod):
|
||||
"""As `MeanPool`, but only includes non-instruction tokens."""
|
||||
|
||||
def __init__(self, model_config: ModelConfig):
|
||||
super().__init__()
|
||||
|
||||
self.model_config = model_config
|
||||
|
||||
tokenizer = cached_tokenizer_from_config(self.model_config)
|
||||
|
||||
# Collect the tokens needed for pattern matching.
|
||||
# "▁<" is different from "_<". The former uses "▁" to indicate that
|
||||
# the next token is the start of a word.
|
||||
# "<0x0A>" is the newline token (i.e. "\n")."
|
||||
self.token_ids = {
|
||||
tok: tokenizer.convert_tokens_to_ids([tok])[0]
|
||||
for tok in ["<s>", "▁<", "<", "|", "embed", ">", "<0x0A>", "user"]
|
||||
}
|
||||
|
||||
def tokens_to_ids(tokens: list[str]) -> np.ndarray:
|
||||
return np.array([self.token_ids[token] for token in tokens])
|
||||
|
||||
self.user_pattern_ids = tokens_to_ids(["▁<", "|", "user", "|", ">", "<0x0A>"])
|
||||
self.embed_newline_pattern_ids = tokens_to_ids(
|
||||
["<0x0A>", "<", "|", "embed", "|", ">", "<0x0A>"]
|
||||
)
|
||||
self.embed_pattern_ids = tokens_to_ids(["▁<", "|", "embed", "|", ">", "<0x0A>"])
|
||||
|
||||
def _find_array(
|
||||
self,
|
||||
arr: np.ndarray,
|
||||
target: np.ndarray,
|
||||
start_idx: int = 0,
|
||||
end_idx: int | None = None,
|
||||
) -> int:
|
||||
"""
|
||||
Find the first occurrence of `target` in `arr` starting from
|
||||
`start_idx`.
|
||||
|
||||
Args:
|
||||
arr: The array to search within.
|
||||
target: The consecutive subsequence to find.
|
||||
start_idx: The starting index to search from (inclusive).
|
||||
end_idx: The ending index to search from (exclusive).
|
||||
|
||||
Returns:
|
||||
The index of the first occurrence of `target` in `arr`.
|
||||
"""
|
||||
if start_idx < 0:
|
||||
raise ValueError("`start_idx` must be non-negative")
|
||||
if len(arr) == 0 or len(target) == 0:
|
||||
raise ValueError("Empty `arr` or `target` not allowed")
|
||||
|
||||
arr_len = len(arr)
|
||||
target_len = len(target)
|
||||
|
||||
if end_idx is None:
|
||||
end_idx = arr_len
|
||||
|
||||
for i in range(start_idx, min(end_idx, arr_len - target_len + 1)):
|
||||
if (arr[i : i + target_len] == target).all():
|
||||
return i
|
||||
|
||||
return -1
|
||||
|
||||
def _get_instruction_len(self, prompt_token_ids: np.ndarray) -> int:
|
||||
"""
|
||||
Get the length of the instruction in the prompt.
|
||||
|
||||
We do a pattern matching to find the instruction in the prompt,
|
||||
and then return the length of the instruction.
|
||||
|
||||
The pattern matching is done using integers instead of strings
|
||||
because the prompt is given as a list of token IDs.
|
||||
"""
|
||||
instruction_len = 0
|
||||
|
||||
# Return no instruction in case of missing BOS token.
|
||||
if prompt_token_ids[0] != self.token_ids["<s>"]:
|
||||
logger.warning(
|
||||
"BOS token not found in prompt, "
|
||||
"thus using empty string for instruction. "
|
||||
"GritLM requires BOS token in prompt."
|
||||
)
|
||||
return instruction_len
|
||||
|
||||
# If user pattern is found in the prompt, that means there should be
|
||||
# a newline token before the embed pattern.
|
||||
embed_pattern_ids = self.embed_pattern_ids
|
||||
if (
|
||||
self._find_array(
|
||||
prompt_token_ids, self.user_pattern_ids, start_idx=1, end_idx=2
|
||||
)
|
||||
== 1
|
||||
):
|
||||
embed_pattern_ids = self.embed_newline_pattern_ids
|
||||
|
||||
# Find the embed pattern in the prompt.
|
||||
found_embed_pattern_idx = self._find_array(
|
||||
prompt_token_ids, embed_pattern_ids, start_idx=1
|
||||
)
|
||||
|
||||
if found_embed_pattern_idx != -1:
|
||||
instruction_len = found_embed_pattern_idx + len(embed_pattern_ids)
|
||||
else:
|
||||
logger.warning(
|
||||
"Query instruction not found in prompt, "
|
||||
"thus using BOS token as instruction instead. "
|
||||
"GritLM requires query instruction in prompt."
|
||||
)
|
||||
instruction_len = 1
|
||||
|
||||
return instruction_len
|
||||
|
||||
def get_supported_tasks(self) -> Set[PoolingTask]:
|
||||
return {"embed"}
|
||||
|
||||
def get_pooling_updates(self, task: PoolingTask) -> PoolingParamsUpdate:
|
||||
return PoolingParamsUpdate(requires_token_ids=True)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
pooling_metadata: PoolingMetadata,
|
||||
) -> SequencePoolingMethodOutput:
|
||||
prompt_lens = pooling_metadata.prompt_lens
|
||||
prompt_token_ids = pooling_metadata.get_prompt_token_ids_cpu()
|
||||
instr_lens = torch.tensor(
|
||||
[
|
||||
self._get_instruction_len(token_ids.numpy())
|
||||
for token_ids in prompt_token_ids
|
||||
],
|
||||
device="cpu",
|
||||
)
|
||||
|
||||
offset = 0
|
||||
pooled_data = list[torch.Tensor]()
|
||||
for prompt_len, instr_len in zip(prompt_lens, instr_lens):
|
||||
pooled_data.append(
|
||||
hidden_states[offset + instr_len : offset + prompt_len].mean(
|
||||
dim=0, dtype=torch.float32
|
||||
)
|
||||
)
|
||||
offset += prompt_len
|
||||
|
||||
return pooled_data
|
||||
|
||||
|
||||
class GritLMPooler(SequencePooler):
|
||||
def __init__(self, model_config: ModelConfig):
|
||||
pooler_config = model_config.pooler_config
|
||||
assert pooler_config is not None
|
||||
|
||||
super().__init__(
|
||||
pooling=(
|
||||
GritLMMeanPool(model_config)
|
||||
if pooler_config.seq_pooling_type == "MEAN"
|
||||
else get_seq_pooling_method(pooler_config.seq_pooling_type)
|
||||
),
|
||||
head=EmbeddingPoolerHead(
|
||||
head_dtype=model_config.head_dtype,
|
||||
activation=PoolerNormalize(),
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
@default_pooling_type(seq_pooling_type="MEAN")
|
||||
class GritLM(LlamaForCausalLM):
|
||||
"""This class implements the embedding model for parasail-ai/GritLM-7B-vllm.
|
||||
|
||||
The class inherits from LlamaForCausalLM and provides a custom pooling
|
||||
layer.
|
||||
|
||||
The main difference between the pooling layer in GritLM and the one in
|
||||
LlamaForCausalLM is that GritLM ignores the query instruction in the prompt
|
||||
when pooling the hidden states.
|
||||
|
||||
Embedding prompts should be in the following format:
|
||||
- With instruction: "<|user|>\nINSTRUCTION\n<|embed|>\nPROMPT".
|
||||
- Without instruction: "<|embed|>\nPROMPT".
|
||||
|
||||
Generation prompts should be in the following format:
|
||||
- "<|user|>\nPROMPT\n<|assistant|>\n"
|
||||
"""
|
||||
|
||||
is_pooling_model = True
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
vllm_config: VllmConfig,
|
||||
prefix: str = "",
|
||||
**kwargs,
|
||||
) -> None:
|
||||
if vllm_config.model_config.runner_type == "pooling":
|
||||
hf_config = vllm_config.model_config.hf_config
|
||||
hf_config.is_causal = False
|
||||
|
||||
vllm_config.cache_config.sliding_window = None
|
||||
|
||||
hf_config.sliding_window = None
|
||||
|
||||
super().__init__(vllm_config=vllm_config, prefix=prefix, **kwargs)
|
||||
|
||||
pooler_config = vllm_config.model_config.pooler_config
|
||||
if pooler_config is not None:
|
||||
self.pooler = DispatchPooler(
|
||||
{
|
||||
"token_embed": pooler_for_token_embed(pooler_config),
|
||||
"embed": GritLMPooler(vllm_config.model_config),
|
||||
}
|
||||
)
|
||||
File diff suppressed because it is too large
Load Diff
@@ -273,7 +273,7 @@ class LlamaDecoderLayer(nn.Module):
|
||||
# By default, Llama uses causal attention as it is a decoder-only model.
|
||||
# You can override the HF config with `is_causal=False` to enable
|
||||
# bidirectional attention, which is used in some embedding models
|
||||
# (e.g. parasail-ai/GritLM-7B-vllm)
|
||||
# (e.g. nvidia/llama-nemotron-embed-1b-v2)
|
||||
if getattr(config, "is_causal", True):
|
||||
attn_type = AttentionType.DECODER
|
||||
else:
|
||||
|
||||
@@ -1,317 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
# Adapted from https://huggingface.co/mosaicml/mpt-7b/tree/main
|
||||
import math
|
||||
from collections.abc import Iterable
|
||||
from itertools import islice
|
||||
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
from transformers import MptConfig
|
||||
|
||||
from vllm.compilation.decorators import support_torch_compile
|
||||
from vllm.config import CacheConfig, VllmConfig
|
||||
from vllm.distributed import (
|
||||
get_pp_group,
|
||||
get_tensor_model_parallel_rank,
|
||||
get_tensor_model_parallel_world_size,
|
||||
)
|
||||
from vllm.model_executor.layers.activation import get_act_fn
|
||||
from vllm.model_executor.layers.attention import Attention
|
||||
from vllm.model_executor.layers.linear import (
|
||||
ColumnParallelLinear,
|
||||
QKVParallelLinear,
|
||||
RowParallelLinear,
|
||||
)
|
||||
from vllm.model_executor.layers.logits_processor import LogitsProcessor
|
||||
from vllm.model_executor.layers.quantization import QuantizationConfig
|
||||
from vllm.model_executor.layers.vocab_parallel_embedding import VocabParallelEmbedding
|
||||
from vllm.sequence import IntermediateTensors
|
||||
|
||||
from .interfaces import SupportsPP
|
||||
from .utils import (
|
||||
AutoWeightsLoader,
|
||||
make_empty_intermediate_tensors_factory,
|
||||
make_layers,
|
||||
maybe_prefix,
|
||||
)
|
||||
|
||||
|
||||
def _get_alibi_slopes(
|
||||
total_num_heads: int,
|
||||
alibi_bias_max: int,
|
||||
) -> torch.Tensor:
|
||||
next_power_of_2 = 2 ** math.ceil(math.log2(total_num_heads))
|
||||
m = torch.arange(1, next_power_of_2 + 1, dtype=torch.float32)
|
||||
m = m.mul(alibi_bias_max / next_power_of_2)
|
||||
slopes = 1.0 / torch.pow(2, m)
|
||||
if next_power_of_2 != total_num_heads:
|
||||
slopes = torch.concat([slopes[1::2], slopes[::2]])[:total_num_heads]
|
||||
return slopes
|
||||
|
||||
|
||||
class MPTAttention(nn.Module):
|
||||
def __init__(
|
||||
self,
|
||||
config: MptConfig,
|
||||
cache_config: CacheConfig | None = None,
|
||||
quant_config: QuantizationConfig | None = None,
|
||||
prefix: str = "",
|
||||
):
|
||||
super().__init__()
|
||||
self.d_model = config.d_model
|
||||
self.total_num_heads = config.n_heads
|
||||
self.head_dim = self.d_model // self.total_num_heads
|
||||
self.clip_qkv = config.attn_config.clip_qkv
|
||||
self.qk_ln = config.attn_config.qk_ln
|
||||
self.alibi_bias_max = config.attn_config.alibi_bias_max
|
||||
if "kv_n_heads" in config.attn_config:
|
||||
self.total_num_kv_heads = config.attn_config.kv_n_heads
|
||||
else:
|
||||
self.total_num_kv_heads = self.total_num_heads
|
||||
assert not config.attn_config.prefix_lm
|
||||
assert config.attn_config.alibi
|
||||
|
||||
# pylint: disable=invalid-name
|
||||
self.Wqkv = QKVParallelLinear(
|
||||
self.d_model,
|
||||
self.d_model // self.total_num_heads,
|
||||
self.total_num_heads,
|
||||
self.total_num_kv_heads,
|
||||
bias=not config.no_bias,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.Wqkv",
|
||||
)
|
||||
if self.qk_ln:
|
||||
self.q_ln = nn.LayerNorm(self.d_model)
|
||||
self.k_ln = nn.LayerNorm(self.d_model)
|
||||
self.out_proj = RowParallelLinear(
|
||||
self.d_model,
|
||||
self.d_model,
|
||||
bias=not config.no_bias,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.out_proj",
|
||||
)
|
||||
|
||||
tp_world_size = get_tensor_model_parallel_world_size()
|
||||
assert self.total_num_heads % tp_world_size == 0
|
||||
self.num_heads = self.total_num_heads // tp_world_size
|
||||
|
||||
if self.total_num_kv_heads >= tp_world_size:
|
||||
# Number of KV heads is greater than TP size, so we partition
|
||||
# the KV heads across multiple tensor parallel GPUs.
|
||||
assert self.total_num_kv_heads % tp_world_size == 0
|
||||
else:
|
||||
# Number of KV heads is less than TP size, so we replicate
|
||||
# the KV heads across multiple tensor parallel GPUs.
|
||||
assert tp_world_size % self.total_num_kv_heads == 0
|
||||
self.num_kv_heads = max(1, self.total_num_kv_heads // tp_world_size)
|
||||
self.q_size = self.num_heads * self.head_dim
|
||||
self.kv_size = self.num_kv_heads * self.head_dim
|
||||
# Create the alibi slopes and slice them.
|
||||
tp_rank = get_tensor_model_parallel_rank()
|
||||
head_start = tp_rank * self.num_heads
|
||||
head_end = (tp_rank + 1) * self.num_heads
|
||||
alibi_slopes = _get_alibi_slopes(self.total_num_heads, self.alibi_bias_max)
|
||||
alibi_slopes = alibi_slopes[head_start:head_end].tolist()
|
||||
|
||||
self.head_dim = self.d_model // self.total_num_heads
|
||||
scaling = self.head_dim**-0.5
|
||||
self.attn = Attention(
|
||||
self.num_heads,
|
||||
self.head_dim,
|
||||
scaling,
|
||||
alibi_slopes=alibi_slopes,
|
||||
num_kv_heads=self.num_kv_heads,
|
||||
cache_config=cache_config,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.attn",
|
||||
)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
position_ids: torch.Tensor,
|
||||
hidden_states: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
del position_ids # unused.
|
||||
qkv, _ = self.Wqkv(hidden_states)
|
||||
if self.clip_qkv is not None:
|
||||
qkv.clamp_(min=-self.clip_qkv, max=self.clip_qkv)
|
||||
q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1)
|
||||
if self.qk_ln:
|
||||
q = self.q_ln(q)
|
||||
k = self.k_ln(k)
|
||||
attn_output = self.attn(q, k, v)
|
||||
output, _ = self.out_proj(attn_output)
|
||||
return output
|
||||
|
||||
|
||||
class MPTMLP(nn.Module):
|
||||
def __init__(
|
||||
self,
|
||||
config: MptConfig,
|
||||
quant_config: QuantizationConfig | None = None,
|
||||
prefix: str = "",
|
||||
):
|
||||
super().__init__()
|
||||
hidden_size = config.d_model
|
||||
expansion_ratio = config.expansion_ratio
|
||||
intermediate_size = expansion_ratio * hidden_size
|
||||
self.up_proj = ColumnParallelLinear(
|
||||
hidden_size,
|
||||
intermediate_size,
|
||||
bias=not config.no_bias,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.up_proj",
|
||||
)
|
||||
self.act = get_act_fn("gelu")
|
||||
self.down_proj = RowParallelLinear(
|
||||
intermediate_size,
|
||||
hidden_size,
|
||||
bias=not config.no_bias,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.down_proj",
|
||||
)
|
||||
|
||||
def forward(self, x: torch.Tensor) -> torch.Tensor:
|
||||
x, _ = self.up_proj(x)
|
||||
x = self.act(x)
|
||||
x, _ = self.down_proj(x)
|
||||
return x
|
||||
|
||||
|
||||
class MPTBlock(nn.Module):
|
||||
def __init__(
|
||||
self,
|
||||
config: MptConfig,
|
||||
cache_config: CacheConfig | None = None,
|
||||
quant_config: QuantizationConfig | None = None,
|
||||
prefix: str = "",
|
||||
):
|
||||
super().__init__()
|
||||
hidden_size = config.d_model
|
||||
self.norm_1 = nn.LayerNorm(hidden_size)
|
||||
self.attn = MPTAttention(
|
||||
config, cache_config, quant_config, prefix=f"{prefix}.attn"
|
||||
)
|
||||
self.norm_2 = nn.LayerNorm(hidden_size)
|
||||
self.ffn = MPTMLP(config, quant_config, prefix=f"{prefix}.ffn")
|
||||
|
||||
def forward(
|
||||
self,
|
||||
position_ids: torch.Tensor,
|
||||
hidden_states: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
x = self.norm_1(hidden_states)
|
||||
x = self.attn(
|
||||
position_ids=position_ids,
|
||||
hidden_states=x,
|
||||
)
|
||||
hidden_states = hidden_states + x
|
||||
x = self.norm_2(hidden_states)
|
||||
x = self.ffn(x)
|
||||
hidden_states = hidden_states + x
|
||||
return hidden_states
|
||||
|
||||
|
||||
@support_torch_compile
|
||||
class MPTModel(nn.Module):
|
||||
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
|
||||
super().__init__()
|
||||
|
||||
config = vllm_config.model_config.hf_config
|
||||
cache_config = vllm_config.cache_config
|
||||
quant_config = vllm_config.quant_config
|
||||
|
||||
assert config.embedding_fraction == 1.0
|
||||
assert config.norm_type == "low_precision_layernorm"
|
||||
|
||||
self.wte = VocabParallelEmbedding(
|
||||
config.vocab_size,
|
||||
config.d_model,
|
||||
)
|
||||
self.start_layer, self.end_layer, self.blocks = make_layers(
|
||||
config.n_layers,
|
||||
lambda prefix: MPTBlock(config, cache_config, quant_config, prefix=prefix),
|
||||
prefix=f"{prefix}.blocks",
|
||||
)
|
||||
self.norm_f = nn.LayerNorm(config.d_model)
|
||||
if config.no_bias:
|
||||
for module in self.modules():
|
||||
if hasattr(module, "bias") and isinstance(module.bias, nn.Parameter):
|
||||
# Remove the bias term in Linear and LayerNorm.
|
||||
module.register_parameter("bias", None)
|
||||
self.make_empty_intermediate_tensors = make_empty_intermediate_tensors_factory(
|
||||
["hidden_states"], config.d_model
|
||||
)
|
||||
|
||||
def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
|
||||
return self.wte(input_ids)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
input_ids: torch.Tensor | None,
|
||||
position_ids: torch.Tensor,
|
||||
intermediate_tensors: IntermediateTensors | None,
|
||||
inputs_embeds: torch.Tensor | None = None,
|
||||
) -> torch.Tensor | IntermediateTensors:
|
||||
if get_pp_group().is_first_rank:
|
||||
if inputs_embeds is not None:
|
||||
hidden_states = inputs_embeds
|
||||
else:
|
||||
hidden_states = self.embed_input_ids(input_ids)
|
||||
else:
|
||||
assert intermediate_tensors is not None
|
||||
hidden_states = intermediate_tensors["hidden_states"]
|
||||
|
||||
for block in islice(self.blocks, self.start_layer, self.end_layer):
|
||||
hidden_states = block(position_ids, hidden_states)
|
||||
if not get_pp_group().is_last_rank:
|
||||
return IntermediateTensors({"hidden_states": hidden_states})
|
||||
hidden_states = self.norm_f(hidden_states)
|
||||
return hidden_states
|
||||
|
||||
|
||||
class MPTForCausalLM(nn.Module, SupportsPP):
|
||||
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
|
||||
super().__init__()
|
||||
config = vllm_config.model_config.hf_config
|
||||
quant_config = vllm_config.quant_config
|
||||
self.config = config
|
||||
self.quant_config = quant_config
|
||||
|
||||
self.transformer = MPTModel(
|
||||
vllm_config=vllm_config, prefix=maybe_prefix(prefix, "transformer")
|
||||
)
|
||||
self.lm_head = self.transformer.wte
|
||||
self.logits_processor = LogitsProcessor(config.vocab_size)
|
||||
self.make_empty_intermediate_tensors = (
|
||||
self.transformer.make_empty_intermediate_tensors
|
||||
)
|
||||
|
||||
def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
|
||||
return self.transformer.embed_input_ids(input_ids)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
input_ids: torch.Tensor | None,
|
||||
positions: torch.Tensor,
|
||||
intermediate_tensors: IntermediateTensors | None = None,
|
||||
inputs_embeds: torch.Tensor | None = None,
|
||||
) -> torch.Tensor | IntermediateTensors:
|
||||
hidden_states = self.transformer(
|
||||
input_ids, positions, intermediate_tensors, inputs_embeds
|
||||
)
|
||||
return hidden_states
|
||||
|
||||
def compute_logits(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
) -> torch.Tensor | None:
|
||||
logits = self.logits_processor(self.lm_head, hidden_states)
|
||||
return logits
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
loader = AutoWeightsLoader(self)
|
||||
return loader.load_weights(weights)
|
||||
@@ -74,7 +74,6 @@ _TEXT_GENERATION_MODELS = {
|
||||
"AfmoeForCausalLM": ("afmoe", "AfmoeForCausalLM"),
|
||||
"ApertusForCausalLM": ("apertus", "ApertusForCausalLM"),
|
||||
"ArceeForCausalLM": ("arcee", "ArceeForCausalLM"),
|
||||
"ArcticForCausalLM": ("arctic", "ArcticForCausalLM"),
|
||||
"AXK1ForCausalLM": ("AXK1", "AXK1ForCausalLM"),
|
||||
"BailingMoeForCausalLM": ("bailing_moe", "BailingMoeForCausalLM"),
|
||||
"BailingMoeV2ForCausalLM": ("bailing_moe", "BailingMoeV2ForCausalLM"),
|
||||
@@ -102,7 +101,6 @@ _TEXT_GENERATION_MODELS = {
|
||||
"ExaoneForCausalLM": ("exaone", "ExaoneForCausalLM"),
|
||||
"Exaone4ForCausalLM": ("exaone4", "Exaone4ForCausalLM"),
|
||||
"ExaoneMoeForCausalLM": ("exaone_moe", "ExaoneMoeForCausalLM"),
|
||||
"Fairseq2LlamaForCausalLM": ("fairseq2_llama", "Fairseq2LlamaForCausalLM"),
|
||||
"FalconForCausalLM": ("falcon", "FalconForCausalLM"),
|
||||
"FalconMambaForCausalLM": ("mamba", "MambaForCausalLM"),
|
||||
"FalconH1ForCausalLM": ("falcon_h1", "FalconH1ForCausalLM"),
|
||||
@@ -129,12 +127,10 @@ _TEXT_GENERATION_MODELS = {
|
||||
"GraniteMoeSharedForCausalLM": ("granitemoeshared", "GraniteMoeSharedForCausalLM"),
|
||||
"GraniteMoeSWAForCausalLM": ("granitemoeshared", "GraniteMoeSharedForCausalLM"),
|
||||
"GraniteSWAForCausalLM": ("granite", "GraniteForCausalLM"),
|
||||
"GritLM": ("gritlm", "GritLM"),
|
||||
"HrmTextForCausalLM": ("hrm_text", "HrmTextForCausalLM"),
|
||||
"HunYuanMoEV1ForCausalLM": ("hunyuan_v1", "HunYuanMoEV1ForCausalLM"),
|
||||
"HunYuanDenseV1ForCausalLM": ("hunyuan_v1", "HunYuanDenseV1ForCausalLM"),
|
||||
"HYV3ForCausalLM": ("hy_v3", "HYV3ForCausalLM"),
|
||||
"HCXVisionForCausalLM": ("hyperclovax_vision", "HCXVisionForCausalLM"),
|
||||
"HCXVisionV2ForCausalLM": ("hyperclovax_vision_v2", "HCXVisionV2ForCausalLM"),
|
||||
"HyperCLOVAXForCausalLM": ("hyperclovax", "HyperCLOVAXForCausalLM"),
|
||||
"InternLM2ForCausalLM": ("internlm2", "InternLM2ForCausalLM"),
|
||||
@@ -178,9 +174,6 @@ _TEXT_GENERATION_MODELS = {
|
||||
"MistralForCausalLM": ("mistral", "MistralForCausalLM"),
|
||||
"MistralLarge3ForCausalLM": ("mistral_large_3", "MistralLarge3ForCausalLM"),
|
||||
"MixtralForCausalLM": ("mixtral", "MixtralForCausalLM"),
|
||||
# transformers's mpt class has lower case
|
||||
"MptForCausalLM": ("mpt", "MPTForCausalLM"),
|
||||
"MPTForCausalLM": ("mpt", "MPTForCausalLM"),
|
||||
"MiMoForCausalLM": ("mimo", "MiMoForCausalLM"),
|
||||
"MiMoV2FlashForCausalLM": ("mimo_v2", "MiMoV2FlashForCausalLM"),
|
||||
"MiMoV2ForCausalLM": ("mimo_v2", "MiMoV2ForCausalLM"),
|
||||
@@ -207,14 +200,12 @@ _TEXT_GENERATION_MODELS = {
|
||||
"Qwen3MoeForCausalLM": ("qwen3_moe", "Qwen3MoeForCausalLM"),
|
||||
"Qwen3_5ForCausalLM": ("qwen3_5", "Qwen3_5ForCausalLM"),
|
||||
"Qwen3_5MoeForCausalLM": ("qwen3_5", "Qwen3_5MoeForCausalLM"),
|
||||
"RWForCausalLM": ("falcon", "FalconForCausalLM"),
|
||||
"SarvamMoEForCausalLM": ("sarvam", "SarvamMoEForCausalLM"),
|
||||
"SarvamMLAForCausalLM": ("sarvam", "SarvamMLAForCausalLM"),
|
||||
"SeedOssForCausalLM": ("seed_oss", "SeedOssForCausalLM"),
|
||||
"Step1ForCausalLM": ("step1", "Step1ForCausalLM"),
|
||||
"Step3TextForCausalLM": ("step3_text", "Step3TextForCausalLM"),
|
||||
"Step3p5ForCausalLM": ("step3p5", "Step3p5ForCausalLM"),
|
||||
"StableLMEpochForCausalLM": ("stablelm", "StablelmForCausalLM"),
|
||||
"StableLmForCausalLM": ("stablelm", "StablelmForCausalLM"),
|
||||
"SolarForCausalLM": ("solar", "SolarForCausalLM"),
|
||||
"TeleChat2ForCausalLM": ("telechat2", "TeleChat2ForCausalLM"),
|
||||
@@ -234,7 +225,6 @@ _EMBEDDING_MODELS = {
|
||||
"Gemma2Model": ("gemma2", "Gemma2ForCausalLM"),
|
||||
"Gemma3TextModel": ("gemma3", "Gemma3Model"),
|
||||
"GlmForCausalLM": ("glm", "GlmForCausalLM"),
|
||||
"GritLM": ("gritlm", "GritLM"),
|
||||
"GteModel": ("bert_with_rope", "SnowflakeGteNewModel"),
|
||||
"GteNewModel": ("bert_with_rope", "GteNewModel"),
|
||||
"JinaEmbeddingsV5Model": ("jina", "JinaEmbeddingsV5Model"),
|
||||
@@ -274,7 +264,6 @@ _EMBEDDING_MODELS = {
|
||||
# Technically Terratorch models work on images, both in
|
||||
# input and output. I am adding it here because it piggy-backs on embedding
|
||||
# models for the time being.
|
||||
"PrithviGeoSpatialMAE": ("terratorch", "Terratorch"),
|
||||
"Terratorch": ("terratorch", "Terratorch"),
|
||||
}
|
||||
|
||||
@@ -360,12 +349,6 @@ _MULTIMODAL_MODELS = {
|
||||
"BagelForConditionalGeneration": ("bagel", "BagelForConditionalGeneration"),
|
||||
"BeeForConditionalGeneration": ("bee", "BeeForConditionalGeneration"),
|
||||
"Blip2ForConditionalGeneration": ("blip2", "Blip2ForConditionalGeneration"),
|
||||
"ChameleonForConditionalGeneration": (
|
||||
"chameleon",
|
||||
"ChameleonForConditionalGeneration",
|
||||
),
|
||||
"Cheers": ("cheers", "CheersForConditionalGeneration"),
|
||||
"CheersForConditionalGeneration": ("cheers", "CheersForConditionalGeneration"),
|
||||
"Cohere2VisionForConditionalGeneration": (
|
||||
"cohere2_vision",
|
||||
"Cohere2VisionForConditionalGeneration",
|
||||
@@ -401,10 +384,6 @@ _MULTIMODAL_MODELS = {
|
||||
"FireRedASR2ForConditionalGeneration",
|
||||
),
|
||||
"FunASRForConditionalGeneration": ("funasr", "FunASRForConditionalGeneration"),
|
||||
"FireRedLIDForConditionalGeneration": (
|
||||
"fireredlid",
|
||||
"FireRedLIDForConditionalGeneration",
|
||||
),
|
||||
"FunAudioChatForConditionalGeneration": (
|
||||
"funaudiochat",
|
||||
"FunAudioChatForConditionalGeneration",
|
||||
@@ -811,6 +790,20 @@ _PREVIOUSLY_SUPPORTED_MODELS = {
|
||||
"FuyuForCausalLM": "0.25.0",
|
||||
"Plamo2ForCausalLM": "0.26.0",
|
||||
"OuroForCausalLM": "0.26.0",
|
||||
"ArcticForCausalLM": "0.28.0",
|
||||
"ChameleonForConditionalGeneration": "0.28.0",
|
||||
"Cheers": "0.28.0",
|
||||
"CheersForConditionalGeneration": "0.28.0",
|
||||
"Fairseq2LlamaForCausalLM": "0.28.0",
|
||||
"FireRedLIDForConditionalGeneration": "0.28.0",
|
||||
"GritLM": "0.28.0",
|
||||
"HCXVisionForCausalLM": "0.28.0",
|
||||
"MPTForCausalLM": "0.28.0",
|
||||
"MptForCausalLM": "0.28.0",
|
||||
"RWForCausalLM": "0.28.0",
|
||||
"StableLMEpochForCausalLM": "0.28.0",
|
||||
# Superseded by the generic "Terratorch" architecture
|
||||
"PrithviGeoSpatialMAE": "0.28.0",
|
||||
}
|
||||
|
||||
_OOT_SUPPORTED_MODELS = {
|
||||
|
||||
Reference in New Issue
Block a user