[New Model]: support GTE NewModel (#17986)
This commit is contained in:
@@ -32,11 +32,18 @@ class BertWithRopeEmbedding(nn.Module):
|
||||
def __init__(self, config: PretrainedConfig):
|
||||
|
||||
super().__init__()
|
||||
assert config.type_vocab_size > 0
|
||||
if config.position_embedding_type not in ["rope", "rotary"]:
|
||||
raise ValueError("Only 'rotary'('rope') position_embedding_type" +
|
||||
" is supported")
|
||||
|
||||
self.word_embeddings = VocabParallelEmbedding(config.vocab_size,
|
||||
config.hidden_size)
|
||||
self.token_type_embeddings = VocabParallelEmbedding(
|
||||
config.type_vocab_size, config.hidden_size)
|
||||
if config.type_vocab_size > 0:
|
||||
self.token_type_embeddings = VocabParallelEmbedding(
|
||||
config.type_vocab_size, config.hidden_size)
|
||||
else:
|
||||
self.token_type_embeddings = None
|
||||
|
||||
self.LayerNorm = nn.LayerNorm(config.hidden_size,
|
||||
eps=config.layer_norm_eps)
|
||||
|
||||
@@ -47,13 +54,17 @@ class BertWithRopeEmbedding(nn.Module):
|
||||
) -> torch.Tensor:
|
||||
input_shape = input_ids.size()
|
||||
inputs_embeds = self.word_embeddings(input_ids)
|
||||
if token_type_ids is None:
|
||||
token_type_ids = torch.zeros(input_shape,
|
||||
dtype=torch.long,
|
||||
device=inputs_embeds.device)
|
||||
|
||||
token_type_embeddings = self.token_type_embeddings(token_type_ids)
|
||||
embeddings = inputs_embeds + token_type_embeddings
|
||||
embeddings = inputs_embeds
|
||||
if self.token_type_embeddings is not None:
|
||||
if token_type_ids is None:
|
||||
token_type_ids = torch.zeros(input_shape,
|
||||
dtype=torch.long,
|
||||
device=inputs_embeds.device)
|
||||
|
||||
token_type_embeddings = self.token_type_embeddings(token_type_ids)
|
||||
embeddings += token_type_embeddings
|
||||
|
||||
embeddings = self.LayerNorm(embeddings)
|
||||
return embeddings
|
||||
|
||||
@@ -321,7 +332,7 @@ class BertWithRopeBlock(nn.Module):
|
||||
if moe:
|
||||
self.mlp = NomicMoELayer(config=config, )
|
||||
else:
|
||||
if config.hidden_act in ["silu", "gelu_and_mul"]:
|
||||
if config.hidden_act in ["silu", "geglu"]:
|
||||
self.mlp = BertWithRopeGatedMLP(
|
||||
hidden_size=config.hidden_size,
|
||||
intermediate_size=config.intermediate_size,
|
||||
@@ -390,6 +401,7 @@ class BertWithRope(nn.Module, SupportsV0Only, SupportsQuant):
|
||||
|
||||
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
|
||||
super().__init__()
|
||||
self.vllm_config = vllm_config
|
||||
self.config = self.config_verify(vllm_config)
|
||||
self.embeddings = BertWithRopeEmbedding(self.config)
|
||||
self.encoder = BertWithRopeEncoder(
|
||||
@@ -420,7 +432,7 @@ class BertWithRope(nn.Module, SupportsV0Only, SupportsQuant):
|
||||
torch.Tensor]]) -> Set[str]:
|
||||
weights = self.hf_to_vllm_mapper.apply(weights)
|
||||
|
||||
if self.config.hidden_act in ["silu", "gelu_and_mul"]:
|
||||
if self.config.hidden_act in ["silu", "geglu"]:
|
||||
stacked_params_mapping = [
|
||||
# (param_name, shard_name, shard_id)
|
||||
("gate_up_proj", "gate_proj", 0),
|
||||
@@ -458,6 +470,8 @@ class BertWithRope(nn.Module, SupportsV0Only, SupportsQuant):
|
||||
|
||||
|
||||
class NomicBertModel(BertWithRope):
|
||||
# for https://huggingface.co/nomic-ai/nomic-bert-2048
|
||||
|
||||
hf_to_vllm_mapper = WeightsMapper(
|
||||
orig_to_new_substr={
|
||||
"emb_ln": "embeddings.LayerNorm",
|
||||
@@ -475,6 +489,9 @@ class NomicBertModel(BertWithRope):
|
||||
|
||||
assert config.__class__.__name__ == "NomicBertConfig"
|
||||
assert config.activation_function in ["swiglu", "gelu"]
|
||||
config.position_embedding_type = getattr(config,
|
||||
"position_embedding_type",
|
||||
"rope")
|
||||
|
||||
if config.activation_function == "swiglu":
|
||||
config.hidden_act = "silu"
|
||||
@@ -512,10 +529,13 @@ class NomicBertModel(BertWithRope):
|
||||
return config
|
||||
|
||||
|
||||
class GteModel(BertWithRope):
|
||||
class GteNewModel(BertWithRope):
|
||||
# for https://huggingface.co/Alibaba-NLP/new-impl
|
||||
|
||||
hf_to_vllm_mapper = WeightsMapper(
|
||||
orig_to_new_substr={
|
||||
"layer": 'layers',
|
||||
"new.": "",
|
||||
"layer": "layers",
|
||||
"attention.qkv_proj": "attn.qkv_proj",
|
||||
"attention.o_proj": "attn.out_proj",
|
||||
})
|
||||
@@ -523,7 +543,7 @@ class GteModel(BertWithRope):
|
||||
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
|
||||
super().__init__(vllm_config=vllm_config, prefix=prefix)
|
||||
|
||||
# GteModel only gate_up_proj does not have bias.
|
||||
# GteNewModel only gate_up_proj does not have bias.
|
||||
# Hack method learned from vllm/model_executor/models/glm.py
|
||||
for layer in self.encoder.layers:
|
||||
layer.mlp.gate_up_proj.bias = None
|
||||
@@ -532,12 +552,10 @@ class GteModel(BertWithRope):
|
||||
def config_verify(self, vllm_config):
|
||||
config = vllm_config.model_config.hf_config
|
||||
|
||||
assert config.__class__.__name__ == "GteConfig"
|
||||
assert config.position_embedding_type == "rope"
|
||||
assert config.__class__.__name__ == "NewConfig"
|
||||
assert config.hidden_act == "gelu"
|
||||
|
||||
config.position_embedding_type = "rotary"
|
||||
config.hidden_act = "gelu_and_mul"
|
||||
config.hidden_act = "geglu"
|
||||
|
||||
head_dim = config.hidden_size // config.num_attention_heads
|
||||
config.rotary_kwargs = {
|
||||
@@ -559,13 +577,52 @@ class GteModel(BertWithRope):
|
||||
else:
|
||||
yield name, weight
|
||||
|
||||
def ignore_unnecessary_layers(self,
|
||||
weights: Iterable[Tuple[str, torch.Tensor]]):
|
||||
for name, weight in weights:
|
||||
if name.startswith("classifier"):
|
||||
continue
|
||||
yield name, weight
|
||||
|
||||
def load_weights(self, weights: Iterable[Tuple[str,
|
||||
torch.Tensor]]) -> Set[str]:
|
||||
weights = self.ignore_unnecessary_layers(weights)
|
||||
weights = self.split_up_gate_proj(weights)
|
||||
return super().load_weights(weights)
|
||||
|
||||
|
||||
class SnowflakeGteNewModel(GteNewModel):
|
||||
# for Snowflake/snowflake-arctic-embed-m-v2.0
|
||||
|
||||
hf_to_vllm_mapper = WeightsMapper(
|
||||
orig_to_new_substr={
|
||||
"layer": "layers",
|
||||
"attention.qkv_proj": "attn.qkv_proj",
|
||||
"attention.o_proj": "attn.out_proj",
|
||||
})
|
||||
|
||||
def config_verify(self, vllm_config):
|
||||
config = vllm_config.model_config.hf_config
|
||||
|
||||
assert config.__class__.__name__ == "GteConfig"
|
||||
assert config.hidden_act == "gelu"
|
||||
|
||||
config.hidden_act = "geglu"
|
||||
|
||||
head_dim = config.hidden_size // config.num_attention_heads
|
||||
config.rotary_kwargs = {
|
||||
"head_size": head_dim,
|
||||
"rotary_dim": getattr(config, "rotary_emb_dim", head_dim),
|
||||
"max_position": config.max_position_embeddings,
|
||||
"base": config.rope_theta,
|
||||
"rope_scaling": getattr(config, "rope_scaling", None)
|
||||
}
|
||||
return config
|
||||
|
||||
|
||||
class JinaRobertaModel(BertWithRope):
|
||||
# for https://huggingface.co/jinaai/jina-embeddings-v3
|
||||
|
||||
hf_to_vllm_mapper = WeightsMapper(
|
||||
orig_to_new_substr={
|
||||
"emb_ln": "embeddings.LayerNorm",
|
||||
@@ -579,6 +636,9 @@ class JinaRobertaModel(BertWithRope):
|
||||
|
||||
def config_verify(self, vllm_config):
|
||||
config = vllm_config.model_config.hf_config
|
||||
|
||||
assert config.__class__.__name__ == "XLMRobertaFlashConfig"
|
||||
|
||||
head_dim = config.hidden_size // config.num_attention_heads
|
||||
config.rotary_kwargs = {
|
||||
"head_size": head_dim,
|
||||
@@ -611,6 +671,7 @@ class JinaRobertaModel(BertWithRope):
|
||||
# This is a temporary solution until we have a better way to handle
|
||||
|
||||
scaling = self.config.lora_alpha / self.config.lora_rank
|
||||
device = self.vllm_config.device_config.device
|
||||
|
||||
weights = {name: weight for name, weight in weights}
|
||||
|
||||
@@ -628,13 +689,13 @@ class JinaRobertaModel(BertWithRope):
|
||||
weight_name = name[:-len(o)]
|
||||
|
||||
if "embeddings" in weight_name:
|
||||
B = weights[weight_name + a][i].cuda().float()
|
||||
A = weights[weight_name + b][i].cuda().float()
|
||||
B = weights[weight_name + a][i].to(device).float()
|
||||
A = weights[weight_name + b][i].to(device).float()
|
||||
else:
|
||||
B = weights[weight_name + b][i].cuda().float()
|
||||
A = weights[weight_name + a][i].cuda().float()
|
||||
B = weights[weight_name + b][i].to(device).float()
|
||||
A = weights[weight_name + a][i].to(device).float()
|
||||
|
||||
weight = (weights[weight_name + o].cuda() +
|
||||
weight = (weights[weight_name + o].to(device) +
|
||||
torch.matmul(B, A).view(shape) * scaling)
|
||||
weight = weight.cpu().to(dtype)
|
||||
|
||||
|
||||
@@ -230,9 +230,12 @@ class ModernBertModel(nn.Module):
|
||||
def forward(
|
||||
self,
|
||||
input_ids: Optional[torch.LongTensor] = None,
|
||||
positions: Optional[torch.Tensor] = None,
|
||||
intermediate_tensors: Optional[IntermediateTensors] = None,
|
||||
inputs_embeds: Optional[torch.Tensor] = None,
|
||||
position_ids: Optional[torch.LongTensor] = None,
|
||||
) -> torch.Tensor:
|
||||
position_ids = positions if positions is not None else position_ids
|
||||
if inputs_embeds is not None:
|
||||
hidden_states = inputs_embeds
|
||||
else:
|
||||
|
||||
@@ -127,7 +127,8 @@ _EMBEDDING_MODELS = {
|
||||
"Gemma2Model": ("gemma2", "Gemma2ForCausalLM"),
|
||||
"GlmForCausalLM": ("glm", "GlmForCausalLM"),
|
||||
"GritLM": ("gritlm", "GritLM"),
|
||||
"GteModel": ("bert_with_rope", "GteModel"),
|
||||
"GteModel": ("bert_with_rope", "SnowflakeGteNewModel"),
|
||||
"GteNewModel": ("bert_with_rope", "GteNewModel"),
|
||||
"InternLM2ForRewardModel": ("internlm2", "InternLM2ForRewardModel"),
|
||||
"JambaForSequenceClassification": ("jamba", "JambaForSequenceClassification"), # noqa: E501
|
||||
"LlamaModel": ("llama", "LlamaForCausalLM"),
|
||||
@@ -137,6 +138,7 @@ _EMBEDDING_MODELS = {
|
||||
if arch == "LlamaForCausalLM"
|
||||
},
|
||||
"MistralModel": ("llama", "LlamaForCausalLM"),
|
||||
"ModernBertModel": ("modernbert", "ModernBertModel"),
|
||||
"NomicBertModel": ("bert_with_rope", "NomicBertModel"),
|
||||
"Phi3ForCausalLM": ("phi3", "Phi3ForCausalLM"),
|
||||
"Qwen2Model": ("qwen2", "Qwen2EmbeddingModel"),
|
||||
|
||||
Reference in New Issue
Block a user