[Bugfix] Fix Dense module loading for sentence-transformers embedding models (simplified V2) (#23408)
Signed-off-by: FFFfff1FFFfff <yifanli0919@gmail.com>
This commit is contained in:
@@ -422,12 +422,23 @@ _ACTIVATION_REGISTRY = LazyDict({
|
||||
lambda: nn.SiLU(),
|
||||
"quick_gelu":
|
||||
lambda: QuickGELU(),
|
||||
"tanh":
|
||||
lambda: nn.Tanh(),
|
||||
"sigmoid":
|
||||
lambda: nn.Sigmoid(),
|
||||
})
|
||||
|
||||
|
||||
def get_act_fn(act_fn_name: str) -> nn.Module:
|
||||
"""Get an activation function by name."""
|
||||
act_fn_name = act_fn_name.lower()
|
||||
|
||||
if act_fn_name.startswith("torch.nn.modules."):
|
||||
activation_name = act_fn_name.split(".")[-1]
|
||||
if activation_name == "identity":
|
||||
return nn.Identity()
|
||||
act_fn_name = activation_name
|
||||
|
||||
if act_fn_name not in _ACTIVATION_REGISTRY:
|
||||
raise ValueError(
|
||||
f"Activation function {act_fn_name!r} is not supported.")
|
||||
|
||||
@@ -5,7 +5,7 @@ from collections.abc import Mapping, Set
|
||||
from dataclasses import dataclass
|
||||
from enum import IntEnum
|
||||
from itertools import groupby
|
||||
from typing import Callable, Optional, TypeVar, Union
|
||||
from typing import Callable, Optional, TypeVar, Union, cast
|
||||
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
@@ -435,9 +435,31 @@ class EmbeddingPoolerHead(PoolerHead):
|
||||
def __init__(self) -> None:
|
||||
super().__init__(activation=PoolerNormalize())
|
||||
|
||||
# Load ST projector if available
|
||||
from vllm.config import get_current_vllm_config
|
||||
from vllm.model_executor.models.adapters import _load_st_projector
|
||||
|
||||
vllm_config = get_current_vllm_config()
|
||||
self.projector = _load_st_projector(
|
||||
vllm_config.model_config) if vllm_config else None
|
||||
|
||||
def forward(self, pooled_data: Union[list[torch.Tensor], torch.Tensor],
|
||||
pooling_metadata: PoolingMetadata):
|
||||
|
||||
# Apply ST projector
|
||||
if self.projector is not None:
|
||||
projector = cast(nn.Module, self.projector)
|
||||
|
||||
def _proj(x: torch.Tensor) -> torch.Tensor:
|
||||
orig_dtype = x.dtype
|
||||
y = projector(x.to(torch.float32))
|
||||
return y.to(orig_dtype)
|
||||
|
||||
if isinstance(pooled_data, torch.Tensor):
|
||||
pooled_data = _proj(pooled_data)
|
||||
else:
|
||||
pooled_data = [_proj(t) for t in pooled_data]
|
||||
|
||||
pooling_params = get_pooling_params(pooling_metadata)
|
||||
|
||||
if isinstance(pooled_data, list):
|
||||
|
||||
Reference in New Issue
Block a user