[Model] Consolidate pooler implementations (#20927)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
This commit is contained in:
@@ -1,22 +1,21 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from abc import ABC, abstractmethod
|
||||
from dataclasses import dataclass
|
||||
from enum import IntEnum
|
||||
from typing import Optional, Union
|
||||
from typing import Callable, Optional, TypeVar, Union
|
||||
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
import torch.nn.functional as F
|
||||
from typing_extensions import assert_never
|
||||
from transformers import PretrainedConfig
|
||||
|
||||
from vllm.config import ModelConfig, PoolerConfig
|
||||
from vllm.model_executor.pooling_metadata import ( # noqa: E501
|
||||
PoolingMetadata as V0PoolingMetadata)
|
||||
from vllm.model_executor.pooling_metadata import PoolingTensors
|
||||
from vllm.sequence import PoolerOutput, PoolingSequenceGroupOutput
|
||||
from vllm.transformers_utils.config import (
|
||||
get_classification_activation_function,
|
||||
get_cross_encoder_activation_function)
|
||||
from vllm.utils import resolve_obj_by_qualname
|
||||
from vllm.v1.pool.metadata import PoolingMetadata as V1PoolingMetadata
|
||||
|
||||
PoolingMetadata = Union[V0PoolingMetadata, V1PoolingMetadata]
|
||||
@@ -31,140 +30,202 @@ class PoolingType(IntEnum):
|
||||
MEAN = 4
|
||||
|
||||
|
||||
class SimplePooler(nn.Module):
|
||||
"""A layer that pools specific information from hidden states.
|
||||
@dataclass(frozen=True)
|
||||
class ResolvedPoolingConfig:
|
||||
pooling_type: PoolingType
|
||||
|
||||
This layer does the following:
|
||||
1. Extracts specific tokens or aggregates data based on pooling method.
|
||||
2. Normalizes output if specified.
|
||||
3. Returns structured results as `PoolerOutput`.
|
||||
normalize: bool
|
||||
softmax: bool
|
||||
step_tag_id: Optional[int]
|
||||
returned_token_ids: Optional[list[int]]
|
||||
|
||||
Attributes:
|
||||
pooling_type: The type of pooling to use.
|
||||
normalize: Whether to normalize the pooled data.
|
||||
"""
|
||||
|
||||
@staticmethod
|
||||
def from_pooling_type(
|
||||
@classmethod
|
||||
def from_config_with_defaults(
|
||||
cls,
|
||||
pooler_config: PoolerConfig,
|
||||
pooling_type: PoolingType,
|
||||
*,
|
||||
normalize: bool,
|
||||
softmax: bool,
|
||||
step_tag_id: Optional[int] = None,
|
||||
returned_token_ids: Optional[list[int]] = None,
|
||||
) -> "SimplePooler":
|
||||
if pooling_type == PoolingType.LAST:
|
||||
assert step_tag_id is None and returned_token_ids is None
|
||||
return LastPool(normalize=normalize, softmax=softmax)
|
||||
if pooling_type == PoolingType.ALL:
|
||||
assert step_tag_id is None and returned_token_ids is None
|
||||
return AllPool(normalize=normalize, softmax=softmax)
|
||||
if pooling_type == PoolingType.CLS:
|
||||
assert step_tag_id is None and returned_token_ids is None
|
||||
return CLSPool(normalize=normalize, softmax=softmax)
|
||||
if pooling_type == PoolingType.MEAN:
|
||||
assert step_tag_id is None and returned_token_ids is None
|
||||
return MeanPool(normalize=normalize, softmax=softmax)
|
||||
if pooling_type == PoolingType.STEP:
|
||||
return StepPool(normalize=normalize,
|
||||
softmax=softmax,
|
||||
step_tag_id=step_tag_id,
|
||||
returned_token_ids=returned_token_ids)
|
||||
) -> "ResolvedPoolingConfig":
|
||||
return cls(
|
||||
pooling_type=PoolingType[pooler_config.pooling_type]
|
||||
if pooler_config.pooling_type is not None else pooling_type,
|
||||
normalize=pooler_config.normalize
|
||||
if pooler_config.normalize is not None else normalize,
|
||||
softmax=pooler_config.softmax
|
||||
if pooler_config.softmax is not None else softmax,
|
||||
step_tag_id=pooler_config.step_tag_id
|
||||
if pooler_config.step_tag_id is not None else step_tag_id,
|
||||
returned_token_ids=pooler_config.returned_token_ids
|
||||
if pooler_config.returned_token_ids is not None else
|
||||
returned_token_ids,
|
||||
)
|
||||
|
||||
assert_never(pooling_type)
|
||||
|
||||
def __init__(self, *, normalize: bool, softmax: bool) -> None:
|
||||
super().__init__()
|
||||
def get_prompt_lens(
|
||||
hidden_states: Union[torch.Tensor, list[torch.Tensor]],
|
||||
pooling_metadata: PoolingMetadata,
|
||||
) -> torch.Tensor:
|
||||
if isinstance(pooling_metadata, V1PoolingMetadata):
|
||||
return pooling_metadata.prompt_lens
|
||||
|
||||
self.head = PoolerHead(normalize=normalize, softmax=softmax)
|
||||
assert isinstance(hidden_states, torch.Tensor)
|
||||
return PoolingTensors.from_pooling_metadata(
|
||||
pooling_metadata, hidden_states.device).prompt_lens
|
||||
|
||||
def get_prompt_lens(
|
||||
self,
|
||||
hidden_states: Union[torch.Tensor, list[torch.Tensor]],
|
||||
pooling_metadata: PoolingMetadata,
|
||||
) -> torch.Tensor:
|
||||
if isinstance(pooling_metadata, V1PoolingMetadata):
|
||||
return pooling_metadata.prompt_lens
|
||||
assert isinstance(hidden_states, torch.Tensor)
|
||||
return PoolingTensors.from_pooling_metadata(
|
||||
pooling_metadata, hidden_states.device).prompt_lens
|
||||
|
||||
def extract_states(
|
||||
self,
|
||||
hidden_states: Union[torch.Tensor, list[torch.Tensor]],
|
||||
pooling_metadata: PoolingMetadata,
|
||||
) -> Union[list[torch.Tensor], torch.Tensor]:
|
||||
raise NotImplementedError
|
||||
def get_classification_activation_function(config: PretrainedConfig):
|
||||
return PoolerClassify()
|
||||
|
||||
def build_output(self, data: torch.Tensor) -> PoolingSequenceGroupOutput:
|
||||
return PoolingSequenceGroupOutput(data)
|
||||
|
||||
def get_cross_encoder_activation_function(config: PretrainedConfig):
|
||||
function_name: Optional[str] = None
|
||||
if (hasattr(config, "sentence_transformers")
|
||||
and "activation_fn" in config.sentence_transformers):
|
||||
function_name = config.sentence_transformers["activation_fn"]
|
||||
elif (hasattr(config, "sbert_ce_default_activation_function")
|
||||
and config.sbert_ce_default_activation_function is not None):
|
||||
function_name = config.sbert_ce_default_activation_function
|
||||
|
||||
if function_name is not None:
|
||||
assert function_name.startswith("torch.nn.modules."), (
|
||||
"Loading of activation functions is restricted to "
|
||||
"torch.nn.modules for security reasons")
|
||||
fn = resolve_obj_by_qualname(function_name)()
|
||||
return PoolerActivation.wraps(fn)
|
||||
|
||||
return PoolerScore()
|
||||
|
||||
|
||||
def build_output(all_data: torch.Tensor) -> PoolerOutput:
|
||||
all_outputs = [PoolingSequenceGroupOutput(data) for data in all_data]
|
||||
return PoolerOutput(outputs=all_outputs)
|
||||
|
||||
|
||||
class BasePooler(nn.Module):
|
||||
|
||||
@abstractmethod
|
||||
def forward(
|
||||
self,
|
||||
hidden_states: Union[torch.Tensor, list[torch.Tensor]],
|
||||
pooling_metadata: PoolingMetadata,
|
||||
) -> PoolerOutput:
|
||||
pooled_data = self.extract_states(hidden_states, pooling_metadata)
|
||||
pooled_data = self.head(pooled_data, pooling_metadata)
|
||||
pooled_outputs = [self.build_output(data) for data in pooled_data]
|
||||
return PoolerOutput(outputs=pooled_outputs)
|
||||
raise NotImplementedError
|
||||
|
||||
|
||||
class CLSPool(SimplePooler):
|
||||
class PoolingMethod(nn.Module, ABC):
|
||||
|
||||
def extract_states(
|
||||
@staticmethod
|
||||
def from_pooling_type(pooling_type: PoolingType) -> "PoolingMethod":
|
||||
if pooling_type == PoolingType.LAST:
|
||||
return LastPool()
|
||||
if pooling_type == PoolingType.ALL:
|
||||
return AllPool()
|
||||
if pooling_type == PoolingType.CLS:
|
||||
return CLSPool()
|
||||
if pooling_type == PoolingType.MEAN:
|
||||
return MeanPool()
|
||||
|
||||
raise NotImplementedError(f"Unsupported method: {pooling_type}")
|
||||
|
||||
@abstractmethod
|
||||
def forward_one(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
prompt_len: Optional[torch.Tensor] = None,
|
||||
) -> torch.Tensor:
|
||||
"""
|
||||
Note:
|
||||
`prompt_len=None` means `prompt_len=len(hidden_states)`.
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
def forward_all(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
prompt_lens: torch.Tensor,
|
||||
) -> Union[list[torch.Tensor], torch.Tensor]:
|
||||
raise NotImplementedError
|
||||
|
||||
def forward(
|
||||
self,
|
||||
hidden_states: Union[torch.Tensor, list[torch.Tensor]],
|
||||
pooling_metadata: PoolingMetadata,
|
||||
) -> Union[list[torch.Tensor], torch.Tensor]:
|
||||
prompt_lens = self.get_prompt_lens(hidden_states, pooling_metadata)
|
||||
prompt_lens = get_prompt_lens(hidden_states, pooling_metadata)
|
||||
|
||||
if isinstance(hidden_states, list):
|
||||
result = []
|
||||
for req_state, prompt_len in zip(hidden_states, prompt_lens):
|
||||
assert prompt_len == req_state.shape[0], \
|
||||
"partial prefill not supported with CLS pooling"
|
||||
result.append(req_state[0])
|
||||
return result
|
||||
return [
|
||||
self.forward_one(h, prompt_len)
|
||||
for h, prompt_len in zip(hidden_states, prompt_lens)
|
||||
]
|
||||
|
||||
return self.forward_all(hidden_states, prompt_lens)
|
||||
|
||||
|
||||
class CLSPool(PoolingMethod):
|
||||
|
||||
def forward_one(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
prompt_len: Optional[torch.Tensor] = None,
|
||||
) -> torch.Tensor:
|
||||
assert prompt_len is None or prompt_len == hidden_states.shape[0], \
|
||||
"partial prefill not supported with CLS pooling"
|
||||
|
||||
return hidden_states[0]
|
||||
|
||||
def forward_all(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
prompt_lens: torch.Tensor,
|
||||
) -> Union[list[torch.Tensor], torch.Tensor]:
|
||||
first_token_flat_indices = torch.zeros_like(prompt_lens)
|
||||
first_token_flat_indices[1:] += torch.cumsum(prompt_lens, dim=0)[:-1]
|
||||
return hidden_states[first_token_flat_indices]
|
||||
|
||||
|
||||
class LastPool(SimplePooler):
|
||||
class LastPool(PoolingMethod):
|
||||
|
||||
def extract_states(
|
||||
def forward_one(
|
||||
self,
|
||||
hidden_states: Union[torch.Tensor, list[torch.Tensor]],
|
||||
pooling_metadata: PoolingMetadata,
|
||||
hidden_states: torch.Tensor,
|
||||
prompt_len: Optional[torch.Tensor] = None,
|
||||
) -> torch.Tensor:
|
||||
return hidden_states[-1]
|
||||
|
||||
def forward_all(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
prompt_lens: torch.Tensor,
|
||||
) -> Union[list[torch.Tensor], torch.Tensor]:
|
||||
if isinstance(hidden_states, list):
|
||||
return [h[-1] for h in hidden_states]
|
||||
|
||||
prompt_lens = self.get_prompt_lens(hidden_states, pooling_metadata)
|
||||
|
||||
last_token_flat_indices = torch.cumsum(prompt_lens, dim=0) - 1
|
||||
return hidden_states[last_token_flat_indices]
|
||||
|
||||
|
||||
class AllPool(SimplePooler):
|
||||
class AllPool(PoolingMethod):
|
||||
|
||||
def extract_states(
|
||||
def forward_one(
|
||||
self,
|
||||
hidden_states: Union[torch.Tensor, list[torch.Tensor]],
|
||||
pooling_metadata: PoolingMetadata,
|
||||
hidden_states: torch.Tensor,
|
||||
prompt_len: Optional[torch.Tensor] = None,
|
||||
) -> torch.Tensor:
|
||||
assert prompt_len is None or prompt_len == hidden_states.shape[0], \
|
||||
"partial prefill not supported with ALL pooling"
|
||||
|
||||
return hidden_states
|
||||
|
||||
def forward_all(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
prompt_lens: torch.Tensor,
|
||||
) -> Union[list[torch.Tensor], torch.Tensor]:
|
||||
prompt_lens = self.get_prompt_lens(hidden_states, pooling_metadata)
|
||||
|
||||
if isinstance(hidden_states, list):
|
||||
for req_state, prompt_len in zip(hidden_states, prompt_lens):
|
||||
assert prompt_len == req_state.shape[0], \
|
||||
"partial prefill not supported with ALL pooling"
|
||||
return hidden_states
|
||||
|
||||
offset = 0
|
||||
pooled_data = list[torch.Tensor]()
|
||||
|
||||
for prompt_len in prompt_lens:
|
||||
pooled_data.append(hidden_states[offset:offset + prompt_len])
|
||||
offset += prompt_len
|
||||
@@ -172,24 +233,23 @@ class AllPool(SimplePooler):
|
||||
return pooled_data
|
||||
|
||||
|
||||
class MeanPool(SimplePooler):
|
||||
class MeanPool(PoolingMethod):
|
||||
|
||||
def extract_states(
|
||||
def forward_one(
|
||||
self,
|
||||
hidden_states: Union[torch.Tensor, list[torch.Tensor]],
|
||||
pooling_metadata: PoolingMetadata,
|
||||
hidden_states: torch.Tensor,
|
||||
prompt_len: Optional[torch.Tensor] = None,
|
||||
) -> torch.Tensor:
|
||||
assert prompt_len is None or prompt_len == hidden_states.shape[0], \
|
||||
"partial prefill not supported with MEAN pooling"
|
||||
|
||||
return hidden_states.mean(dim=0, dtype=torch.float32)
|
||||
|
||||
def forward_all(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
prompt_lens: torch.Tensor,
|
||||
) -> Union[list[torch.Tensor], torch.Tensor]:
|
||||
prompt_lens = self.get_prompt_lens(hidden_states, pooling_metadata)
|
||||
|
||||
if isinstance(hidden_states, list):
|
||||
result = []
|
||||
for req_state, prompt_len in zip(hidden_states, prompt_lens):
|
||||
assert prompt_len == req_state.shape[0], \
|
||||
"partial prefill not supported with mean pooling"
|
||||
result.append(torch.mean(req_state, dim=0,
|
||||
dtype=torch.float32))
|
||||
return result
|
||||
|
||||
# Use float32 for torch.cumsum in MeanPool,
|
||||
# otherwise precision will be lost significantly.
|
||||
cumsum = torch.cumsum(hidden_states, dim=0, dtype=torch.float32)
|
||||
@@ -203,78 +263,127 @@ class MeanPool(SimplePooler):
|
||||
hidden_states[start_indices]) / prompt_lens.unsqueeze(1)
|
||||
|
||||
|
||||
class StepPool(SimplePooler):
|
||||
_T = TypeVar("_T", torch.Tensor, list[torch.Tensor])
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
normalize: bool,
|
||||
softmax: bool,
|
||||
step_tag_id: Optional[int] = None,
|
||||
returned_token_ids: Optional[list[int]] = None,
|
||||
):
|
||||
super().__init__(normalize=normalize, softmax=softmax)
|
||||
|
||||
self.step_tag_id = step_tag_id
|
||||
self.returned_token_ids = returned_token_ids
|
||||
class BasePoolerActivation(nn.Module, ABC):
|
||||
|
||||
def get_prompt_token_ids(
|
||||
self,
|
||||
pooling_metadata: PoolingMetadata,
|
||||
) -> list[torch.Tensor]:
|
||||
if isinstance(pooling_metadata, V1PoolingMetadata):
|
||||
return [
|
||||
pooling_metadata.prompt_token_ids[i, :num]
|
||||
for i, num in enumerate(pooling_metadata.prompt_lens)
|
||||
]
|
||||
return [
|
||||
torch.tensor(seq_data_i.prompt_token_ids)
|
||||
for seq_data_i in pooling_metadata.seq_data.values()
|
||||
]
|
||||
@abstractmethod
|
||||
def forward(self, pooled_data: _T) -> _T:
|
||||
# shape:
|
||||
# classify (& score) -> (batch_size, num_classes)
|
||||
# embed -> (batch_size, embedding_dim) or list(embedding_dim)
|
||||
# (batch_size, dimensions) or list(dimensions) if using MRL
|
||||
raise NotImplementedError
|
||||
|
||||
def extract_states(
|
||||
self,
|
||||
hidden_states: Union[torch.Tensor, list[torch.Tensor]],
|
||||
pooling_metadata: PoolingMetadata,
|
||||
) -> Union[list[torch.Tensor], torch.Tensor]:
|
||||
prompt_lens = self.get_prompt_lens(hidden_states, pooling_metadata)
|
||||
prompt_token_ids = self.get_prompt_token_ids(pooling_metadata)
|
||||
|
||||
pooled_data_lst = list[torch.Tensor]()
|
||||
if isinstance(hidden_states, list):
|
||||
for req_state, prompt_len in zip(hidden_states, prompt_lens):
|
||||
assert prompt_len == req_state.shape[0], \
|
||||
"partial prefill not supported with step pooling"
|
||||
pooled_data_lst = hidden_states
|
||||
else:
|
||||
offset = 0
|
||||
for prompt_len in prompt_lens:
|
||||
pooled_data_i = hidden_states[offset:offset + prompt_len]
|
||||
offset += prompt_len
|
||||
pooled_data_lst.append(pooled_data_i)
|
||||
class PoolerActivation(BasePoolerActivation):
|
||||
|
||||
pooled_data = list[torch.Tensor]()
|
||||
returned_token_ids = self.returned_token_ids
|
||||
step_tag_id = self.step_tag_id
|
||||
@staticmethod
|
||||
def wraps(module: nn.Module):
|
||||
if isinstance(module, nn.Identity):
|
||||
return PoolerIdentity()
|
||||
if isinstance(module, (nn.Sigmoid, nn.Softmax)):
|
||||
return PoolerClassify()
|
||||
|
||||
for data, token_id in zip(pooled_data_lst, prompt_token_ids):
|
||||
if returned_token_ids is not None and len(returned_token_ids) > 0:
|
||||
data = data[:, returned_token_ids]
|
||||
return LambdaPoolerActivation(module)
|
||||
|
||||
if step_tag_id is not None:
|
||||
data = data[token_id == step_tag_id]
|
||||
pooled_data.append(data)
|
||||
@abstractmethod
|
||||
def forward_chunk(self, pooled_data: torch.Tensor) -> torch.Tensor:
|
||||
raise NotImplementedError
|
||||
|
||||
def forward(self, pooled_data: _T) -> _T:
|
||||
if isinstance(pooled_data, list):
|
||||
return [self.forward_chunk(data) for data in pooled_data]
|
||||
|
||||
return self.forward_chunk(pooled_data)
|
||||
|
||||
|
||||
class PoolerIdentity(PoolerActivation):
|
||||
|
||||
def forward_chunk(self, pooled_data: torch.Tensor) -> torch.Tensor:
|
||||
return pooled_data
|
||||
|
||||
|
||||
class PoolerNormalize(PoolerActivation):
|
||||
|
||||
def forward_chunk(self, pooled_data: torch.Tensor) -> torch.Tensor:
|
||||
x = F.normalize(pooled_data.float(), p=2, dim=-1)
|
||||
return x.to(pooled_data.dtype)
|
||||
|
||||
|
||||
class PoolerClassify(PoolerActivation):
|
||||
|
||||
def forward_chunk(self, pooled_data: torch.Tensor) -> torch.Tensor:
|
||||
num_labels = pooled_data.shape[-1]
|
||||
if num_labels < 2:
|
||||
return F.sigmoid(pooled_data.float()).to(pooled_data.dtype)
|
||||
|
||||
return F.softmax(pooled_data.float(), dim=-1).to(pooled_data.dtype)
|
||||
|
||||
|
||||
class PoolerScore(PoolerActivation):
|
||||
|
||||
def forward_chunk(self, pooled_data: torch.Tensor) -> torch.Tensor:
|
||||
num_labels = pooled_data.shape[-1]
|
||||
if num_labels < 2:
|
||||
return F.sigmoid(pooled_data.float()).to(pooled_data.dtype)
|
||||
|
||||
return pooled_data
|
||||
|
||||
|
||||
class PoolerHead(nn.Module):
|
||||
class LambdaPoolerActivation(PoolerActivation):
|
||||
|
||||
def __init__(self, *, normalize: bool, softmax: bool) -> None:
|
||||
def __init__(self, fn: Callable[[torch.Tensor], torch.Tensor]):
|
||||
super().__init__()
|
||||
|
||||
self.normalize = normalize
|
||||
self.softmax = softmax
|
||||
self.fn = fn
|
||||
|
||||
def forward_chunk(self, pooled_data: torch.Tensor) -> torch.Tensor:
|
||||
return self.fn(pooled_data)
|
||||
|
||||
|
||||
class PoolerHead(nn.Module):
|
||||
|
||||
@classmethod
|
||||
def from_config_with_defaults(
|
||||
cls,
|
||||
pooler_config: PoolerConfig,
|
||||
pooling_type: PoolingType,
|
||||
normalize: bool,
|
||||
softmax: bool,
|
||||
) -> "PoolerHead":
|
||||
resolved_config = ResolvedPoolingConfig.from_config_with_defaults(
|
||||
pooler_config=pooler_config,
|
||||
pooling_type=pooling_type,
|
||||
normalize=normalize,
|
||||
softmax=softmax,
|
||||
step_tag_id=None,
|
||||
returned_token_ids=None,
|
||||
)
|
||||
|
||||
return cls.from_config(resolved_config)
|
||||
|
||||
@classmethod
|
||||
def from_config(cls, pooler_config: ResolvedPoolingConfig) -> "PoolerHead":
|
||||
if pooler_config.normalize and pooler_config.softmax:
|
||||
raise ValueError("`normalize=True` and `softmax=True` should not "
|
||||
"be set together")
|
||||
|
||||
activation: PoolerActivation
|
||||
if pooler_config.normalize:
|
||||
activation = PoolerNormalize()
|
||||
elif pooler_config.softmax:
|
||||
activation = PoolerClassify()
|
||||
else:
|
||||
activation = PoolerIdentity()
|
||||
|
||||
return cls(activation)
|
||||
|
||||
def __init__(self, activation: PoolerActivation) -> None:
|
||||
super().__init__()
|
||||
|
||||
self.activation = activation
|
||||
|
||||
def forward(self, pooled_data: Union[list[torch.Tensor], torch.Tensor],
|
||||
pooling_metadata: PoolingMetadata):
|
||||
@@ -312,35 +421,21 @@ class PoolerHead(nn.Module):
|
||||
for vecs, d in zip(pooled_data, dimensions_list)
|
||||
]
|
||||
|
||||
if self.normalize:
|
||||
if isinstance(pooled_data, list):
|
||||
pooled_data = [
|
||||
F.normalize(data, p=2, dim=-1) for data in pooled_data
|
||||
]
|
||||
else:
|
||||
pooled_data = F.normalize(pooled_data, p=2, dim=-1)
|
||||
|
||||
if self.softmax:
|
||||
if isinstance(pooled_data, list):
|
||||
pooled_data = [
|
||||
F.softmax(data, dim=-1)
|
||||
if data.shape[-1] >= 2 else F.sigmoid(data)
|
||||
for data in pooled_data
|
||||
]
|
||||
else:
|
||||
if pooled_data.shape[-1] >= 2:
|
||||
pooled_data = F.softmax(pooled_data, dim=-1)
|
||||
else:
|
||||
pooled_data = F.sigmoid(pooled_data)
|
||||
|
||||
# shape:
|
||||
# classify (& score) -> (batch_size, num_classes)
|
||||
# embed -> (batch_size, embedding_dim) or list(embedding_dim)
|
||||
# (batch_size, dimensions) or list(dimensions) if using MRL
|
||||
return pooled_data
|
||||
return self.activation(pooled_data)
|
||||
|
||||
|
||||
class Pooler(nn.Module):
|
||||
class SimplePooler(BasePooler):
|
||||
"""A layer that pools specific information from hidden states.
|
||||
|
||||
This layer does the following:
|
||||
1. Extracts specific tokens or aggregates data based on pooling method.
|
||||
2. Normalizes output if specified.
|
||||
3. Returns structured results as `PoolerOutput`.
|
||||
|
||||
Attributes:
|
||||
pooling_type: The type of pooling to use.
|
||||
normalize: Whether to normalize the pooled data.
|
||||
"""
|
||||
|
||||
@classmethod
|
||||
def from_config_with_defaults(
|
||||
@@ -349,23 +444,146 @@ class Pooler(nn.Module):
|
||||
pooling_type: PoolingType,
|
||||
normalize: bool,
|
||||
softmax: bool,
|
||||
) -> "SimplePooler":
|
||||
resolved_config = ResolvedPoolingConfig.from_config_with_defaults(
|
||||
pooler_config=pooler_config,
|
||||
pooling_type=pooling_type,
|
||||
normalize=normalize,
|
||||
softmax=softmax,
|
||||
)
|
||||
assert resolved_config.pooling_type != PoolingType.STEP
|
||||
|
||||
return cls.from_config(resolved_config)
|
||||
|
||||
@classmethod
|
||||
def from_config(
|
||||
cls,
|
||||
pooler_config: ResolvedPoolingConfig,
|
||||
) -> "SimplePooler":
|
||||
pooling = PoolingMethod.from_pooling_type(pooler_config.pooling_type)
|
||||
head = PoolerHead.from_config(pooler_config)
|
||||
|
||||
return cls(pooling, head)
|
||||
|
||||
def __init__(self, pooling: PoolingMethod, head: PoolerHead) -> None:
|
||||
super().__init__()
|
||||
|
||||
self.pooling = pooling
|
||||
self.head = head
|
||||
|
||||
def forward(
|
||||
self,
|
||||
hidden_states: Union[torch.Tensor, list[torch.Tensor]],
|
||||
pooling_metadata: PoolingMetadata,
|
||||
) -> PoolerOutput:
|
||||
pooled_data = self.pooling(hidden_states, pooling_metadata)
|
||||
pooled_data = self.head(pooled_data, pooling_metadata)
|
||||
return build_output(pooled_data)
|
||||
|
||||
|
||||
class StepPooler(BasePooler):
|
||||
|
||||
@classmethod
|
||||
def from_config(cls, pooler_config: ResolvedPoolingConfig) -> "StepPooler":
|
||||
assert pooler_config.pooling_type == PoolingType.STEP
|
||||
|
||||
return cls(
|
||||
PoolerHead.from_config(pooler_config),
|
||||
step_tag_id=pooler_config.step_tag_id,
|
||||
returned_token_ids=pooler_config.returned_token_ids,
|
||||
)
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
head: PoolerHead,
|
||||
*,
|
||||
step_tag_id: Optional[int] = None,
|
||||
returned_token_ids: Optional[list[int]] = None,
|
||||
) -> SimplePooler:
|
||||
return SimplePooler.from_pooling_type(
|
||||
pooling_type=PoolingType[pooler_config.pooling_type]
|
||||
if pooler_config.pooling_type is not None else pooling_type,
|
||||
normalize=pooler_config.normalize
|
||||
if pooler_config.normalize is not None else normalize,
|
||||
softmax=pooler_config.softmax
|
||||
if pooler_config.softmax is not None else softmax,
|
||||
step_tag_id=pooler_config.step_tag_id
|
||||
if pooler_config.step_tag_id is not None else step_tag_id,
|
||||
returned_token_ids=pooler_config.returned_token_ids
|
||||
if pooler_config.returned_token_ids is not None else
|
||||
returned_token_ids,
|
||||
) -> None:
|
||||
super().__init__()
|
||||
|
||||
self.pooling = AllPool()
|
||||
self.head = head
|
||||
self.step_tag_id = step_tag_id
|
||||
self.returned_token_ids = returned_token_ids
|
||||
|
||||
def get_prompt_token_ids(
|
||||
self,
|
||||
pooling_metadata: PoolingMetadata,
|
||||
) -> list[torch.Tensor]:
|
||||
if isinstance(pooling_metadata, V1PoolingMetadata):
|
||||
return [
|
||||
pooling_metadata.prompt_token_ids[i, :num]
|
||||
for i, num in enumerate(pooling_metadata.prompt_lens)
|
||||
]
|
||||
return [
|
||||
torch.tensor(seq_data_i.prompt_token_ids)
|
||||
for seq_data_i in pooling_metadata.seq_data.values()
|
||||
]
|
||||
|
||||
def extract_states(
|
||||
self,
|
||||
hidden_states: Union[torch.Tensor, list[torch.Tensor]],
|
||||
pooling_metadata: PoolingMetadata,
|
||||
) -> Union[list[torch.Tensor], torch.Tensor]:
|
||||
pooled_data_lst = self.pooling(hidden_states, pooling_metadata)
|
||||
prompt_token_ids = self.get_prompt_token_ids(pooling_metadata)
|
||||
|
||||
pooled_data = list[torch.Tensor]()
|
||||
returned_token_ids = self.returned_token_ids
|
||||
step_tag_id = self.step_tag_id
|
||||
|
||||
for data, token_id in zip(pooled_data_lst, prompt_token_ids):
|
||||
if returned_token_ids is not None and len(returned_token_ids) > 0:
|
||||
data = data[:, returned_token_ids]
|
||||
|
||||
if step_tag_id is not None:
|
||||
data = data[token_id == step_tag_id]
|
||||
pooled_data.append(data)
|
||||
|
||||
return pooled_data
|
||||
|
||||
def forward(
|
||||
self,
|
||||
hidden_states: Union[torch.Tensor, list[torch.Tensor]],
|
||||
pooling_metadata: PoolingMetadata,
|
||||
) -> PoolerOutput:
|
||||
pooled_data = self.extract_states(hidden_states, pooling_metadata)
|
||||
pooled_data = self.head(pooled_data, pooling_metadata)
|
||||
return build_output(pooled_data)
|
||||
|
||||
|
||||
class Pooler(nn.Module):
|
||||
|
||||
@staticmethod
|
||||
def from_config_with_defaults(
|
||||
pooler_config: PoolerConfig,
|
||||
pooling_type: PoolingType,
|
||||
normalize: bool,
|
||||
softmax: bool,
|
||||
step_tag_id: Optional[int] = None,
|
||||
returned_token_ids: Optional[list[int]] = None,
|
||||
) -> BasePooler:
|
||||
resolved_config = ResolvedPoolingConfig.from_config_with_defaults(
|
||||
pooler_config=pooler_config,
|
||||
pooling_type=pooling_type,
|
||||
normalize=normalize,
|
||||
softmax=softmax,
|
||||
step_tag_id=step_tag_id,
|
||||
returned_token_ids=returned_token_ids,
|
||||
)
|
||||
|
||||
if pooling_type == PoolingType.STEP:
|
||||
return StepPooler.from_config(resolved_config)
|
||||
|
||||
return SimplePooler.from_config(resolved_config)
|
||||
|
||||
|
||||
PoolingFn = Callable[
|
||||
[Union[torch.Tensor, list[torch.Tensor]], PoolingMetadata],
|
||||
Union[torch.Tensor, list[torch.Tensor]]]
|
||||
ClassifierFn = Callable[[torch.Tensor], torch.Tensor]
|
||||
|
||||
|
||||
class ClassifierPooler(nn.Module):
|
||||
"""A pooling layer for classification tasks.
|
||||
@@ -382,69 +600,39 @@ class ClassifierPooler(nn.Module):
|
||||
def __init__(
|
||||
self,
|
||||
config: ModelConfig,
|
||||
classifier: nn.Module,
|
||||
pooler: Optional[nn.Module] = None,
|
||||
):
|
||||
pooling: PoolingFn,
|
||||
classifier: ClassifierFn,
|
||||
act_fn: Optional[PoolerActivation] = None,
|
||||
) -> None:
|
||||
super().__init__()
|
||||
|
||||
self.pooling = pooling
|
||||
self.classifier = classifier
|
||||
self.pooler = pooler
|
||||
|
||||
self.classification_act_fn = get_classification_activation_function(
|
||||
config.hf_config)
|
||||
config.hf_config) if act_fn is None else act_fn
|
||||
self.cross_encoder_act_fn = get_cross_encoder_activation_function(
|
||||
config.hf_config)
|
||||
config.hf_config) if act_fn is None else act_fn
|
||||
|
||||
def _get_act_fn(self, use_cross_encoder: bool):
|
||||
return (self.cross_encoder_act_fn
|
||||
if use_cross_encoder else self.classification_act_fn)
|
||||
|
||||
def get_prompt_lens(
|
||||
self,
|
||||
hidden_states: Union[torch.Tensor, list[torch.Tensor]],
|
||||
pooling_metadata: PoolingMetadata,
|
||||
) -> torch.Tensor:
|
||||
if isinstance(pooling_metadata, V1PoolingMetadata):
|
||||
return pooling_metadata.prompt_lens
|
||||
assert isinstance(hidden_states, torch.Tensor)
|
||||
return PoolingTensors.from_pooling_metadata(
|
||||
pooling_metadata, hidden_states.device).prompt_lens
|
||||
|
||||
def forward(
|
||||
self,
|
||||
hidden_states: Union[torch.Tensor, list[torch.Tensor]],
|
||||
pooling_metadata: PoolingMetadata,
|
||||
) -> PoolerOutput:
|
||||
"""Pools sentence pair scores from the hidden_states."""
|
||||
prompt_lens = self.get_prompt_lens(hidden_states, pooling_metadata)
|
||||
pooled_data = self.pooling(hidden_states, pooling_metadata)
|
||||
|
||||
pooled_data = list[torch.Tensor]()
|
||||
if isinstance(hidden_states, list):
|
||||
for req_state, prompt_len in zip(hidden_states, prompt_lens):
|
||||
assert prompt_len == req_state.shape[0], \
|
||||
"partial prefill not supported with classifier"
|
||||
pooled_data = hidden_states
|
||||
# apply classifier once on the full batch if possible
|
||||
if isinstance(pooled_data, torch.Tensor):
|
||||
pooled_output = self.classifier(pooled_data)
|
||||
elif len({data.shape for data in pooled_data}) <= 1:
|
||||
pooled_output = self.classifier(torch.stack(pooled_data))
|
||||
else:
|
||||
offset = 0
|
||||
for prompt_len in prompt_lens:
|
||||
pooled_data_i = hidden_states[offset:offset + prompt_len]
|
||||
offset += prompt_len
|
||||
pooled_data.append(pooled_data_i)
|
||||
|
||||
pooled_data_lst = []
|
||||
for pooled_data_i in pooled_data:
|
||||
|
||||
if self.pooler is not None:
|
||||
final_shape_tensor = self.pooler(pooled_data_i)
|
||||
else:
|
||||
final_shape_tensor = self.classifier(pooled_data_i)
|
||||
|
||||
pooled_data_lst.append(final_shape_tensor)
|
||||
|
||||
pooled_output = torch.stack(pooled_data_lst)
|
||||
|
||||
if self.pooler is not None:
|
||||
# apply classifier once on the full batch if possible
|
||||
pooled_output = self.classifier(pooled_output)
|
||||
pooled_output = [self.classifier(data) for data in pooled_data]
|
||||
|
||||
if isinstance(pooling_metadata, V0PoolingMetadata):
|
||||
use_cross_encoder_list = [
|
||||
@@ -469,5 +657,4 @@ class ClassifierPooler(nn.Module):
|
||||
pooled_output)
|
||||
])
|
||||
|
||||
pooled_outputs = [PoolingSequenceGroupOutput(data) for data in scores]
|
||||
return PoolerOutput(outputs=pooled_outputs)
|
||||
return build_output(scores)
|
||||
|
||||
Reference in New Issue
Block a user