# SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project from abc import ABC, abstractmethod from collections.abc import Set from typing import TypeAlias import torch import torch.nn as nn from vllm.model_executor.layers.pooler import ActivationFn, ClassifierFn, ProjectorFn from vllm.pooling_params import PoolingParams from vllm.tasks import PoolingTask from vllm.v1.pool.metadata import PoolingMetadata from .methods import TokenPoolingMethodOutputItem TokenPoolerHeadOutputItem: TypeAlias = torch.Tensor | None def _mean_pool_chunks(data: torch.Tensor, chunk_size: int) -> torch.Tensor: """Mean-pool complete contextual states, accumulating in FP32.""" chunk_size = min(chunk_size, len(data)) num_full, remainder = divmod(len(data), chunk_size) full_end = num_full * chunk_size full_chunks = data[:full_end].reshape(num_full, chunk_size, data.shape[-1]) if not remainder: return full_chunks.mean(dim=1, dtype=torch.float32) return torch.cat( [ full_chunks.mean(dim=1, dtype=torch.float32), data[full_end:].mean(dim=0, keepdim=True, dtype=torch.float32), ] ) class TokenPoolerHead(nn.Module, ABC): @abstractmethod def get_supported_tasks(self) -> Set[PoolingTask]: raise NotImplementedError @abstractmethod def forward_chunk( self, pooled_data: TokenPoolingMethodOutputItem, pooling_param: PoolingParams, ) -> TokenPoolerHeadOutputItem: raise NotImplementedError def forward( self, pooled_data: list[TokenPoolingMethodOutputItem], pooling_metadata: PoolingMetadata, ) -> list[TokenPoolerHeadOutputItem]: pooling_params = pooling_metadata.pooling_params if len(pooled_data) != len(pooling_params): raise ValueError( f"pooled_data length ({len(pooled_data)}) does not match " f"pooling_params length ({len(pooling_params)})" ) return [self.forward_chunk(d, p) for d, p in zip(pooled_data, pooling_params)] class TokenEmbeddingPoolerHead(TokenPoolerHead): def __init__( self, head_dtype: torch.dtype | str | None = None, projector: ProjectorFn | None = None, activation: ActivationFn | None = None, ) -> None: super().__init__() self.head_dtype = head_dtype self.projector = projector self.activation = activation def extra_repr(self) -> str: attrs = [] if self.head_dtype is not None: attrs.append(f"head_dtype={self.head_dtype}") if self.projector is not None: attrs.append("projector=True") if self.activation is not None: attrs.append(f"activation={self.activation.__class__.__name__}") return ", ".join(attrs) def get_supported_tasks(self) -> Set[PoolingTask]: return {"token_embed"} def forward_chunk( self, pooled_data: TokenPoolingMethodOutputItem, pooling_param: PoolingParams, ) -> TokenPoolerHeadOutputItem: # for unfinished chunked prefill if pooled_data is None: return None if (late_chunking := pooling_param.late_chunking_params) is not None: means = _mean_pool_chunks(pooled_data, late_chunking.chunk_size) pooled_data = ( means.to(pooled_data.dtype) if self.head_dtype is None else means ) if self.head_dtype is not None: pooled_data = pooled_data.to(self.head_dtype) # pooled_data shape: [n_tokens, hidden_size] # Apply ST projector if self.projector is not None: embeddings = self.projector(pooled_data) else: embeddings = pooled_data # embeddings shape: [n_tokens, embedding_size] # for matryoshka representation if pooling_param.dimensions is not None: embeddings = embeddings[..., : pooling_param.dimensions] # for normalize if self.activation is not None and pooling_param.use_activation: embeddings = self.activation(embeddings) # embeddings shape: [n_tokens, embedding_size] return embeddings class TokenClassifierPoolerHead(TokenPoolerHead): def __init__( self, classifier: ClassifierFn | None = None, logit_mean: float | None = None, logit_sigma: float | None = None, head_dtype: torch.dtype | str | None = None, activation: ActivationFn | None = None, ) -> None: super().__init__() self.classifier = classifier self.logit_mean = logit_mean self.logit_sigma = logit_sigma self.head_dtype = head_dtype self.activation = activation def extra_repr(self) -> str: attrs = [] if self.head_dtype is not None: attrs.append(f"head_dtype={self.head_dtype}") if self.classifier is not None: attrs.append("classifier=True") if self.logit_mean is not None: attrs.append(f"logit_mean={self.logit_mean}") if self.logit_sigma is not None: attrs.append(f"logit_sigma={self.logit_sigma}") if self.activation is not None: attrs.append(f"activation={self.activation.__class__.__name__}") return ", ".join(attrs) def get_supported_tasks(self) -> Set[PoolingTask]: return {"token_classify"} def forward_chunk( self, pooled_data: TokenPoolingMethodOutputItem, pooling_param: PoolingParams, ) -> TokenPoolerHeadOutputItem: # for unfinished chunked prefill if pooled_data is None: return None if self.head_dtype is not None: pooled_data = pooled_data.to(self.head_dtype) # hidden_states shape: [n_token, hidden_size] if self.classifier is not None: logits = self.classifier(pooled_data) else: logits = pooled_data # logits shape: [n_token, num_labels] # Affine score calibration: activation((logit - mean) / sigma) if self.logit_mean is not None: logits = logits - self.logit_mean if self.logit_sigma is not None: logits = logits / self.logit_sigma if self.activation is not None and pooling_param.use_activation: logits = self.activation(logits) # logits shape: [n_token, num_labels] return logits