[Core] Set pooling params based on task and model (#21128)

Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
This commit is contained in:
Cyrus Leung
2025-07-18 05:41:17 -07:00
committed by GitHub
parent 4adc66f64d
commit 45badd05d0
24 changed files with 509 additions and 241 deletions
+95 -54
View File
@@ -3,7 +3,7 @@
from abc import ABC, abstractmethod
from dataclasses import dataclass
from enum import IntEnum
from typing import Callable, Literal, Optional, TypeVar, Union
from typing import Callable, Optional, TypeVar, Union
import torch
import torch.nn as nn
@@ -15,13 +15,12 @@ from vllm.config import ModelConfig, PoolerConfig
from vllm.model_executor.pooling_metadata import ( # noqa: E501
PoolingMetadata as V0PoolingMetadata)
from vllm.model_executor.pooling_metadata import PoolingTensors
from vllm.pooling_params import PoolingParams
from vllm.pooling_params import PoolingParams, PoolingTask
from vllm.sequence import PoolerOutput, PoolingSequenceGroupOutput
from vllm.utils import resolve_obj_by_qualname
from vllm.v1.pool.metadata import PoolingMetadata as V1PoolingMetadata
PoolingMetadata = Union[V0PoolingMetadata, V1PoolingMetadata]
PoolingTask = Literal["encode", "embed", "classify", "score"]
class PoolingType(IntEnum):
@@ -67,6 +66,15 @@ class ResolvedPoolingConfig:
)
@dataclass(frozen=True)
class PoolingParamsUpdate:
requires_token_ids: bool = False
"""Set this flag to enable `get_prompt_token_ids` for your pooler."""
def apply(self, params: PoolingParams) -> None:
params.requires_token_ids = self.requires_token_ids
class Pooler(nn.Module, ABC):
"""The interface required for all poolers used in pooling models in vLLM."""
@@ -93,7 +101,10 @@ class Pooler(nn.Module, ABC):
return SimplePooler.from_config(resolved_config)
def get_pooling_params(self, task: PoolingTask) -> Optional[PoolingParams]:
def get_pooling_updates(
self,
task: PoolingTask,
) -> Optional[PoolingParamsUpdate]:
"""
Construct the pooling parameters to use for a task,
or `None` if the task is not supported.
@@ -121,6 +132,23 @@ def get_prompt_lens(
pooling_metadata, hidden_states.device).prompt_lens
def get_prompt_token_ids(
pooling_metadata: PoolingMetadata) -> list[torch.Tensor]:
if isinstance(pooling_metadata, V1PoolingMetadata):
assert pooling_metadata.prompt_token_ids is not None, (
"Please set `requires_token_ids=True` in `get_pooling_updates`")
return [
pooling_metadata.prompt_token_ids[i, :num]
for i, num in enumerate(pooling_metadata.prompt_lens)
]
return [
torch.tensor(seq_data_i.prompt_token_ids)
for seq_data_i in pooling_metadata.seq_data.values()
]
def get_classification_activation_function(config: PretrainedConfig):
return PoolerClassify()
@@ -165,7 +193,10 @@ class PoolingMethod(nn.Module, ABC):
raise NotImplementedError(f"Unsupported method: {pooling_type}")
@abstractmethod
def get_pooling_params(self, task: PoolingTask) -> Optional[PoolingParams]:
def get_pooling_updates(
self,
task: PoolingTask,
) -> Optional[PoolingParamsUpdate]:
raise NotImplementedError
@abstractmethod
@@ -206,11 +237,14 @@ class PoolingMethod(nn.Module, ABC):
class CLSPool(PoolingMethod):
def get_pooling_params(self, task: PoolingTask) -> Optional[PoolingParams]:
def get_pooling_updates(
self,
task: PoolingTask,
) -> Optional[PoolingParamsUpdate]:
# The equalities are split up to keep mypy happy
if (task == "encode" or task == "embed" or task == "classify"
or task == "score"):
return PoolingParams()
return PoolingParamsUpdate()
assert_never(task)
@@ -236,11 +270,14 @@ class CLSPool(PoolingMethod):
class LastPool(PoolingMethod):
def get_pooling_params(self, task: PoolingTask) -> Optional[PoolingParams]:
def get_pooling_updates(
self,
task: PoolingTask,
) -> Optional[PoolingParamsUpdate]:
# The equalities are split up to keep mypy happy
if (task == "encode" or task == "embed" or task == "classify"
or task == "score"):
return PoolingParams()
return PoolingParamsUpdate()
assert_never(task)
@@ -262,9 +299,12 @@ class LastPool(PoolingMethod):
class AllPool(PoolingMethod):
def get_pooling_params(self, task: PoolingTask) -> Optional[PoolingParams]:
def get_pooling_updates(
self,
task: PoolingTask,
) -> Optional[PoolingParamsUpdate]:
if task == "encode":
return PoolingParams()
return PoolingParamsUpdate()
# The equalities are split up to keep mypy happy
if task == "embed" or task == "classify" or task == "score":
@@ -299,11 +339,14 @@ class AllPool(PoolingMethod):
class MeanPool(PoolingMethod):
def get_pooling_params(self, task: PoolingTask) -> Optional[PoolingParams]:
def get_pooling_updates(
self,
task: PoolingTask,
) -> Optional[PoolingParamsUpdate]:
# The equalities are split up to keep mypy happy
if (task == "encode" or task == "embed" or task == "classify"
or task == "score"):
return PoolingParams()
return PoolingParamsUpdate()
assert_never(task)
@@ -520,8 +563,11 @@ class SimplePooler(Pooler):
self.pooling = pooling
self.head = head
def get_pooling_params(self, task: PoolingTask) -> Optional[PoolingParams]:
return self.pooling.get_pooling_params(task)
def get_pooling_updates(
self,
task: PoolingTask,
) -> Optional[PoolingParamsUpdate]:
return self.pooling.get_pooling_updates(task)
def forward(
self,
@@ -559,27 +605,13 @@ class StepPooler(Pooler):
self.step_tag_id = step_tag_id
self.returned_token_ids = returned_token_ids
def get_prompt_token_ids(
self,
pooling_metadata: PoolingMetadata,
) -> list[torch.Tensor]:
if isinstance(pooling_metadata, V1PoolingMetadata):
return [
pooling_metadata.prompt_token_ids[i, :num]
for i, num in enumerate(pooling_metadata.prompt_lens)
]
return [
torch.tensor(seq_data_i.prompt_token_ids)
for seq_data_i in pooling_metadata.seq_data.values()
]
def extract_states(
self,
hidden_states: Union[torch.Tensor, list[torch.Tensor]],
pooling_metadata: PoolingMetadata,
) -> Union[list[torch.Tensor], torch.Tensor]:
pooled_data_lst = self.pooling(hidden_states, pooling_metadata)
prompt_token_ids = self.get_prompt_token_ids(pooling_metadata)
prompt_token_ids = get_prompt_token_ids(pooling_metadata)
pooled_data = list[torch.Tensor]()
returned_token_ids = self.returned_token_ids
@@ -595,9 +627,12 @@ class StepPooler(Pooler):
return pooled_data
def get_pooling_params(self, task: PoolingTask) -> Optional[PoolingParams]:
def get_pooling_updates(
self,
task: PoolingTask,
) -> Optional[PoolingParamsUpdate]:
if task == "encode":
return PoolingParams(logits_processing_needs_token_ids=True)
return PoolingParamsUpdate(requires_token_ids=True)
# The equalities are split up to keep mypy happy
if task == "embed" or task == "classify" or task == "score":
@@ -650,19 +685,24 @@ class ClassifierPooler(nn.Module):
self.cross_encoder_act_fn = get_cross_encoder_activation_function(
config.hf_config) if act_fn is None else act_fn
def _get_act_fn(self, use_cross_encoder: bool):
return (self.cross_encoder_act_fn
if use_cross_encoder else self.classification_act_fn)
def _get_act_fn(self, task: PoolingTask):
if task == "encode" or task == "classify":
return self.classification_act_fn
if task == "score":
return self.cross_encoder_act_fn
raise ValueError(f"Unsupported task: {task!r}")
def get_pooling_updates(
self,
task: PoolingTask,
) -> Optional[PoolingParamsUpdate]:
# The equalities are split up to keep mypy happy
if task == "encode" or task == "classify" or task == "score":
return PoolingParamsUpdate()
def get_pooling_params(self, task: PoolingTask) -> Optional[PoolingParams]:
if task == "encode":
return PoolingParams()
if task == "embed":
return None
if task == "classify":
return PoolingParams()
if task == "score":
return PoolingParams(use_cross_encoder=True)
assert_never(task)
@@ -682,27 +722,28 @@ class ClassifierPooler(nn.Module):
else:
pooled_output = [self.classifier(data) for data in pooled_data]
task_list: list[PoolingTask]
if isinstance(pooling_metadata, V0PoolingMetadata):
use_cross_encoder_list = [
pooling_param.use_cross_encoder
for _, pooling_param in pooling_metadata.seq_groups
task_list = [
task for _, pooling_param in pooling_metadata.seq_groups
if (task := pooling_param.task) is not None
]
else:
use_cross_encoder_list = [
pooling_param.use_cross_encoder
for pooling_param in pooling_metadata.pooling_params
task_list = [
task for pooling_param in pooling_metadata.pooling_params
if (task := pooling_param.task) is not None
]
assert len(task_list) == len(pooled_output)
# shape of scores: (batch_size, num_labels)
if all(use_cross_encoder == use_cross_encoder_list[0]
for use_cross_encoder in use_cross_encoder_list):
act_fn = self._get_act_fn(use_cross_encoder_list[0])
if len(set(task_list)) <= 1:
act_fn = self._get_act_fn(task_list[0])
scores = act_fn(pooled_output)
else:
scores = torch.stack([
self._get_act_fn(use_cross_encoder)(vecs)
for use_cross_encoder, vecs in zip(use_cross_encoder_list,
pooled_output)
self._get_act_fn(task)(vecs)
for task, vecs in zip(task_list, pooled_output)
])
return build_output(scores)
+8 -4
View File
@@ -18,13 +18,14 @@ from vllm.model_executor.layers.linear import (ColumnParallelLinear,
QKVParallelLinear,
RowParallelLinear)
from vllm.model_executor.layers.pooler import (ClassifierPooler, Pooler,
PoolingMethod, PoolingTask,
PoolingMethod,
PoolingParamsUpdate,
PoolingType)
from vllm.model_executor.layers.quantization import QuantizationConfig
from vllm.model_executor.layers.vocab_parallel_embedding import (
VocabParallelEmbedding)
from vllm.model_executor.pooling_metadata import PoolingMetadata
from vllm.pooling_params import PoolingParams
from vllm.pooling_params import PoolingTask
from vllm.sequence import IntermediateTensors
from .interfaces import SupportsCrossEncoding, SupportsQuant, SupportsV0Only
@@ -91,8 +92,11 @@ class BertPooler(Pooler):
self.dense = nn.Linear(config.hidden_size, config.hidden_size)
self.activation = nn.Tanh()
def get_pooling_params(self, task: PoolingTask) -> Optional[PoolingParams]:
return self.pooling.get_pooling_params(task)
def get_pooling_updates(
self,
task: PoolingTask,
) -> Optional[PoolingParamsUpdate]:
return self.pooling.get_pooling_updates(task)
def forward(
self,
+125 -78
View File
@@ -1,18 +1,24 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from array import array
from typing import Optional, Union
import numpy as np
import torch
import torch.nn as nn
from typing_extensions import assert_never
from vllm.config import ModelConfig, VllmConfig
from vllm.logger import init_logger
from vllm.model_executor.layers.pooler import PoolerHead, PoolerNormalize
from vllm.model_executor.layers.pooler import (Pooler, PoolerHead,
PoolerNormalize,
PoolingParamsUpdate,
build_output, get_prompt_lens,
get_prompt_token_ids)
from vllm.model_executor.models.llama import LlamaForCausalLM
from vllm.model_executor.pooling_metadata import (PoolingMetadata,
PoolingTensors)
from vllm.sequence import PoolerOutput, PoolingSequenceGroupOutput
from vllm.model_executor.pooling_metadata import PoolingMetadata
from vllm.pooling_params import PoolingTask
from vllm.sequence import PoolerOutput
from vllm.transformers_utils.tokenizer import cached_tokenizer_from_config
from .interfaces import SupportsV0Only
@@ -20,7 +26,8 @@ from .interfaces import SupportsV0Only
logger = init_logger(__name__)
class GritLMPooler(nn.Module):
class GritLMMeanPool(nn.Module):
"""As `MeanPool`, but only includes non-instruction tokens."""
def __init__(self, model_config: ModelConfig):
super().__init__()
@@ -38,8 +45,8 @@ class GritLMPooler(nn.Module):
for tok in ["<s>", "▁<", "<", "|", "embed", ">", "<0x0A>", "user"]
}
def tokens_to_ids(tokens: list[str]) -> array:
return array("i", [self.token_ids[token] for token in tokens])
def tokens_to_ids(tokens: list[str]) -> np.ndarray:
return np.array([self.token_ids[token] for token in tokens])
self.user_pattern_ids = tokens_to_ids(
["▁<", "|", "user", "|", ">", "<0x0A>"])
@@ -48,32 +55,44 @@ class GritLMPooler(nn.Module):
self.embed_pattern_ids = tokens_to_ids(
["▁<", "|", "embed", "|", ">", "<0x0A>"])
self.head = PoolerHead(PoolerNormalize())
def _find_array(self, arr: array, target: array, start_idx: int) -> int:
def _find_array(
self,
arr: np.ndarray,
target: np.ndarray,
start_idx: int = 0,
end_idx: Optional[int] = None,
) -> int:
"""
Find the first occurrence of target in arr starting from start_idx.
Find the first occurrence of `target` in `arr` starting from
`start_idx`.
Args:
arr: The array to search within
target: The consecutive subsequence to find
start_idx: The starting index to search from
arr: The array to search within.
target: The consecutive subsequence to find.
start_idx: The starting index to search from (inclusive).
end_idx: The ending index to search from (exclusive).
Returns:
int: The index of the first occurrence of target in arr.
The index of the first occurrence of `target` in `arr`.
"""
if start_idx < 0:
raise ValueError("start_idx must be non-negative")
if not target or not arr:
raise ValueError("Empty arr or target not allowed")
raise ValueError("`start_idx` must be non-negative")
if len(arr) == 0 or len(target) == 0:
raise ValueError("Empty `arr` or `target` not allowed")
arr_len = len(arr)
target_len = len(target)
for i in range(start_idx, len(arr) - target_len + 1):
if arr[i:i + target_len] == target:
if end_idx is None:
end_idx = arr_len
for i in range(start_idx, min(end_idx, arr_len - target_len + 1)):
if (arr[i:i + target_len] == target).all():
return i
return -1
def _get_instruction_len(self, prompt_token_ids: array) -> int:
def _get_instruction_len(self, prompt_token_ids: np.ndarray) -> int:
"""
Get the length of the instruction in the prompt.
@@ -83,7 +102,6 @@ class GritLMPooler(nn.Module):
The pattern matching is done using integers instead of strings
because the prompt is given as a list of token IDs.
"""
instruction_len = 0
# Return no instruction in case of missing BOS token.
@@ -98,7 +116,8 @@ class GritLMPooler(nn.Module):
embed_pattern_ids = self.embed_pattern_ids
if self._find_array(prompt_token_ids,
self.user_pattern_ids,
start_idx=1) == 1:
start_idx=1,
end_idx=2) == 1:
embed_pattern_ids = self.embed_newline_pattern_ids
# Find the embed pattern in the prompt.
@@ -116,64 +135,92 @@ class GritLMPooler(nn.Module):
return instruction_len
def get_pooling_updates(
self,
task: PoolingTask,
) -> Optional[PoolingParamsUpdate]:
# The equalities are split up to keep mypy happy
if task == "encode" or task == "embed":
return PoolingParamsUpdate(requires_token_ids=True)
if task == "classify" or task == "score":
return None
assert_never(task)
def forward_one(
self,
hidden_states: torch.Tensor,
prompt_len: Optional[torch.Tensor] = None,
instr_len: Optional[torch.Tensor] = None,
) -> torch.Tensor:
assert prompt_len is None or prompt_len == hidden_states.shape[0], \
"partial prefill not supported with MEAN pooling"
return hidden_states[instr_len:].mean(dim=0, dtype=torch.float32)
def forward_all(
self,
hidden_states: torch.Tensor,
prompt_lens: torch.Tensor,
instr_lens: torch.Tensor,
) -> Union[list[torch.Tensor], torch.Tensor]:
offset = 0
pooled_data = list[torch.Tensor]()
for prompt_len, instr_len in zip(prompt_lens, instr_lens):
pooled_data.append(hidden_states[offset + instr_len:offset +
prompt_len].mean(
dim=0, dtype=torch.float32))
offset += prompt_len
return pooled_data
def forward(
self,
hidden_states: Union[torch.Tensor, list[torch.Tensor]],
pooling_metadata: PoolingMetadata,
) -> Union[list[torch.Tensor], torch.Tensor]:
prompt_lens = get_prompt_lens(hidden_states, pooling_metadata)
instr_lens = torch.tensor(
[
self._get_instruction_len(token_ids.cpu().numpy())
for token_ids in get_prompt_token_ids(pooling_metadata)
],
device=prompt_lens.device,
)
if isinstance(hidden_states, list):
return [
self.forward_one(h, prompt_len, instr_len) for h, prompt_len,
instr_len in zip(hidden_states, prompt_lens, instr_lens)
]
return self.forward_all(hidden_states, prompt_lens, instr_lens)
class GritLMPooler(Pooler):
def __init__(self, model_config: ModelConfig):
super().__init__()
self.pooling = GritLMMeanPool(model_config)
self.head = PoolerHead(PoolerNormalize())
def get_pooling_updates(
self,
task: PoolingTask,
) -> Optional[PoolingParamsUpdate]:
return self.pooling.get_pooling_updates(task)
def forward(
self,
hidden_states: torch.Tensor,
pooling_metadata: PoolingMetadata,
) -> PoolerOutput:
"""
Pool the hidden states by summing the embeddings of
non-instruction tokens.
"""
prompts_token_ids = [
token_ids.prompt_token_ids_array
for _, token_ids in pooling_metadata.seq_data.items()
]
instruction_lens = torch.tensor(
[
self._get_instruction_len(prompt_token_ids)
for prompt_token_ids in prompts_token_ids
],
device=hidden_states.device,
)
prompt_lens = PoolingTensors.from_pooling_metadata(
pooling_metadata, hidden_states.device).prompt_lens
mask = torch.zeros_like(hidden_states, dtype=torch.bool)
start_idx = 0
for prompt_len, instruction_len in zip(prompt_lens, instruction_lens):
end_idx = start_idx + prompt_len
mask[start_idx + instruction_len:end_idx] = True
start_idx = end_idx
masked_hidden_states = hidden_states.masked_fill(~mask, 0.0)
sum_embeddings = torch.zeros(len(prompt_lens),
hidden_states.size(1),
device=hidden_states.device)
start_idx = 0
for i, prompt_len in enumerate(prompt_lens):
end_idx = start_idx + prompt_len
sum_embeddings[i] = masked_hidden_states[start_idx:end_idx].sum(
dim=0)
start_idx = end_idx
num_non_instruction_tokens = prompt_lens - instruction_lens
mean_embeddings = sum_embeddings / num_non_instruction_tokens.unsqueeze(
1)
pooled_data = self.head(mean_embeddings,
pooling_metadata=pooling_metadata)
pooled_outputs = [
PoolingSequenceGroupOutput(data) for data in pooled_data
]
return PoolerOutput(outputs=pooled_outputs)
pooled_data = self.pooling(hidden_states, pooling_metadata)
pooled_data = self.head(pooled_data, pooling_metadata)
return build_output(pooled_data)
class GritLM(LlamaForCausalLM, SupportsV0Only):
@@ -202,7 +249,7 @@ class GritLM(LlamaForCausalLM, SupportsV0Only):
prefix: str = "",
**kwargs,
) -> None:
# Use full attention for pooling
# Use full attention for pooling (this is why V1 is not supported yet)
if vllm_config.model_config.runner_type == "pooling":
hf_config = vllm_config.model_config.hf_config
hf_config.is_causal = False
-7
View File
@@ -599,13 +599,6 @@ def supports_cross_encoding(
return is_pooling_model(model) and _supports_cross_encoding(model)
def has_step_pooler(model: Union[type[object], object]) -> bool:
"""Check if the model uses step pooler."""
from vllm.model_executor.layers.pooler import StepPooler
return is_pooling_model(model) and isinstance(model.pooler, StepPooler)
class SupportsQuant:
"""The interface required for all models that support quantization."""
+8 -4
View File
@@ -14,14 +14,15 @@ from vllm.distributed import get_tensor_model_parallel_world_size
from vllm.model_executor.layers.linear import (QKVParallelLinear,
RowParallelLinear)
from vllm.model_executor.layers.pooler import (ClassifierPooler, Pooler,
PoolingMethod, PoolingTask,
PoolingMethod,
PoolingParamsUpdate,
PoolingType)
from vllm.model_executor.layers.rotary_embedding import RotaryEmbedding
from vllm.model_executor.layers.vocab_parallel_embedding import (
VocabParallelEmbedding)
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
from vllm.model_executor.pooling_metadata import PoolingMetadata
from vllm.pooling_params import PoolingParams
from vllm.pooling_params import PoolingTask
from vllm.sequence import IntermediateTensors
from .interfaces import SupportsCrossEncoding, SupportsV0Only
@@ -270,8 +271,11 @@ class ModernBertPooler(Pooler):
eps=config.norm_eps,
bias=config.norm_bias)
def get_pooling_params(self, task: PoolingTask) -> Optional[PoolingParams]:
return self.pooling.get_pooling_params(task)
def get_pooling_updates(
self,
task: PoolingTask,
) -> Optional[PoolingParamsUpdate]:
return self.pooling.get_pooling_updates(task)
def forward(
self,