[Core] Set pooling params based on task and model (#21128)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
This commit is contained in:
@@ -18,13 +18,14 @@ from vllm.model_executor.layers.linear import (ColumnParallelLinear,
|
||||
QKVParallelLinear,
|
||||
RowParallelLinear)
|
||||
from vllm.model_executor.layers.pooler import (ClassifierPooler, Pooler,
|
||||
PoolingMethod, PoolingTask,
|
||||
PoolingMethod,
|
||||
PoolingParamsUpdate,
|
||||
PoolingType)
|
||||
from vllm.model_executor.layers.quantization import QuantizationConfig
|
||||
from vllm.model_executor.layers.vocab_parallel_embedding import (
|
||||
VocabParallelEmbedding)
|
||||
from vllm.model_executor.pooling_metadata import PoolingMetadata
|
||||
from vllm.pooling_params import PoolingParams
|
||||
from vllm.pooling_params import PoolingTask
|
||||
from vllm.sequence import IntermediateTensors
|
||||
|
||||
from .interfaces import SupportsCrossEncoding, SupportsQuant, SupportsV0Only
|
||||
@@ -91,8 +92,11 @@ class BertPooler(Pooler):
|
||||
self.dense = nn.Linear(config.hidden_size, config.hidden_size)
|
||||
self.activation = nn.Tanh()
|
||||
|
||||
def get_pooling_params(self, task: PoolingTask) -> Optional[PoolingParams]:
|
||||
return self.pooling.get_pooling_params(task)
|
||||
def get_pooling_updates(
|
||||
self,
|
||||
task: PoolingTask,
|
||||
) -> Optional[PoolingParamsUpdate]:
|
||||
return self.pooling.get_pooling_updates(task)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
|
||||
@@ -1,18 +1,24 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from array import array
|
||||
from typing import Optional, Union
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
from typing_extensions import assert_never
|
||||
|
||||
from vllm.config import ModelConfig, VllmConfig
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.pooler import PoolerHead, PoolerNormalize
|
||||
from vllm.model_executor.layers.pooler import (Pooler, PoolerHead,
|
||||
PoolerNormalize,
|
||||
PoolingParamsUpdate,
|
||||
build_output, get_prompt_lens,
|
||||
get_prompt_token_ids)
|
||||
from vllm.model_executor.models.llama import LlamaForCausalLM
|
||||
from vllm.model_executor.pooling_metadata import (PoolingMetadata,
|
||||
PoolingTensors)
|
||||
from vllm.sequence import PoolerOutput, PoolingSequenceGroupOutput
|
||||
from vllm.model_executor.pooling_metadata import PoolingMetadata
|
||||
from vllm.pooling_params import PoolingTask
|
||||
from vllm.sequence import PoolerOutput
|
||||
from vllm.transformers_utils.tokenizer import cached_tokenizer_from_config
|
||||
|
||||
from .interfaces import SupportsV0Only
|
||||
@@ -20,7 +26,8 @@ from .interfaces import SupportsV0Only
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
class GritLMPooler(nn.Module):
|
||||
class GritLMMeanPool(nn.Module):
|
||||
"""As `MeanPool`, but only includes non-instruction tokens."""
|
||||
|
||||
def __init__(self, model_config: ModelConfig):
|
||||
super().__init__()
|
||||
@@ -38,8 +45,8 @@ class GritLMPooler(nn.Module):
|
||||
for tok in ["<s>", "▁<", "<", "|", "embed", ">", "<0x0A>", "user"]
|
||||
}
|
||||
|
||||
def tokens_to_ids(tokens: list[str]) -> array:
|
||||
return array("i", [self.token_ids[token] for token in tokens])
|
||||
def tokens_to_ids(tokens: list[str]) -> np.ndarray:
|
||||
return np.array([self.token_ids[token] for token in tokens])
|
||||
|
||||
self.user_pattern_ids = tokens_to_ids(
|
||||
["▁<", "|", "user", "|", ">", "<0x0A>"])
|
||||
@@ -48,32 +55,44 @@ class GritLMPooler(nn.Module):
|
||||
self.embed_pattern_ids = tokens_to_ids(
|
||||
["▁<", "|", "embed", "|", ">", "<0x0A>"])
|
||||
|
||||
self.head = PoolerHead(PoolerNormalize())
|
||||
|
||||
def _find_array(self, arr: array, target: array, start_idx: int) -> int:
|
||||
def _find_array(
|
||||
self,
|
||||
arr: np.ndarray,
|
||||
target: np.ndarray,
|
||||
start_idx: int = 0,
|
||||
end_idx: Optional[int] = None,
|
||||
) -> int:
|
||||
"""
|
||||
Find the first occurrence of target in arr starting from start_idx.
|
||||
Find the first occurrence of `target` in `arr` starting from
|
||||
`start_idx`.
|
||||
|
||||
Args:
|
||||
arr: The array to search within
|
||||
target: The consecutive subsequence to find
|
||||
start_idx: The starting index to search from
|
||||
arr: The array to search within.
|
||||
target: The consecutive subsequence to find.
|
||||
start_idx: The starting index to search from (inclusive).
|
||||
end_idx: The ending index to search from (exclusive).
|
||||
|
||||
Returns:
|
||||
int: The index of the first occurrence of target in arr.
|
||||
The index of the first occurrence of `target` in `arr`.
|
||||
"""
|
||||
if start_idx < 0:
|
||||
raise ValueError("start_idx must be non-negative")
|
||||
if not target or not arr:
|
||||
raise ValueError("Empty arr or target not allowed")
|
||||
raise ValueError("`start_idx` must be non-negative")
|
||||
if len(arr) == 0 or len(target) == 0:
|
||||
raise ValueError("Empty `arr` or `target` not allowed")
|
||||
|
||||
arr_len = len(arr)
|
||||
target_len = len(target)
|
||||
for i in range(start_idx, len(arr) - target_len + 1):
|
||||
if arr[i:i + target_len] == target:
|
||||
|
||||
if end_idx is None:
|
||||
end_idx = arr_len
|
||||
|
||||
for i in range(start_idx, min(end_idx, arr_len - target_len + 1)):
|
||||
if (arr[i:i + target_len] == target).all():
|
||||
return i
|
||||
|
||||
return -1
|
||||
|
||||
def _get_instruction_len(self, prompt_token_ids: array) -> int:
|
||||
def _get_instruction_len(self, prompt_token_ids: np.ndarray) -> int:
|
||||
"""
|
||||
Get the length of the instruction in the prompt.
|
||||
|
||||
@@ -83,7 +102,6 @@ class GritLMPooler(nn.Module):
|
||||
The pattern matching is done using integers instead of strings
|
||||
because the prompt is given as a list of token IDs.
|
||||
"""
|
||||
|
||||
instruction_len = 0
|
||||
|
||||
# Return no instruction in case of missing BOS token.
|
||||
@@ -98,7 +116,8 @@ class GritLMPooler(nn.Module):
|
||||
embed_pattern_ids = self.embed_pattern_ids
|
||||
if self._find_array(prompt_token_ids,
|
||||
self.user_pattern_ids,
|
||||
start_idx=1) == 1:
|
||||
start_idx=1,
|
||||
end_idx=2) == 1:
|
||||
embed_pattern_ids = self.embed_newline_pattern_ids
|
||||
|
||||
# Find the embed pattern in the prompt.
|
||||
@@ -116,64 +135,92 @@ class GritLMPooler(nn.Module):
|
||||
|
||||
return instruction_len
|
||||
|
||||
def get_pooling_updates(
|
||||
self,
|
||||
task: PoolingTask,
|
||||
) -> Optional[PoolingParamsUpdate]:
|
||||
# The equalities are split up to keep mypy happy
|
||||
if task == "encode" or task == "embed":
|
||||
return PoolingParamsUpdate(requires_token_ids=True)
|
||||
|
||||
if task == "classify" or task == "score":
|
||||
return None
|
||||
|
||||
assert_never(task)
|
||||
|
||||
def forward_one(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
prompt_len: Optional[torch.Tensor] = None,
|
||||
instr_len: Optional[torch.Tensor] = None,
|
||||
) -> torch.Tensor:
|
||||
assert prompt_len is None or prompt_len == hidden_states.shape[0], \
|
||||
"partial prefill not supported with MEAN pooling"
|
||||
|
||||
return hidden_states[instr_len:].mean(dim=0, dtype=torch.float32)
|
||||
|
||||
def forward_all(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
prompt_lens: torch.Tensor,
|
||||
instr_lens: torch.Tensor,
|
||||
) -> Union[list[torch.Tensor], torch.Tensor]:
|
||||
offset = 0
|
||||
pooled_data = list[torch.Tensor]()
|
||||
|
||||
for prompt_len, instr_len in zip(prompt_lens, instr_lens):
|
||||
pooled_data.append(hidden_states[offset + instr_len:offset +
|
||||
prompt_len].mean(
|
||||
dim=0, dtype=torch.float32))
|
||||
offset += prompt_len
|
||||
|
||||
return pooled_data
|
||||
|
||||
def forward(
|
||||
self,
|
||||
hidden_states: Union[torch.Tensor, list[torch.Tensor]],
|
||||
pooling_metadata: PoolingMetadata,
|
||||
) -> Union[list[torch.Tensor], torch.Tensor]:
|
||||
prompt_lens = get_prompt_lens(hidden_states, pooling_metadata)
|
||||
instr_lens = torch.tensor(
|
||||
[
|
||||
self._get_instruction_len(token_ids.cpu().numpy())
|
||||
for token_ids in get_prompt_token_ids(pooling_metadata)
|
||||
],
|
||||
device=prompt_lens.device,
|
||||
)
|
||||
|
||||
if isinstance(hidden_states, list):
|
||||
return [
|
||||
self.forward_one(h, prompt_len, instr_len) for h, prompt_len,
|
||||
instr_len in zip(hidden_states, prompt_lens, instr_lens)
|
||||
]
|
||||
|
||||
return self.forward_all(hidden_states, prompt_lens, instr_lens)
|
||||
|
||||
|
||||
class GritLMPooler(Pooler):
|
||||
|
||||
def __init__(self, model_config: ModelConfig):
|
||||
super().__init__()
|
||||
|
||||
self.pooling = GritLMMeanPool(model_config)
|
||||
self.head = PoolerHead(PoolerNormalize())
|
||||
|
||||
def get_pooling_updates(
|
||||
self,
|
||||
task: PoolingTask,
|
||||
) -> Optional[PoolingParamsUpdate]:
|
||||
return self.pooling.get_pooling_updates(task)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
pooling_metadata: PoolingMetadata,
|
||||
) -> PoolerOutput:
|
||||
"""
|
||||
Pool the hidden states by summing the embeddings of
|
||||
non-instruction tokens.
|
||||
"""
|
||||
prompts_token_ids = [
|
||||
token_ids.prompt_token_ids_array
|
||||
for _, token_ids in pooling_metadata.seq_data.items()
|
||||
]
|
||||
|
||||
instruction_lens = torch.tensor(
|
||||
[
|
||||
self._get_instruction_len(prompt_token_ids)
|
||||
for prompt_token_ids in prompts_token_ids
|
||||
],
|
||||
device=hidden_states.device,
|
||||
)
|
||||
|
||||
prompt_lens = PoolingTensors.from_pooling_metadata(
|
||||
pooling_metadata, hidden_states.device).prompt_lens
|
||||
|
||||
mask = torch.zeros_like(hidden_states, dtype=torch.bool)
|
||||
|
||||
start_idx = 0
|
||||
for prompt_len, instruction_len in zip(prompt_lens, instruction_lens):
|
||||
end_idx = start_idx + prompt_len
|
||||
mask[start_idx + instruction_len:end_idx] = True
|
||||
start_idx = end_idx
|
||||
|
||||
masked_hidden_states = hidden_states.masked_fill(~mask, 0.0)
|
||||
|
||||
sum_embeddings = torch.zeros(len(prompt_lens),
|
||||
hidden_states.size(1),
|
||||
device=hidden_states.device)
|
||||
|
||||
start_idx = 0
|
||||
for i, prompt_len in enumerate(prompt_lens):
|
||||
end_idx = start_idx + prompt_len
|
||||
sum_embeddings[i] = masked_hidden_states[start_idx:end_idx].sum(
|
||||
dim=0)
|
||||
start_idx = end_idx
|
||||
|
||||
num_non_instruction_tokens = prompt_lens - instruction_lens
|
||||
mean_embeddings = sum_embeddings / num_non_instruction_tokens.unsqueeze(
|
||||
1)
|
||||
|
||||
pooled_data = self.head(mean_embeddings,
|
||||
pooling_metadata=pooling_metadata)
|
||||
|
||||
pooled_outputs = [
|
||||
PoolingSequenceGroupOutput(data) for data in pooled_data
|
||||
]
|
||||
|
||||
return PoolerOutput(outputs=pooled_outputs)
|
||||
pooled_data = self.pooling(hidden_states, pooling_metadata)
|
||||
pooled_data = self.head(pooled_data, pooling_metadata)
|
||||
return build_output(pooled_data)
|
||||
|
||||
|
||||
class GritLM(LlamaForCausalLM, SupportsV0Only):
|
||||
@@ -202,7 +249,7 @@ class GritLM(LlamaForCausalLM, SupportsV0Only):
|
||||
prefix: str = "",
|
||||
**kwargs,
|
||||
) -> None:
|
||||
# Use full attention for pooling
|
||||
# Use full attention for pooling (this is why V1 is not supported yet)
|
||||
if vllm_config.model_config.runner_type == "pooling":
|
||||
hf_config = vllm_config.model_config.hf_config
|
||||
hf_config.is_causal = False
|
||||
|
||||
@@ -599,13 +599,6 @@ def supports_cross_encoding(
|
||||
return is_pooling_model(model) and _supports_cross_encoding(model)
|
||||
|
||||
|
||||
def has_step_pooler(model: Union[type[object], object]) -> bool:
|
||||
"""Check if the model uses step pooler."""
|
||||
from vllm.model_executor.layers.pooler import StepPooler
|
||||
|
||||
return is_pooling_model(model) and isinstance(model.pooler, StepPooler)
|
||||
|
||||
|
||||
class SupportsQuant:
|
||||
"""The interface required for all models that support quantization."""
|
||||
|
||||
|
||||
@@ -14,14 +14,15 @@ from vllm.distributed import get_tensor_model_parallel_world_size
|
||||
from vllm.model_executor.layers.linear import (QKVParallelLinear,
|
||||
RowParallelLinear)
|
||||
from vllm.model_executor.layers.pooler import (ClassifierPooler, Pooler,
|
||||
PoolingMethod, PoolingTask,
|
||||
PoolingMethod,
|
||||
PoolingParamsUpdate,
|
||||
PoolingType)
|
||||
from vllm.model_executor.layers.rotary_embedding import RotaryEmbedding
|
||||
from vllm.model_executor.layers.vocab_parallel_embedding import (
|
||||
VocabParallelEmbedding)
|
||||
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
|
||||
from vllm.model_executor.pooling_metadata import PoolingMetadata
|
||||
from vllm.pooling_params import PoolingParams
|
||||
from vllm.pooling_params import PoolingTask
|
||||
from vllm.sequence import IntermediateTensors
|
||||
|
||||
from .interfaces import SupportsCrossEncoding, SupportsV0Only
|
||||
@@ -270,8 +271,11 @@ class ModernBertPooler(Pooler):
|
||||
eps=config.norm_eps,
|
||||
bias=config.norm_bias)
|
||||
|
||||
def get_pooling_params(self, task: PoolingTask) -> Optional[PoolingParams]:
|
||||
return self.pooling.get_pooling_params(task)
|
||||
def get_pooling_updates(
|
||||
self,
|
||||
task: PoolingTask,
|
||||
) -> Optional[PoolingParamsUpdate]:
|
||||
return self.pooling.get_pooling_updates(task)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
|
||||
Reference in New Issue
Block a user