[Model] Support math-shepherd-mistral-7b-prm model (#9697)
Signed-off-by: Went-Liang <wenteng_liang@163.com>
This commit is contained in:
@@ -1,8 +1,10 @@
|
||||
from enum import IntEnum
|
||||
from typing import List, Optional
|
||||
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
|
||||
from vllm.config import PoolerConfig
|
||||
from vllm.model_executor.pooling_metadata import (PoolingMetadata,
|
||||
PoolingTensors)
|
||||
from vllm.sequence import EmbeddingSequenceGroupOutput, PoolerOutput
|
||||
@@ -13,6 +15,7 @@ class PoolingType(IntEnum):
|
||||
LAST = 0
|
||||
ALL = 1
|
||||
CLS = 2
|
||||
STEP = 3
|
||||
|
||||
|
||||
class Pooler(nn.Module):
|
||||
@@ -28,15 +31,47 @@ class Pooler(nn.Module):
|
||||
normalize: Whether to normalize the pooled data.
|
||||
"""
|
||||
|
||||
def __init__(self,
|
||||
pooling_type: PoolingType,
|
||||
normalize: bool,
|
||||
softmax: bool = False):
|
||||
def __init__(
|
||||
self,
|
||||
pooling_type: PoolingType,
|
||||
normalize: bool,
|
||||
softmax: bool,
|
||||
step_tag_id: Optional[int] = None,
|
||||
returned_token_ids: Optional[List[int]] = None,
|
||||
):
|
||||
super().__init__()
|
||||
|
||||
self.pooling_type = pooling_type
|
||||
self.normalize = normalize
|
||||
self.softmax = softmax
|
||||
self.step_tag_id = step_tag_id
|
||||
self.returned_token_ids = returned_token_ids
|
||||
|
||||
@classmethod
|
||||
def from_config_with_defaults(
|
||||
cls,
|
||||
pooler_config: PoolerConfig,
|
||||
pooling_type: PoolingType,
|
||||
normalize: bool,
|
||||
softmax: bool,
|
||||
step_tag_id: Optional[int] = None,
|
||||
returned_token_ids: Optional[List[int]] = None,
|
||||
) -> Optional["Pooler"]:
|
||||
if pooler_config is None:
|
||||
return None
|
||||
return cls(
|
||||
pooling_type=PoolingType[pooler_config.pooling_type]
|
||||
if pooler_config.pooling_type is not None else pooling_type,
|
||||
normalize=pooler_config.pooling_norm
|
||||
if pooler_config.pooling_norm is not None else normalize,
|
||||
softmax=pooler_config.pooling_softmax
|
||||
if pooler_config.pooling_softmax is not None else softmax,
|
||||
step_tag_id=pooler_config.pooling_step_tag_id
|
||||
if pooler_config.pooling_step_tag_id is not None else step_tag_id,
|
||||
returned_token_ids=pooler_config.pooling_returned_token_ids
|
||||
if pooler_config.pooling_returned_token_ids is not None else
|
||||
returned_token_ids,
|
||||
)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
@@ -62,6 +97,25 @@ class Pooler(nn.Module):
|
||||
for prompt_len in prompt_lens:
|
||||
pooled_data.append(hidden_states[offset:offset + prompt_len])
|
||||
offset += prompt_len
|
||||
elif self.pooling_type == PoolingType.STEP:
|
||||
if self.returned_token_ids is not None and len(
|
||||
self.returned_token_ids) > 0:
|
||||
logits = hidden_states[:,
|
||||
self.returned_token_ids].softmax(dim=-1)
|
||||
else:
|
||||
logits = hidden_states.softmax(dim=-1)
|
||||
offset = 0
|
||||
pooled_data = []
|
||||
for prompt_len, seq_data_i in zip(
|
||||
prompt_lens, pooling_metadata.seq_data.values()):
|
||||
if self.step_tag_id is None:
|
||||
pooled_data.append(logits[offset:offset + prompt_len])
|
||||
else:
|
||||
step_idxs = torch.tensor(
|
||||
seq_data_i.prompt_token_ids) == self.step_tag_id
|
||||
pooled_data.append(logits[offset:offset +
|
||||
prompt_len][step_idxs])
|
||||
offset += prompt_len
|
||||
else:
|
||||
raise ValueError(f"Invalid pooling type: {self.pooling_type}")
|
||||
|
||||
|
||||
Reference in New Issue
Block a user