[Model] Support Qwen2.5-Math-RM-72B (#8896)

This commit is contained in:
Zilin Zhu
2024-09-28 21:19:39 -07:00
committed by GitHub
parent 2e7fe7e79f
commit bc2ef1f77c
3 changed files with 170 additions and 0 deletions
+7
View File
@@ -11,6 +11,7 @@ from vllm.sequence import EmbeddingSequenceGroupOutput, PoolerOutput
class PoolingType(IntEnum):
"""Enumeration for different types of pooling methods."""
LAST = 0
ALL = 1
class Pooler(nn.Module):
@@ -43,6 +44,12 @@ class Pooler(nn.Module):
if self.pooling_type == PoolingType.LAST:
last_token_flat_indices = torch.cumsum(prompt_lens, dim=0) - 1
pooled_data = hidden_states[last_token_flat_indices]
elif self.pooling_type == PoolingType.ALL:
offset = 0
pooled_data = []
for prompt_len in prompt_lens:
pooled_data.append(hidden_states[offset:offset + prompt_len])
offset += prompt_len
else:
raise ValueError(f"Invalid pooling type: {self.pooling_type}")