Convert formatting to use ruff instead of yapf + isort (#26247)
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
This commit is contained in:
@@ -9,11 +9,15 @@ import torch.nn as nn
|
||||
|
||||
from vllm.config import ModelConfig, VllmConfig
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.pooler import (DispatchPooler, Pooler,
|
||||
PoolerHead, PoolerNormalize,
|
||||
PoolingParamsUpdate,
|
||||
get_prompt_lens,
|
||||
get_prompt_token_ids)
|
||||
from vllm.model_executor.layers.pooler import (
|
||||
DispatchPooler,
|
||||
Pooler,
|
||||
PoolerHead,
|
||||
PoolerNormalize,
|
||||
PoolingParamsUpdate,
|
||||
get_prompt_lens,
|
||||
get_prompt_token_ids,
|
||||
)
|
||||
from vllm.model_executor.models.llama import LlamaForCausalLM
|
||||
from vllm.tasks import PoolingTask
|
||||
from vllm.transformers_utils.tokenizer import cached_tokenizer_from_config
|
||||
@@ -47,12 +51,11 @@ class GritLMMeanPool(nn.Module):
|
||||
def tokens_to_ids(tokens: list[str]) -> np.ndarray:
|
||||
return np.array([self.token_ids[token] for token in tokens])
|
||||
|
||||
self.user_pattern_ids = tokens_to_ids(
|
||||
["▁<", "|", "user", "|", ">", "<0x0A>"])
|
||||
self.user_pattern_ids = tokens_to_ids(["▁<", "|", "user", "|", ">", "<0x0A>"])
|
||||
self.embed_newline_pattern_ids = tokens_to_ids(
|
||||
["<0x0A>", "<", "|", "embed", "|", ">", "<0x0A>"])
|
||||
self.embed_pattern_ids = tokens_to_ids(
|
||||
["▁<", "|", "embed", "|", ">", "<0x0A>"])
|
||||
["<0x0A>", "<", "|", "embed", "|", ">", "<0x0A>"]
|
||||
)
|
||||
self.embed_pattern_ids = tokens_to_ids(["▁<", "|", "embed", "|", ">", "<0x0A>"])
|
||||
|
||||
def _find_array(
|
||||
self,
|
||||
@@ -86,7 +89,7 @@ class GritLMMeanPool(nn.Module):
|
||||
end_idx = arr_len
|
||||
|
||||
for i in range(start_idx, min(end_idx, arr_len - target_len + 1)):
|
||||
if (arr[i:i + target_len] == target).all():
|
||||
if (arr[i : i + target_len] == target).all():
|
||||
return i
|
||||
|
||||
return -1
|
||||
@@ -105,31 +108,37 @@ class GritLMMeanPool(nn.Module):
|
||||
|
||||
# Return no instruction in case of missing BOS token.
|
||||
if prompt_token_ids[0] != self.token_ids["<s>"]:
|
||||
logger.warning("BOS token not found in prompt, "
|
||||
"thus using empty string for instruction. "
|
||||
"GritLM requires BOS token in prompt.")
|
||||
logger.warning(
|
||||
"BOS token not found in prompt, "
|
||||
"thus using empty string for instruction. "
|
||||
"GritLM requires BOS token in prompt."
|
||||
)
|
||||
return instruction_len
|
||||
|
||||
# If user pattern is found in the prompt, that means there should be
|
||||
# a newline token before the embed pattern.
|
||||
embed_pattern_ids = self.embed_pattern_ids
|
||||
if self._find_array(prompt_token_ids,
|
||||
self.user_pattern_ids,
|
||||
start_idx=1,
|
||||
end_idx=2) == 1:
|
||||
if (
|
||||
self._find_array(
|
||||
prompt_token_ids, self.user_pattern_ids, start_idx=1, end_idx=2
|
||||
)
|
||||
== 1
|
||||
):
|
||||
embed_pattern_ids = self.embed_newline_pattern_ids
|
||||
|
||||
# Find the embed pattern in the prompt.
|
||||
found_embed_pattern_idx = self._find_array(prompt_token_ids,
|
||||
embed_pattern_ids,
|
||||
start_idx=1)
|
||||
found_embed_pattern_idx = self._find_array(
|
||||
prompt_token_ids, embed_pattern_ids, start_idx=1
|
||||
)
|
||||
|
||||
if found_embed_pattern_idx != -1:
|
||||
instruction_len = found_embed_pattern_idx + len(embed_pattern_ids)
|
||||
else:
|
||||
logger.warning("Query instruction not found in prompt, "
|
||||
"thus using BOS token as instruction instead. "
|
||||
"GritLM requires query instruction in prompt.")
|
||||
logger.warning(
|
||||
"Query instruction not found in prompt, "
|
||||
"thus using BOS token as instruction instead. "
|
||||
"GritLM requires query instruction in prompt."
|
||||
)
|
||||
instruction_len = 1
|
||||
|
||||
return instruction_len
|
||||
@@ -146,8 +155,9 @@ class GritLMMeanPool(nn.Module):
|
||||
prompt_len: Optional[torch.Tensor] = None,
|
||||
instr_len: Optional[torch.Tensor] = None,
|
||||
) -> torch.Tensor:
|
||||
assert prompt_len is None or prompt_len == hidden_states.shape[0], \
|
||||
assert prompt_len is None or prompt_len == hidden_states.shape[0], (
|
||||
"partial prefill not supported with MEAN pooling"
|
||||
)
|
||||
|
||||
return hidden_states[instr_len:].mean(dim=0, dtype=torch.float32)
|
||||
|
||||
@@ -161,9 +171,11 @@ class GritLMMeanPool(nn.Module):
|
||||
pooled_data = list[torch.Tensor]()
|
||||
|
||||
for prompt_len, instr_len in zip(prompt_lens, instr_lens):
|
||||
pooled_data.append(hidden_states[offset + instr_len:offset +
|
||||
prompt_len].mean(
|
||||
dim=0, dtype=torch.float32))
|
||||
pooled_data.append(
|
||||
hidden_states[offset + instr_len : offset + prompt_len].mean(
|
||||
dim=0, dtype=torch.float32
|
||||
)
|
||||
)
|
||||
offset += prompt_len
|
||||
|
||||
return pooled_data
|
||||
@@ -184,15 +196,16 @@ class GritLMMeanPool(nn.Module):
|
||||
|
||||
if isinstance(hidden_states, list):
|
||||
return [
|
||||
self.forward_one(h, prompt_len, instr_len) for h, prompt_len,
|
||||
instr_len in zip(hidden_states, prompt_lens, instr_lens)
|
||||
self.forward_one(h, prompt_len, instr_len)
|
||||
for h, prompt_len, instr_len in zip(
|
||||
hidden_states, prompt_lens, instr_lens
|
||||
)
|
||||
]
|
||||
|
||||
return self.forward_all(hidden_states, prompt_lens, instr_lens)
|
||||
|
||||
|
||||
class GritLMPooler(Pooler):
|
||||
|
||||
def __init__(self, model_config: ModelConfig):
|
||||
super().__init__()
|
||||
|
||||
@@ -254,9 +267,9 @@ class GritLM(LlamaForCausalLM):
|
||||
|
||||
pooler_config = vllm_config.model_config.pooler_config
|
||||
if pooler_config is not None:
|
||||
self.pooler = DispatchPooler({
|
||||
"encode":
|
||||
Pooler.for_encode(pooler_config),
|
||||
"embed":
|
||||
GritLMPooler(vllm_config.model_config),
|
||||
})
|
||||
self.pooler = DispatchPooler(
|
||||
{
|
||||
"encode": Pooler.for_encode(pooler_config),
|
||||
"embed": GritLMPooler(vllm_config.model_config),
|
||||
}
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user