[CI] Make mistral tests pass (#4596)
This commit is contained in:
@@ -272,6 +272,68 @@ class HfRunner:
|
||||
all_logprobs.append(seq_logprobs)
|
||||
return all_logprobs
|
||||
|
||||
def generate_greedy_logprobs_limit(
|
||||
self,
|
||||
prompts: List[str],
|
||||
max_tokens: int,
|
||||
num_logprobs: int,
|
||||
) -> List[Tuple[List[int], str]]:
|
||||
all_logprobs = []
|
||||
all_output_ids = []
|
||||
all_output_strs = []
|
||||
|
||||
for prompt in prompts:
|
||||
input_ids = self.tokenizer(prompt, return_tensors="pt").input_ids
|
||||
output = self.model.generate(
|
||||
input_ids.cuda(),
|
||||
use_cache=True,
|
||||
do_sample=False,
|
||||
max_new_tokens=max_tokens,
|
||||
output_hidden_states=True,
|
||||
return_dict_in_generate=True,
|
||||
)
|
||||
|
||||
seq_logprobs = []
|
||||
for _, hidden_states in enumerate(output.hidden_states):
|
||||
last_hidden_states = hidden_states[-1][0]
|
||||
logits = torch.matmul(
|
||||
last_hidden_states,
|
||||
self.model.get_output_embeddings().weight.t(),
|
||||
)
|
||||
if getattr(self.model.get_output_embeddings(), "bias",
|
||||
None) is not None:
|
||||
logits += self.model.get_output_embeddings(
|
||||
).bias.unsqueeze(0)
|
||||
logprobs = torch.nn.functional.log_softmax(logits,
|
||||
dim=-1,
|
||||
dtype=torch.float32)
|
||||
seq_logprobs.append(logprobs)
|
||||
|
||||
# convert to dict
|
||||
seq_logprobs_lst = []
|
||||
for tok_idx, tok_logprobs in enumerate(seq_logprobs):
|
||||
# drop prompt logprobs
|
||||
if tok_idx == 0:
|
||||
tok_logprobs = tok_logprobs[-1, :].reshape(1, -1)
|
||||
topk = tok_logprobs.topk(num_logprobs)
|
||||
|
||||
tok_logprobs_dct = {}
|
||||
for token_id, logprob in zip(topk.indices[0], topk.values[0]):
|
||||
tok_logprobs_dct[token_id.item()] = logprob.item()
|
||||
|
||||
seq_logprobs_lst.append(tok_logprobs_dct)
|
||||
|
||||
all_logprobs.append(seq_logprobs_lst)
|
||||
seq_ids = output.sequences[0]
|
||||
output_len = seq_ids.shape[0] - input_ids.shape[1]
|
||||
output_ids = seq_ids[-output_len:]
|
||||
all_output_ids.append(output_ids.tolist())
|
||||
all_output_strs.append(self.tokenizer.decode(output_ids))
|
||||
|
||||
outputs = zip(all_output_ids, all_output_strs, all_logprobs)
|
||||
return [(output_ids, output_str, output_logprobs)
|
||||
for output_ids, output_str, output_logprobs in outputs]
|
||||
|
||||
def __del__(self):
|
||||
del self.model
|
||||
cleanup()
|
||||
|
||||
Reference in New Issue
Block a user