[misc] hide best_of from engine (#9261)
Co-authored-by: Brendan Wong <bjwpokemon@gmail.com>
This commit is contained in:
co-authored by
Brendan Wong
parent
94bf9ae4e9
commit
cbc2ef5529
@@ -49,7 +49,7 @@ class ModelInputForTPU(ModelRunnerInputBase):
|
||||
t: torch.Tensor
|
||||
p: torch.Tensor
|
||||
num_samples: int
|
||||
best_of: List[int]
|
||||
n: List[int]
|
||||
seq_groups: List[List[int]]
|
||||
is_first_multi_step: bool = True
|
||||
is_last_step: bool = True
|
||||
@@ -65,7 +65,7 @@ class ModelInputForTPU(ModelRunnerInputBase):
|
||||
"t": self.t,
|
||||
"p": self.p,
|
||||
"num_samples": self.num_samples,
|
||||
"best_of": self.best_of,
|
||||
"n": self.n,
|
||||
"seq_groups": self.seq_groups,
|
||||
"is_first_multi_step": self.is_first_multi_step,
|
||||
"is_last_step": self.is_last_step,
|
||||
@@ -435,7 +435,7 @@ class TPUModelRunner(ModelRunnerBase[ModelInputForTPU]):
|
||||
assert len(seq_group_metadata_list) > 0
|
||||
t = []
|
||||
p = []
|
||||
best_of = []
|
||||
n = []
|
||||
for seq_group_metadata in seq_group_metadata_list:
|
||||
sampling_params = seq_group_metadata.sampling_params
|
||||
t.append(sampling_params.temperature)
|
||||
@@ -448,11 +448,11 @@ class TPUModelRunner(ModelRunnerBase[ModelInputForTPU]):
|
||||
raise NotImplementedError(
|
||||
"Top-k sampling is currently disabled for the TPU backend "
|
||||
"due to performance issues.")
|
||||
if sampling_params.best_of > _MAX_NUM_SAMPLES:
|
||||
if sampling_params.n > _MAX_NUM_SAMPLES:
|
||||
raise NotImplementedError(
|
||||
f"Best of > {_MAX_NUM_SAMPLES} is not supported by the TPU "
|
||||
"backend.")
|
||||
best_of.append(sampling_params.best_of)
|
||||
n.append(sampling_params.n)
|
||||
if sampling_params.logprobs is not None:
|
||||
raise NotImplementedError(
|
||||
"logprobs is not currently supported by the TPU backend.")
|
||||
@@ -465,7 +465,7 @@ class TPUModelRunner(ModelRunnerBase[ModelInputForTPU]):
|
||||
num_seqs = len(seq_group_metadata.seq_data)
|
||||
t += [t[-1]] * (num_seqs - 1)
|
||||
p += [p[-1]] * (num_seqs - 1)
|
||||
best_of += [best_of[-1]] * (num_seqs - 1)
|
||||
n += [n[-1]] * (num_seqs - 1)
|
||||
|
||||
num_paddings = padded_batch_size - len(t)
|
||||
t += [1.0] * num_paddings
|
||||
@@ -473,7 +473,7 @@ class TPUModelRunner(ModelRunnerBase[ModelInputForTPU]):
|
||||
|
||||
t = torch.tensor(t, dtype=torch.float32, device="cpu")
|
||||
p = torch.tensor(p, dtype=torch.float32, device="cpu")
|
||||
return t, p, best_of
|
||||
return t, p, n
|
||||
|
||||
def prepare_model_input(
|
||||
self,
|
||||
@@ -493,8 +493,8 @@ class TPUModelRunner(ModelRunnerBase[ModelInputForTPU]):
|
||||
inputs = self._prepare_decode(seq_group_metadata_list)
|
||||
input_tokens, input_positions, attn_metadata, input_lens = inputs
|
||||
padded_batch_size = input_tokens.shape[0]
|
||||
t, p, best_of = self._prepare_sample(seq_group_metadata_list,
|
||||
padded_batch_size)
|
||||
t, p, n = self._prepare_sample(seq_group_metadata_list,
|
||||
padded_batch_size)
|
||||
num_samples = _MAX_NUM_SAMPLES if is_prompt else 1
|
||||
|
||||
seq_groups = [
|
||||
@@ -502,8 +502,7 @@ class TPUModelRunner(ModelRunnerBase[ModelInputForTPU]):
|
||||
for metadata in seq_group_metadata_list
|
||||
]
|
||||
return ModelInputForTPU(input_tokens, input_positions, attn_metadata,
|
||||
input_lens, t, p, num_samples, best_of,
|
||||
seq_groups)
|
||||
input_lens, t, p, num_samples, n, seq_groups)
|
||||
|
||||
def make_model_input_from_broadcasted_tensor_dict(
|
||||
self, tensor_dict: Dict[str, Any]) -> ModelInputForTPU:
|
||||
@@ -609,7 +608,7 @@ class TPUModelRunner(ModelRunnerBase[ModelInputForTPU]):
|
||||
assert len(seq_ids) == 1
|
||||
seq_id = seq_ids[0]
|
||||
seq_outputs = []
|
||||
for j in range(model_input.best_of[i]):
|
||||
for j in range(model_input.n[i]):
|
||||
next_token_id = next_token_ids[i][j]
|
||||
seq_outputs.append(
|
||||
SequenceOutput(seq_id, next_token_id,
|
||||
|
||||
Reference in New Issue
Block a user