[Bugfix] Fix beam search candidate indexing when logprobs count varies (#47126)
Signed-off-by: chaunceyjiang <chaunceyjiang@gmail.com>
This commit is contained in:
@@ -97,12 +97,7 @@ class BeamSearchOnlineMixin(ABC):
|
||||
|
||||
output = [x[0] for x in await asyncio.gather(*tasks)]
|
||||
|
||||
new_beams = []
|
||||
# Store all new tokens generated by beam
|
||||
all_beams_token_id = []
|
||||
# Store the cumulative probability of all tokens
|
||||
# generated by beam search
|
||||
all_beams_logprob = []
|
||||
candidates = []
|
||||
# Iterate through all beam inference results
|
||||
for i, result in enumerate(output):
|
||||
current_beam = all_beams[i]
|
||||
@@ -131,65 +126,64 @@ class BeamSearchOnlineMixin(ABC):
|
||||
|
||||
if result.outputs[0].logprobs is not None:
|
||||
logprobs = result.outputs[0].logprobs[0]
|
||||
all_beams_token_id.extend(list(logprobs.keys()))
|
||||
all_beams_logprob.extend(
|
||||
[
|
||||
current_beam.cum_logprob + obj.logprob
|
||||
for obj in logprobs.values()
|
||||
]
|
||||
)
|
||||
|
||||
# Handle the token for the end of sentence (EOS)
|
||||
all_beams_token_id = np.array(all_beams_token_id)
|
||||
all_beams_logprob = np.array(all_beams_logprob)
|
||||
|
||||
if not ignore_eos:
|
||||
# Get the index position of eos token in all generated results
|
||||
eos_idx = np.where(all_beams_token_id == eos_token_id)[0]
|
||||
for idx in eos_idx:
|
||||
current_beam = all_beams[idx // logprobs_num]
|
||||
result = output[idx // logprobs_num]
|
||||
assert result.outputs[0].logprobs is not None
|
||||
logprobs_entry = result.outputs[0].logprobs[0]
|
||||
completed.append(
|
||||
BeamSearchSequence(
|
||||
orig_prompt=prompt,
|
||||
tokens=current_beam.tokens + [eos_token_id]
|
||||
if include_stop_str_in_output
|
||||
else current_beam.tokens,
|
||||
logprobs=current_beam.logprobs + [logprobs_entry],
|
||||
cum_logprob=float(all_beams_logprob[idx]),
|
||||
finish_reason="stop",
|
||||
stop_reason=eos_token_id,
|
||||
for token_id, logprob_obj in logprobs.items():
|
||||
candidate_logprob = (
|
||||
current_beam.cum_logprob + logprob_obj.logprob
|
||||
)
|
||||
)
|
||||
# After processing, set the log probability of the eos condition
|
||||
# to negative infinity.
|
||||
all_beams_logprob[eos_idx] = -np.inf
|
||||
if token_id == eos_token_id and not ignore_eos:
|
||||
completed.append(
|
||||
BeamSearchSequence(
|
||||
orig_prompt=prompt,
|
||||
tokens=current_beam.tokens + [eos_token_id]
|
||||
if include_stop_str_in_output
|
||||
else current_beam.tokens,
|
||||
logprobs=current_beam.logprobs + [logprobs],
|
||||
cum_logprob=candidate_logprob,
|
||||
finish_reason="stop",
|
||||
stop_reason=eos_token_id,
|
||||
)
|
||||
)
|
||||
else:
|
||||
candidates.append(
|
||||
(
|
||||
candidate_logprob,
|
||||
int(token_id),
|
||||
current_beam,
|
||||
logprobs,
|
||||
)
|
||||
)
|
||||
|
||||
# Processing non-EOS tokens
|
||||
# Get indices of the top beam_width probabilities
|
||||
topn_idx = np.argpartition(np.negative(all_beams_logprob), beam_width)[
|
||||
:beam_width
|
||||
]
|
||||
candidate_logprobs = np.fromiter(
|
||||
(candidate[0] for candidate in candidates),
|
||||
dtype=np.float64,
|
||||
count=len(candidates),
|
||||
)
|
||||
if len(candidates) <= beam_width:
|
||||
topn_idx = np.argsort(-candidate_logprobs)
|
||||
else:
|
||||
topn_idx = np.argpartition(
|
||||
-candidate_logprobs,
|
||||
beam_width - 1,
|
||||
)[:beam_width]
|
||||
topn_idx = topn_idx[np.argsort(-candidate_logprobs[topn_idx])]
|
||||
|
||||
new_beams = []
|
||||
for idx in topn_idx:
|
||||
current_beam = all_beams[idx // logprobs_num]
|
||||
result = output[idx // logprobs_num]
|
||||
token_id = int(all_beams_token_id[idx])
|
||||
assert result.outputs[0].logprobs is not None
|
||||
logprobs_entry = result.outputs[0].logprobs[0]
|
||||
cum_logprob, token_id, current_beam, logprobs = candidates[int(idx)]
|
||||
new_beams.append(
|
||||
BeamSearchSequence(
|
||||
orig_prompt=prompt,
|
||||
tokens=current_beam.tokens + [token_id],
|
||||
logprobs=current_beam.logprobs + [logprobs_entry],
|
||||
logprobs=current_beam.logprobs + [logprobs],
|
||||
lora_request=current_beam.lora_request,
|
||||
cum_logprob=float(all_beams_logprob[idx]),
|
||||
cum_logprob=cum_logprob,
|
||||
)
|
||||
)
|
||||
|
||||
all_beams = new_beams
|
||||
if not all_beams:
|
||||
break
|
||||
|
||||
completed.extend(all_beams)
|
||||
sorted_completed = sorted(completed, key=sort_beams_key, reverse=True)
|
||||
|
||||
Reference in New Issue
Block a user