[Bugfix][SpecDecode] apply sampling parameters to target probabilities for consistency in rejection sampling. (#10198)
Signed-off-by: jeongin601 <0200angela@gmail.com> Signed-off-by: jeong_in.bae <jeong_in.bae@navercorp.com>
This commit is contained in:
@@ -307,28 +307,16 @@ class BatchExpansionTop1Scorer(SpeculativeScorer):
|
||||
token_ids_to_score = self._get_token_ids_to_score(
|
||||
proposal_token_ids[batch_index])
|
||||
|
||||
# Use simpler sampling parameters apart from for final token
|
||||
# (in particular don't do seeded sampling) since those sampled tokens
|
||||
# aren't used.
|
||||
# We don't replace the sampling_params in the greedy case because
|
||||
# this also controls whether the probs get modified in the sampler
|
||||
# (see use of _modify_greedy_probs_inplace there).
|
||||
sampling_params = input_seq_group_metadata.sampling_params
|
||||
non_bonus_sampling_params = DEFAULT_SIMPLE_SAMPLING_PARAMS \
|
||||
if sampling_params.temperature else sampling_params
|
||||
|
||||
target_seq_group_metadata_list: List[SequenceGroupMetadata] = []
|
||||
last_index = len(token_ids_to_score) - 1
|
||||
for i, token_ids in enumerate(token_ids_to_score):
|
||||
target_sampling_params = sampling_params if i == last_index \
|
||||
else non_bonus_sampling_params
|
||||
target_seq_group_metadata_list.append(
|
||||
self._create_single_target_seq_group_metadata(
|
||||
input_seq_group_metadata,
|
||||
input_seq_id,
|
||||
next(target_seq_ids_iter),
|
||||
token_ids,
|
||||
sampling_params=target_sampling_params,
|
||||
sampling_params=sampling_params,
|
||||
))
|
||||
|
||||
return target_seq_group_metadata_list
|
||||
|
||||
Reference in New Issue
Block a user